/[pdpsoft]/trunk/fetchcrl/fetch-crl3.pl
ViewVC logotype

Contents of /trunk/fetchcrl/fetch-crl3.pl

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1758 - (show annotations) (download) (as text)
Fri Jun 11 15:39:04 2010 UTC (11 years, 5 months ago) by davidg
Original Path: fetchcrl/fetch-crl3.pl
File MIME type: text/x-prolog
File size: 13340 byte(s)
Initial import of fetch-crl3

1 #! /usr/bin/perl -w
2 #
3 # @(#)$Id$
4 #
5 # Copyright 2010 David Groep, Nationaal instituut voor
6 # subatomaire fysica NIKHEF
7 #
8 # Licensed under the Apache License, Version 2.0 (the "License");
9 # you may not use this file except in compliance with the License.
10 # You may obtain a copy of the License at
11 #
12 # http://www.apache.org/licenses/LICENSE-2.0
13 #
14 # Unless required by applicable law or agreed to in writing, software
15 # distributed under the License is distributed on an "AS IS" BASIS,
16 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 # See the License for the specific language governing permissions and
18 # limitations under the License.
19 #
20 #
21 package main;
22
23 use strict;
24 use Getopt::Long qw(:config no_ignore_case bundling);
25 use POSIX;
26 eval { require LWP or die; }; $@ and die "Please install libwww-perl (LWP)\n";
27
28 # import modules that are needed but still external
29 # (the installed version may have these packages embedded in-line)
30 #
31 require ConfigTiny and import ConfigTiny unless defined &ConfigTiny::new;
32 require TrustAnchor and import TrustAnchor unless defined &TrustAnchor::new;
33 require CRLWriter and import CRLWriter unless defined &CRLWriter::new;
34 require FCLog and import FCLog unless defined &FCLog::new;
35 require OSSL and import OSSL unless defined &OSSL::new;
36 require CRL and import CRL unless defined &CRL::new;
37
38 my $use_DataDumper = eval { require Data::Dumper; };
39 my $use_IOSelect = eval { require IO::Select; };
40
41 use vars qw/ $log $cnf /;
42
43
44 # ###########################################################################
45 #
46 #
47 ($cnf,$log) = &init_configuration();
48
49 # verify local installation sanity for loaded modules
50 $::log->getverbose > 6 and ! $use_DataDumper and
51 $::log->err("Cannot set verbosity higher than 6 without Data::Dumper") and
52 exit(1);
53 $::cnf->{_}->{parallelism} and ! $use_IOSelect and
54 $::log->err("Cannot use parallel retrieval without IO::Select") and
55 exit(1);
56
57 $use_DataDumper and $::log->verb(7,Data::Dumper::Dumper($cnf));
58
59 # set safe path if so requested
60 $cnf->{_}->{path} and $ENV{"PATH"} = $cnf->{_}->{path} and
61 $::log->verb(5,"Set PATH to",$ENV{"PATH"});
62
63 # wait up to randomwait seconds to spread download load
64 $cnf->{_}->{randomwait} and do {
65 my $wtime = int(rand($cnf->{_}->{randomwait}));
66 $::log->verb(2,"Sleeping $wtime seconds before continuing");
67 sleep($wtime);
68 };
69
70
71 # the list of trust anchors to process comes from the command line and
72 # all files in the infodir that are metadata or crl urls
73 # in the next phase, the suffix will be stripped and the info file
74 # when present preferred over the crlurl
75 #
76 my @metafiles = @ARGV;
77 $::cnf->{_}->{"infodir"} and do {
78 foreach my $fn (
79 map { glob ( $::cnf->{_}->{"infodir"} . "/$_" ); } "*.info", "*.crl_url"
80 ) {
81 $fn =~ /.*\/([^\/]+)(\.crl_url|\.info)$/;
82 push @metafiles, $1 unless grep /$1/,@metafiles or not defined $1;
83 }
84 };
85
86 @metafiles or
87 $log->err("No trust anchors to process") and exit($log->exitstatus);
88
89 if ( $::cnf->{_}->{parallelism} ) {
90 &parallel_metafiles($::cnf->{_}->{parallelism}, @metafiles);
91 } else {
92 &process_metafiles( @metafiles );
93 }
94
95 $log->flush;
96 exit($log->exitstatus);
97
98
99 # ###########################################################################
100 #
101 #
102 sub init_configuration() {
103 my ($cnf,$log);
104
105 my ($configfile,$agingtolerance,$infodir,$statedir,$cadir,$httptimeout);
106 my ($output);
107 my @formats;
108 my $verbosity;
109 my $quiet=0;
110 my $help=0;
111 my $debuglevel;
112 my $parallelism=0;
113 my $randomwait;
114
115 $log = FCLog->new("qualified");
116
117 &GetOptions(
118 "c|config=s" => \$configfile,
119 "l|infodir=s" => \$infodir,
120 "cadir=s" => \$cadir,
121 "s|statedir=s" => \$statedir,
122 "T|httptimeout=i" => \$httptimeout,
123 "o|output=s" => \$output,
124 "format=s@" => \@formats,
125 "v|verbose+" => \$verbosity,
126 "h|help+" => \$help,
127 "q|quiet+" => \$quiet,
128 "d|debug+" => \$debuglevel,
129 "p|parallelism=i" => \$parallelism,
130 "a|agingtolerance=i" => \$agingtolerance,
131 "r|randomwait=i" => \$randomwait,
132 ) or &help and exit(1);
133
134 $help and &help and exit(0);
135
136 $configfile ||= ( -e "/etc/fetch-crl.cnf" and "/etc/fetch-crl.cnf" );
137 ($quiet > 0) and $verbosity = -$quiet;
138
139 $cnf = ConfigTiny->new();
140 $configfile and
141 $cnf->read($configfile) || die "Invalid config file $configfile:\n " .
142 $cnf->errstr . "\n";
143
144 # command-line option overrides
145 $cnf->{_}->{agingtolerance} = $agingtolerance if defined $agingtolerance;
146 $cnf->{_}->{infodir} = $infodir if defined $infodir;
147 $cnf->{_}->{cadir} = $cadir if defined $cadir;
148 $cnf->{_}->{statedir} = $statedir if defined $statedir;
149 $cnf->{_}->{httptimeout} = $httptimeout if defined $httptimeout;
150 $cnf->{_}->{verbosity} = $verbosity if defined $verbosity;
151 $cnf->{_}->{debuglevel} = $debuglevel if defined $debuglevel;
152 $cnf->{_}->{output} = $output if defined $output;
153 $cnf->{_}->{formats} = join "",@formats if @formats;
154 $cnf->{_}->{parallelism} = $parallelism if $parallelism;
155 $cnf->{_}->{randomwait} = $randomwait if defined $randomwait;
156
157 # key default values
158 defined $cnf->{_}->{version} or $cnf->{_}->{version} = "3+";
159 defined $cnf->{_}->{packager} or $cnf->{_}->{packager} = "EUGridPMA";
160 defined $cnf->{_}->{openssl} or $cnf->{_}->{openssl} = "openssl";
161 defined $cnf->{_}->{agingtolerance} or $cnf->{_}->{agingtolerance} ||= 24;
162 defined $cnf->{_}->{infodir} or $cnf->{_}->{infodir} = '/etc/grid-security/certificates';
163 defined $cnf->{_}->{output} or $cnf->{_}->{output} = $cnf->{_}->{infodir};
164 defined $cnf->{_}->{cadir} or $cnf->{_}->{cadir} = $cnf->{_}->{infodir};
165 defined $cnf->{_}->{statedir} or $cnf->{_}->{statedir} = "/var/cache/fetch-crl" if -d "/var/cache/fetch-crl" and -w "/var/cache/fetch-crl";
166 defined $cnf->{_}->{formats} or $cnf->{_}->{formats} = "openssl";
167 defined $cnf->{_}->{opensslmode} or $cnf->{_}->{opensslmode} = "dual";
168 defined $cnf->{_}->{httptimeout} or $cnf->{_}->{httptimeout} = 120;
169 defined $cnf->{_}->{nametemplate_der} or
170 $cnf->{_}->{nametemplate_der} = "\@ANCHORNAME\@.\@R\@.crl";
171 defined $cnf->{_}->{nametemplate_pem} or
172 $cnf->{_}->{nametemplate_pem} = "\@ANCHORNAME\@.\@R\@.crl.pem";
173 defined $cnf->{_}->{catemplate} or
174 $cnf->{_}->{catemplate} = "\@ALIAS\@.pem".
175 "\@ALIAS\@.\@R\@\@ANCHORNAME\@.\@R\@";
176
177 $cnf->{_}->{nonssverify} ||= 0;
178 $cnf->{_}->{nocache} ||= 0;
179 $cnf->{_}->{verbosity} ||= 0;
180 $cnf->{_}->{debuglevel} ||= 0;
181
182 $cnf->{_}->{stateless} and delete $cnf->{_}->{statedir};
183
184 # expand array keys in config
185 defined $cnf->{_}->{formats} and
186 @{$cnf->{_}->{formats_}} = split(/[;,\s]+/,$cnf->{_}->{formats});
187
188 # sanity check on configuration
189 $cnf->{_}->{statedir} and ! -d $cnf->{_}->{statedir} and
190 die "Invalid state directory " . $cnf->{_}->{statedir} . "\n";
191 $cnf->{_}->{cadir} ||= ".";
192 $cnf->{_}->{infodir} and ! -d $cnf->{_}->{infodir} and
193 die "Invalid meta-data directory ".$cnf->{_}->{infodir}."\n";
194
195 # initialize logging
196 $log->flush;
197 $cnf->{_}->{logmode} and $log->destremove("qualified") and do {
198 foreach ( split(/[,]+/,$cnf->{_}->{logmode}) ) {
199 if ( /^syslog$/ ) { $log->destadd($_,$cnf->{_}->{syslogfacility}); }
200 elsif ( /^(direct|qualified|cache)$/ ) { $log->destadd($_); }
201 else { die "Invalid log destination $_, exiting.\n"; }
202 }
203 };
204 $log->setverbose($cnf->{_}->{verbosity});
205 $log->setdebug($cnf->{_}->{debuglevel});
206
207 return ($cnf,$log);
208 }
209
210 # ###########################################################################
211 #
212 #
213 sub help() {
214 (my $name = $0) =~ s/.*\///;
215 print <<EOHELP;
216 The fetch-crl utility will retrieve certificate revocation lists (CRLs) for
217 a set of installed trust anchors, based on crl_url files or IGTF-style info
218 files. It will install these for use with OpenSSL, NSS or third-party tools.
219
220 Usage: $name [-c|--config configfile] [-l|--infodir path]
221 [--cadir path] [-s|--statedir path] [-o|--output path] [--format \@formats]
222 [-T|--httptimeout seconds] [-p|--parallelism n]
223 [-a|--agingtolerance hours] [-r|--randomwait seconds]
224 [-v|--verbose] [-h|--help] [-q|--quiet] [-d|--debug level]
225
226 Options:
227 -c | --config path
228 Read configuration data from path, default: /etc/fetch-crl.cnf
229 -l | --infodir path
230 Location of the trust anchor meta-data files (crl_url or info),
231 default: /etc/grid-security/certificates
232 --cadir path
233 Location of the trust anchors (default to infodir)
234 -s | --statedir path
235 Location of the historic state data (for caching and delayed-warning)
236 -T | --httptimeout sec
237 Maximum time in seconds to wait for retrieval or a single URL
238 -o | --output path
239 Location of the CRLs written (global default, defaults to infodir
240 --format \@formats
241 Format(s) in which the CRLs will be written (openssl, pem, der, nss)
242 -v | --verbose
243 Become more talkative
244 -q | --quiet
245 Become really quiet (overrides verbosity)
246 -p | --parallelism n
247 Run up to n parallel trust anchor retrieval processes
248 -a | --agingtolerance hours
249 Be quiet for up to hours hours before raising an error. Until
250 the tolerance has passed, only warnings are raised
251 -r | --randomwait seconds
252 Introduce a random delay of up to seconds seconds before starting
253 any retrieval processes
254 -h | --help
255 This help text
256
257 EOHELP
258
259 return 1;
260 }
261
262 # ###########################################################################
263 #
264 #
265 sub process_metafiles(@) {
266 my @metafiles = @_;
267
268 foreach my $f ( @metafiles ) {
269 my $ta = TrustAnchor->new();
270 $cnf->{_}->{"infodir"} and $ta->setInfodir($cnf->{_}->{"infodir"});
271 $ta->loadAnchor($f) or next;
272 $ta->saveLogMode() and $ta->setLogMode();
273 $ta->loadCAfiles() or next;
274 $ta->loadState() or next;
275 $ta->retrieve or next;
276 $ta->verifyAndConvertCRLs or next;
277
278 my $writer = CRLWriter->new($ta);
279 $writer->writeall() or next;
280 $ta->saveState() or next;
281 $ta->restoreLogMode();
282 }
283
284 return 1;
285 }
286
287 sub parallel_metafiles($@) {
288 my $parallelism = shift;
289 my @metafiles = @_;
290
291 my %pids = (); # file handle by processID
292 my %metafile_by_fh = (); # reverse map
293 my $readset = new IO::Select();
294 my %logoutput = ();
295
296 $| = 1;
297
298 $::log->verb(2,"starting up to $parallelism worker processes");
299
300 while ( @metafiles or scalar keys %pids ) {
301 # loop until we have started all possible retrievals AND have
302 # collected all possible output
303
304 ( @metafiles and (scalar keys %pids < $parallelism) ) and do {
305 # we have metafiles left, and have spare process slots
306 my $metafile = shift @metafiles;
307
308
309 $logoutput{$metafile} = "";
310
311 my $cout;
312 my $cpid = open $cout, "-|";
313 defined $cpid and defined $cout or
314 $::log->err("Cannot fork ($metafile): $!") and next;
315
316 $::log->verb(5,"LOOP: starting process $cpid for $metafile");
317
318 if ( $cpid == 0 ) { # I'm the child that should care for $metafile
319 $0 = "fetch-crl worker $metafile";
320 $::log->cleanse();
321 $::log->destadd("qualified");
322 &process_metafiles($metafile);
323 $::log->flush;
324 exit($::log->exitstatus);
325 } else { # parent
326 $pids{$cpid} = $cout;
327 $readset->add($cout);
328 $metafile_by_fh{$cout} = $metafile;
329 }
330 };
331
332 # do a select loop over the outstanding requests to collect messages
333 # if we are in the process of starting more processes, we just
334 # briefly poll out pending output so as not to have blocking
335 # children, but if we have started as many children as we ought to
336 # we put in a longer timeout -- any output on a handle will
337 # get us out of the select and into flushing mode again
338 my $timeout = (@metafiles && (scalar keys %pids < $parallelism) ? 0.1:1);
339
340 $::log->verb(6,"PLOOP: select with timeout $timeout");
341 my ( $rh_set ) = IO::Select->select($readset, undef, undef, $timeout);
342
343 foreach my $fh ( @$rh_set ) {
344 my $metafile = $metafile_by_fh{$fh};
345 # we know there is at least one byte to read, but also that
346 # any client sends complete
347 while (1) {
348 my $char;
349 my $length = sysread $fh, $char, 1;
350 if ( $length ) {
351 $logoutput{$metafile} .= $char;
352 $char eq "\n" and last;
353 } else {
354 #expected a char but got eof
355 $readset->remove($fh);
356 close($fh);
357 map {
358 $pids{$_} == $fh and
359 waitpid($_,WNOHANG) and
360 delete $pids{$_} and
361 $::log->verb(5,"Collected pid $_ (rc=$?),",
362 length($logoutput{$metafile}),"bytes log output");
363 } keys %pids;
364 last;
365 }
366 }
367 }
368 }
369
370 # log out all collected log data from our children
371 foreach my $metafile ( sort keys %logoutput ) {
372 foreach my $line ( split(/\n/,$logoutput{$metafile}) ) {
373 $line =~ /^ERROR\s+(.*)$/ and $::log->err($1);
374 $line =~ /^WARN\s+(.*)$/ and $::log->warn($1);
375 $line =~ /^VERBOSE\((\d+)\)\s+(.*)$/ and $::log->verb($1,$2);
376 $line =~ /^DEBUG\((\d+)\)\s+(.*)$/ and $::log->debug($1,$2);
377 }
378 }
379
380 return 1;
381 }

Properties

Name Value
svn:executable *

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28