/[pdpsoft]/nl.nikhef.pdp.fetchcrl/trunk/fetch-crl3.pl.cin
ViewVC logotype

Contents of /nl.nikhef.pdp.fetchcrl/trunk/fetch-crl3.pl.cin

Parent Directory Parent Directory | Revision Log Revision Log


Revision 3335 - (show annotations) (download)
Thu Nov 25 08:34:59 2021 UTC (6 months, 4 weeks ago) by davidg
File size: 17343 byte(s)
clarify version in built fetch-crl composite

1 #! /usr/bin/perl -w
2 #
3 # @(#)$Id$
4 # build version @VERSION@, release @RELEASE@
5 #
6 # Copyright 2010-2021 David Groep, Nationaal instituut voor
7 # subatomaire fysica NIKHEF
8 #
9 # Licensed under the Apache License, Version 2.0 (the "License");
10 # you may not use this file except in compliance with the License.
11 # You may obtain a copy of the License at
12 #
13 # http://www.apache.org/licenses/LICENSE-2.0
14 #
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20 #
21 #
22 package main;
23
24 use strict;
25 use Getopt::Long qw(:config no_ignore_case bundling);
26 use POSIX;
27 eval { require LWP or die; }; $@ and die "Please install libwww-perl (LWP)\n";
28
29 my $sccsid = '@(#)fetch-crl3 version @VERSION@';
30
31 # import modules that are needed but still external
32 # (the installed version may have these packages embedded in-line)
33 #
34 require ConfigTiny and import ConfigTiny unless defined &ConfigTiny::new;
35 require TrustAnchor and import TrustAnchor unless defined &TrustAnchor::new;
36 require CRLWriter and import CRLWriter unless defined &CRLWriter::new;
37 require FCLog and import FCLog unless defined &FCLog::new;
38 require OSSL and import OSSL unless defined &OSSL::new;
39 require CRL and import CRL unless defined &CRL::new;
40
41 my $use_DataDumper = eval { require Data::Dumper; };
42 my $use_IOSelect = eval { require IO::Select; };
43
44 use vars qw/ $log $cnf /;
45
46
47 # ###########################################################################
48 #
49 #
50 ($cnf,$log) = &init_configuration();
51
52 # use Net::INET6Glue if so requested (is not a default module)
53 if ( $cnf->{_}->{inet6glue} ) {
54 eval { require Net::INET6Glue::INET_is_INET6 or die; };
55 $@ and die "Please install Net::INET6Glue before enabling inet6glue config\n";
56 }
57
58 # verify local installation sanity for loaded modules
59 $::log->getverbose > 6 and ! $use_DataDumper and
60 $::log->err("Cannot set verbosity higher than 6 without Data::Dumper") and
61 exit(1);
62 $::cnf->{_}->{parallelism} and ! $use_IOSelect and
63 $::log->err("Cannot use parallel retrieval without IO::Select") and
64 exit(1);
65
66 $use_DataDumper and $::log->verb(7,Data::Dumper::Dumper($cnf));
67
68 # set safe path if so requested
69 $cnf->{_}->{path} and $ENV{"PATH"} = $cnf->{_}->{path} and
70 $::log->verb(5,"Set PATH to",$ENV{"PATH"});
71
72 # set rcmode if present in config
73 defined $cnf->{_}->{rcmode} and do {
74 $::log->verb(4,"Setting exit status mode to ".$cnf->{_}->{rcmode});
75 $::log->setrcmode($cnf->{_}->{rcmode}) or exit($log->exitstatus);
76 $::log->verb(2,"Exit status mode is set to ".$cnf->{_}->{rcmode});
77 };
78
79 # wait up to randomwait seconds to spread download load
80 $cnf->{_}->{randomwait} and do {
81 my $wtime = int(rand($cnf->{_}->{randomwait}));
82 $::log->verb(2,"Sleeping $wtime seconds before continuing");
83 sleep($wtime);
84 };
85
86
87 # the list of trust anchors to process comes from the command line and
88 # all files in the infodir that are metadata or crl urls
89 # in the next phase, the suffix will be stripped and the info file
90 # when present preferred over the crlurl
91 #
92 my @metafiles = @ARGV;
93 $::cnf->{_}->{"infodir"} and do {
94 foreach my $fn (
95 map { glob ( $::cnf->{_}->{"infodir"} . "/$_" ); } "*.info", "*.crl_url"
96 ) {
97 next if $::cnf->{_}->{nosymlinks} and -l $fn;
98 $fn =~ /.*\/([^\/]+)(\.crl_url|\.info)$/;
99 push @metafiles, $1 unless grep /^$1$/,@metafiles or not defined $1;
100 }
101 };
102
103 @metafiles or
104 $log->warn("No trust anchors to process") and exit($log->exitstatus);
105
106 if ( $::cnf->{_}->{parallelism} ) {
107 &parallel_metafiles($::cnf->{_}->{parallelism}, @metafiles);
108 } else {
109 &process_metafiles( @metafiles );
110 }
111
112 # run any post-processing
113 if ( $::cnf->{_}->{"postexec"} ) {
114 my @args = ( $::cnf->{_}->{"postexec"},
115 "v1", "global",
116 $::cnf->{_}->{"infodir"}, $::cnf->{_}->{"cadir"}, $::cnf->{_}->{"output"} );
117 $::log->verb(2,"Executing global postscript @args");
118 my $postrc = system(@args);
119 if ( $postrc == -1 ) {
120 $::log->err("Cannot execute global postexec program: $!");
121 } elsif ( $postrc > 0 ) {
122 $::log->err("Global postexec program returned error code ".($? >> 8));
123 }
124 }
125
126 $log->flush;
127 exit($log->exitstatus);
128
129
130 # ###########################################################################
131 #
132 #
133 sub init_configuration() {
134 my ($cnf,$log);
135
136 my ($configfile,$agingtolerance,$infodir,$statedir,$cadir,$httptimeout);
137 my ($output);
138 my @formats;
139 my $verbosity;
140 my $quiet=0;
141 my $help=0;
142 my $showversion=0;
143 my $debuglevel;
144 my $parallelism=0;
145 my $randomwait;
146 my $nosymlinks;
147 my $cfgdir;
148 my $inet6glue=0;
149 my %directives;
150
151 $log = FCLog->new("qualified");
152
153 &GetOptions(
154 "c|config=s" => \$configfile,
155 "l|infodir=s" => \$infodir,
156 "cadir=s" => \$cadir,
157 "s|statedir=s" => \$statedir,
158 "cfgdir=s" => \$cfgdir,
159 "T|httptimeout=i" => \$httptimeout,
160 "o|output=s" => \$output,
161 "format=s@" => \@formats,
162 "define=s" => \%directives,
163 "v|verbose+" => \$verbosity,
164 "h|help+" => \$help,
165 "V|version+" => \$showversion,
166 "q|quiet+" => \$quiet,
167 "d|debug+" => \$debuglevel,
168 "p|parallelism=i" => \$parallelism,
169 "nosymlinks+" => \$nosymlinks,
170 "a|agingtolerance=i" => \$agingtolerance,
171 "r|randomwait=i" => \$randomwait,
172 "inet6glue+" => \$inet6glue,
173 ) or &help and exit(1);
174
175 $help and &help and exit(0);
176 $showversion and &showversion and exit(0);
177
178 $configfile ||= ( -e "/etc/fetch-crl.conf" and "/etc/fetch-crl.conf" );
179 $configfile ||= ( -e "/etc/fetch-crl.cnf" and "/etc/fetch-crl.cnf" );
180
181 $cnf = ConfigTiny->new();
182 $configfile and
183 $cnf->read($configfile) || die "Invalid config file $configfile:\n " .
184 $cnf->errstr . "\n";
185
186 ( defined $cnf->{_}->{cfgdir} and $cfgdir = $cnf->{_}->{cfgdir} )
187 unless defined $cfgdir;
188 $cfgdir ||= "/etc/fetch-crl.d";
189 if ( defined $cfgdir and -d $cfgdir and opendir(my $dh,$cfgdir) ) {
190 while ( my $fn = readdir $dh ) {
191 -f "$cfgdir/$fn" and -r "$cfgdir/$fn" and $cnf->read("$cfgdir/$fn");
192 }
193 close $dh;
194 }
195
196 # add defined from the command line to the configuration, to the
197 # main section _ thereof unless there is a colon in the key
198 foreach my $k ( keys %directives ) {
199 my $section ="_";
200 my $dvalue = $directives{$k};
201 if ( $k =~ m/(\w+):(.*)/ ) {
202 $section = $1;
203 $k=$2;
204 }
205 $cnf->{$section}->{$k} = $dvalue;
206 }
207
208 # command-line option overrides
209 $cnf->{_}->{agingtolerance} = $agingtolerance if defined $agingtolerance;
210 $cnf->{_}->{infodir} = $infodir if defined $infodir;
211 $cnf->{_}->{cadir} = $cadir if defined $cadir;
212 $cnf->{_}->{statedir} = $statedir if defined $statedir;
213 $cnf->{_}->{httptimeout} = $httptimeout if defined $httptimeout;
214 $cnf->{_}->{verbosity} = $verbosity if defined $verbosity;
215 $cnf->{_}->{debuglevel} = $debuglevel if defined $debuglevel;
216 $cnf->{_}->{output} = $output if defined $output;
217 $cnf->{_}->{formats} = join "\001",@formats if @formats;
218 $cnf->{_}->{parallelism} = $parallelism if $parallelism;
219 $cnf->{_}->{randomwait} = $randomwait if defined $randomwait;
220 $cnf->{_}->{nosymlinks} = $nosymlinks if defined $nosymlinks;
221 $cnf->{_}->{inet6glue} = $inet6glue if $inet6glue;
222
223 # deal with interaction of verbosity in logfile and quiet option
224 # since a noquiet config option can cancel it
225 if ( not defined $cnf->{_}->{noquiet} ) {
226 if ( $quiet == 1) { $cnf->{_}->{verbosity} = -1; }
227 } else {
228 if ( $quiet >= 2) { $cnf->{_}->{verbosity} = -1; }
229 }
230
231 # key default values
232 defined $cnf->{_}->{version} or $cnf->{_}->{version} = "3+";
233 defined $cnf->{_}->{packager} or $cnf->{_}->{packager} = "EUGridPMA";
234 defined $cnf->{_}->{openssl} or $cnf->{_}->{openssl} = "openssl";
235 defined $cnf->{_}->{agingtolerance} or $cnf->{_}->{agingtolerance} ||= 24;
236 defined $cnf->{_}->{infodir} or $cnf->{_}->{infodir} = '/etc/grid-security/certificates';
237 defined $cnf->{_}->{output} or $cnf->{_}->{output} = $cnf->{_}->{infodir};
238 defined $cnf->{_}->{cadir} or $cnf->{_}->{cadir} = $cnf->{_}->{infodir};
239 defined $cnf->{_}->{statedir} or $cnf->{_}->{statedir} = "/var/cache/fetch-crl" if -d "/var/cache/fetch-crl" and -w "/var/cache/fetch-crl";
240 defined $cnf->{_}->{formats} or $cnf->{_}->{formats} = "openssl";
241 defined $cnf->{_}->{opensslmode} or $cnf->{_}->{opensslmode} = "dual";
242 defined $cnf->{_}->{httptimeout} or $cnf->{_}->{httptimeout} = 120;
243 defined $cnf->{_}->{expirestolerance} or $cnf->{_}->{expirestolerance} = (7*60*60); # at least 7 hrs should nextUpdate be beyond the cache FreshUntil
244 defined $cnf->{_}->{maxcachetime} or $cnf->{_}->{maxcachetime} = (4*24*60*60); # arbitrarily set it at 4 days
245 defined $cnf->{_}->{nametemplate_der} or
246 $cnf->{_}->{nametemplate_der} = "\@ANCHORNAME\@.\@R\@.crl";
247 defined $cnf->{_}->{nametemplate_pem} or
248 $cnf->{_}->{nametemplate_pem} = "\@ANCHORNAME\@.\@R\@.crl.pem";
249 defined $cnf->{_}->{catemplate} or
250 $cnf->{_}->{catemplate} = "\@ALIAS\@.pem\001".
251 "\@ALIAS\@.\@R\@\001\@ANCHORNAME\@.\@R\@";
252
253 $cnf->{_}->{nonssverify} ||= 0;
254 $cnf->{_}->{nocache} ||= 0;
255 $cnf->{_}->{nosymlinks} ||= 0;
256 $cnf->{_}->{verbosity} ||= 0;
257 $cnf->{_}->{debuglevel} ||= 0;
258 $cnf->{_}->{inet6glue} ||= 0;
259
260 $cnf->{_}->{stateless} and delete $cnf->{_}->{statedir};
261
262 # expand array keys in config
263 defined $cnf->{_}->{formats} and
264 @{$cnf->{_}->{formats_}} = split(/[\001;,\s]+/,$cnf->{_}->{formats});
265
266 # sanity check on configuration
267 $cnf->{_}->{statedir} and ! -d $cnf->{_}->{statedir} and
268 die "Invalid state directory " . $cnf->{_}->{statedir} . "\n";
269 $cnf->{_}->{infodir} and ! -d $cnf->{_}->{infodir} and
270 die "Invalid meta-data directory ".$cnf->{_}->{infodir}."\n";
271
272 # initialize logging
273 $log->flush;
274 $cnf->{_}->{logmode} and $log->destremove("qualified") and do {
275 foreach ( split(/[,\001]+/,$cnf->{_}->{logmode}) ) {
276 if ( /^syslog$/ ) { $log->destadd($_,$cnf->{_}->{syslogfacility}); }
277 elsif ( /^(direct|qualified|cache)$/ ) { $log->destadd($_); }
278 else { die "Invalid log destination $_, exiting.\n"; }
279 }
280 };
281 $log->setverbose($cnf->{_}->{verbosity});
282 $log->setdebug($cnf->{_}->{debuglevel});
283
284 return ($cnf,$log);
285 }
286
287 # ###########################################################################
288 #
289 #
290 sub showversion() {
291 (my $name = $0) =~ s/.*\///;
292 print "$name version @VERSION@\n";
293 return 1;
294 }
295
296 sub help() {
297 (my $name = $0) =~ s/.*\///;
298 print <<EOHELP;
299 The fetch-crl utility will retrieve certificate revocation lists (CRLs) for
300 a set of installed trust anchors, based on crl_url files or IGTF-style info
301 files. It will install these for use with OpenSSL, NSS or third-party tools.
302
303 Usage: $name [-c|--config configfile] [-l|--infodir path]
304 [--cadir path] [-s|--statedir path] [-o|--output path] [--format \@formats]
305 [-T|--httptimeout seconds] [-p|--parallelism n] [--nosymlinks]
306 [-a|--agingtolerance hours] [-r|--randomwait seconds]
307 [-v|--verbose] [-h|--help] [-q|--quiet] [-d|--debug level]
308
309 Options:
310 -c | --config path
311 Read configuration data from path, default: /etc/fetch-crl.conf
312 -l | --infodir path
313 Location of the trust anchor meta-data files (crl_url or info),
314 default: /etc/grid-security/certificates
315 --cadir path
316 Location of the trust anchors (default to infodir)
317 -s | --statedir path
318 Location of the historic state data (for caching and delayed-warning)
319 -T | --httptimeout sec
320 Maximum time in seconds to wait for retrieval or a single URL
321 -o | --output path
322 Location of the CRLs written (global default, defaults to infodir
323 --format \@formats
324 Format(s) in which the CRLs will be written (openssl, pem, der, nss)
325 --nosymlinks
326 Do not include meta-data files that are symlinks
327 -v | --verbose
328 Become more talkative
329 -q | --quiet
330 Become really quiet (overrides verbosity)
331 -p | --parallelism n
332 Run up to n parallel trust anchor retrieval processes
333 -a | --agingtolerance hours
334 Be quiet for up to hours hours before raising an error. Until
335 the tolerance has passed, only warnings are raised
336 -r | --randomwait seconds
337 Introduce a random delay of up to seconds seconds before starting
338 any retrieval processes
339 -h | --help
340 This help text
341
342 Version: @VERSION@
343 EOHELP
344
345 return 1;
346 }
347
348 # ###########################################################################
349 #
350 #
351 sub process_metafiles(@) {
352 my @metafiles = @_;
353
354 foreach my $f ( @metafiles ) {
355 my $ta = TrustAnchor->new();
356 $cnf->{_}->{"infodir"} and $ta->setInfodir($cnf->{_}->{"infodir"});
357 $ta->loadAnchor($f) or next;
358 $ta->saveLogMode() and $ta->setLogMode();
359 $ta->loadState() or next;
360
361 # using the HASH in the CA filename templates requires the CRL
362 # is retrieved first to determinte the hash
363 if ( $cnf->{_}->{"catemplate"} =~ /\@HASH\@/ ) {
364 $ta->retrieve or next;
365 $ta->loadCAfiles() or next;
366 } else {
367 $ta->loadCAfiles() or next;
368 $ta->retrieve or next;
369 }
370
371 $ta->verifyAndConvertCRLs or next;
372
373 my $writer = CRLWriter->new($ta);
374 $writer->writeall() or next;
375 $ta->saveState() or next;
376
377 if ( $::cnf->{$ta->{"alias"}}->{"postexec"} ) {
378 my @args = ( $::cnf->{$ta->{"alias"}}->{"postexec"},
379 "v1", "ta",
380 $ta->{"alias"}, $ta->{"filename"}, $::cnf->{_}->{"cadir"}, $::cnf->{_}->{"output"} );
381 $::log->verb(2,"Executing postscript for ".$ta->{"alias"}.": @args");
382 my $postrc = system(@args);
383 if ( $postrc == -1 ) {
384 $::log->err("Cannot execute postexec program for".$ta->{"alias"}.": $!");
385 } elsif ( $postrc > 0 ) {
386 $::log->err("postexec program for ".$ta->{"alias"}." returned error code ".($? >> 8));
387 }
388 }
389 $ta->restoreLogMode();
390 }
391
392 return 1;
393 }
394
395 sub parallel_metafiles($@) {
396 my $parallelism = shift;
397 my @metafiles = @_;
398
399 my %pids = (); # file handle by processID
400 my %metafile_by_fh = (); # reverse map
401 my $readset = new IO::Select();
402 my %logoutput = ();
403
404 $| = 1;
405
406 $::log->verb(2,"starting up to $parallelism worker processes");
407
408 while ( @metafiles or scalar keys %pids ) {
409 # loop until we have started all possible retrievals AND have
410 # collected all possible output
411
412 ( @metafiles and (scalar keys %pids < $parallelism) ) and do {
413 # we have metafiles left, and have spare process slots
414 my $metafile = shift @metafiles;
415
416
417 $logoutput{$metafile} = "";
418
419 my $cout;
420 my $cpid = open $cout, "-|";
421 defined $cpid and defined $cout or
422 $::log->err("Cannot fork ($metafile): $!") and next;
423
424 $::log->verb(5,"LOOP: starting process $cpid for $metafile");
425
426 if ( $cpid == 0 ) { # I'm the child that should care for $metafile
427 $0 = "fetch-crl worker $metafile";
428 $::log->cleanse();
429 $::log->destadd("qualified");
430 &process_metafiles($metafile);
431 $::log->flush;
432 exit($::log->exitstatus);
433 } else { # parent
434 $pids{$cpid} = $cout;
435 $readset->add($cout);
436 $metafile_by_fh{$cout} = $metafile;
437 }
438 };
439
440 # do a select loop over the outstanding requests to collect messages
441 # if we are in the process of starting more processes, we just
442 # briefly poll out pending output so as not to have blocking
443 # children, but if we have started as many children as we ought to
444 # we put in a longer timeout -- any output on a handle will
445 # get us out of the select and into flushing mode again
446 my $timeout = (@metafiles && (scalar keys %pids < $parallelism) ? 0.1:1);
447
448 $::log->verb(6,"PLOOP: select with timeout $timeout");
449 my ( $rh_set ) = IO::Select->select($readset, undef, undef, $timeout);
450
451 foreach my $fh ( @$rh_set ) {
452 my $metafile = $metafile_by_fh{$fh};
453 # we know there is at least one byte to read, but also that
454 # any client sends complete
455 while (1) {
456 my $char;
457 my $length = sysread $fh, $char, 1;
458 if ( $length ) {
459 $logoutput{$metafile} .= $char;
460 $char eq "\n" and last;
461 } else {
462 #expected a char but got eof
463 $readset->remove($fh);
464 close($fh);
465 map {
466 $pids{$_} == $fh and
467 waitpid($_,WNOHANG) and
468 delete $pids{$_} and
469 $::log->verb(5,"Collected pid $_ (rc=$?),",
470 length($logoutput{$metafile}),"bytes log output");
471 } keys %pids;
472 last;
473 }
474 }
475 }
476 }
477
478 # log out all collected log data from our children
479 foreach my $metafile ( sort keys %logoutput ) {
480 foreach my $line ( split(/\n/,$logoutput{$metafile}) ) {
481 $line =~ /^ERROR\s+(.*)$/ and $::log->err($1);
482 $line =~ /^WARN\s+(.*)$/ and $::log->warn($1);
483 $line =~ /^VERBOSE\((\d+)\)\s+(.*)$/ and $::log->verb($1,$2);
484 $line =~ /^DEBUG\((\d+)\)\s+(.*)$/ and $::log->debug($1,$2);
485 }
486 }
487
488 return 1;
489 }

Properties

Name Value
svn:executable *
svn:keywords Date Author Revision Id

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28