/[pdpsoft]/nl.nikhef.pdp.fetchcrl/trunk/fetch-crl3.pl.cin
ViewVC logotype

Annotation of /nl.nikhef.pdp.fetchcrl/trunk/fetch-crl3.pl.cin

Parent Directory Parent Directory | Revision Log Revision Log


Revision 3107 - (hide annotations) (download)
Fri Sep 9 15:32:11 2016 UTC (5 years, 1 month ago) by davidg
File size: 17264 byte(s)
Add post-exec capability

1 davidg 1758 #! /usr/bin/perl -w
2     #
3     # @(#)$Id$
4     #
5     # Copyright 2010 David Groep, Nationaal instituut voor
6     # subatomaire fysica NIKHEF
7     #
8     # Licensed under the Apache License, Version 2.0 (the "License");
9     # you may not use this file except in compliance with the License.
10     # You may obtain a copy of the License at
11     #
12     # http://www.apache.org/licenses/LICENSE-2.0
13     #
14     # Unless required by applicable law or agreed to in writing, software
15     # distributed under the License is distributed on an "AS IS" BASIS,
16     # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17     # See the License for the specific language governing permissions and
18     # limitations under the License.
19     #
20     #
21     package main;
22    
23     use strict;
24     use Getopt::Long qw(:config no_ignore_case bundling);
25     use POSIX;
26     eval { require LWP or die; }; $@ and die "Please install libwww-perl (LWP)\n";
27    
28 davidg 2649 my $sccsid = '@(#)$Id$';
29 davidg 2646
30 davidg 1758 # import modules that are needed but still external
31     # (the installed version may have these packages embedded in-line)
32     #
33     require ConfigTiny and import ConfigTiny unless defined &ConfigTiny::new;
34     require TrustAnchor and import TrustAnchor unless defined &TrustAnchor::new;
35     require CRLWriter and import CRLWriter unless defined &CRLWriter::new;
36     require FCLog and import FCLog unless defined &FCLog::new;
37     require OSSL and import OSSL unless defined &OSSL::new;
38     require CRL and import CRL unless defined &CRL::new;
39    
40     my $use_DataDumper = eval { require Data::Dumper; };
41     my $use_IOSelect = eval { require IO::Select; };
42    
43     use vars qw/ $log $cnf /;
44    
45    
46     # ###########################################################################
47     #
48     #
49     ($cnf,$log) = &init_configuration();
50    
51 davidg 2604 # use Net::INET6Glue if so requested (is not a default module)
52     if ( $cnf->{_}->{inet6glue} ) {
53 davidg 2605 eval { require Net::INET6Glue::INET_is_INET6 or die; };
54 davidg 2604 $@ and die "Please install Net::INET6Glue before enabling inet6glue config\n";
55     }
56    
57 davidg 1758 # verify local installation sanity for loaded modules
58     $::log->getverbose > 6 and ! $use_DataDumper and
59     $::log->err("Cannot set verbosity higher than 6 without Data::Dumper") and
60     exit(1);
61     $::cnf->{_}->{parallelism} and ! $use_IOSelect and
62     $::log->err("Cannot use parallel retrieval without IO::Select") and
63     exit(1);
64    
65     $use_DataDumper and $::log->verb(7,Data::Dumper::Dumper($cnf));
66    
67     # set safe path if so requested
68     $cnf->{_}->{path} and $ENV{"PATH"} = $cnf->{_}->{path} and
69     $::log->verb(5,"Set PATH to",$ENV{"PATH"});
70 davidg 2690
71     # set rcmode if present in config
72 davidg 2691 defined $cnf->{_}->{rcmode} and do {
73 davidg 2692 $::log->verb(4,"Setting exit status mode to ".$cnf->{_}->{rcmode});
74     $::log->setrcmode($cnf->{_}->{rcmode}) or exit($log->exitstatus);
75 davidg 2691 $::log->verb(2,"Exit status mode is set to ".$cnf->{_}->{rcmode});
76 davidg 2690 };
77 davidg 1758
78     # wait up to randomwait seconds to spread download load
79     $cnf->{_}->{randomwait} and do {
80     my $wtime = int(rand($cnf->{_}->{randomwait}));
81     $::log->verb(2,"Sleeping $wtime seconds before continuing");
82     sleep($wtime);
83     };
84    
85    
86     # the list of trust anchors to process comes from the command line and
87     # all files in the infodir that are metadata or crl urls
88     # in the next phase, the suffix will be stripped and the info file
89     # when present preferred over the crlurl
90     #
91     my @metafiles = @ARGV;
92     $::cnf->{_}->{"infodir"} and do {
93     foreach my $fn (
94     map { glob ( $::cnf->{_}->{"infodir"} . "/$_" ); } "*.info", "*.crl_url"
95     ) {
96 davidg 1878 next if $::cnf->{_}->{nosymlinks} and -l $fn;
97 davidg 1758 $fn =~ /.*\/([^\/]+)(\.crl_url|\.info)$/;
98 davidg 2188 push @metafiles, $1 unless grep /^$1$/,@metafiles or not defined $1;
99 davidg 1758 }
100     };
101    
102     @metafiles or
103 davidg 2715 $log->warn("No trust anchors to process") and exit($log->exitstatus);
104 davidg 1758
105     if ( $::cnf->{_}->{parallelism} ) {
106     &parallel_metafiles($::cnf->{_}->{parallelism}, @metafiles);
107     } else {
108     &process_metafiles( @metafiles );
109     }
110    
111 davidg 3107 # run any post-processing
112     if ( $::cnf->{_}->{"postexec"} ) {
113     my @args = ( $::cnf->{_}->{"postexec"},
114     "v1", "global",
115     $::cnf->{_}->{"infodir"}, $::cnf->{_}->{"cadir"}, $::cnf->{_}->{"output"} );
116     $::log->verb(2,"Executing global postscript @args");
117     my $postrc = system(@args);
118     if ( $postrc == -1 ) {
119     $::log->err("Cannot execute global postexec program: $!");
120     } elsif ( $postrc > 0 ) {
121     $::log->err("Global postexec program returned error code ".($? >> 8));
122     }
123     }
124    
125 davidg 1758 $log->flush;
126     exit($log->exitstatus);
127    
128    
129     # ###########################################################################
130     #
131     #
132     sub init_configuration() {
133     my ($cnf,$log);
134    
135     my ($configfile,$agingtolerance,$infodir,$statedir,$cadir,$httptimeout);
136     my ($output);
137     my @formats;
138     my $verbosity;
139     my $quiet=0;
140     my $help=0;
141 davidg 2646 my $showversion=0;
142 davidg 1758 my $debuglevel;
143     my $parallelism=0;
144     my $randomwait;
145 davidg 1878 my $nosymlinks;
146 davidg 2084 my $cfgdir;
147 davidg 2604 my $inet6glue=0;
148 davidg 2692 my %directives;
149 davidg 1758
150     $log = FCLog->new("qualified");
151    
152     &GetOptions(
153     "c|config=s" => \$configfile,
154     "l|infodir=s" => \$infodir,
155     "cadir=s" => \$cadir,
156     "s|statedir=s" => \$statedir,
157 davidg 2084 "cfgdir=s" => \$cfgdir,
158 davidg 1758 "T|httptimeout=i" => \$httptimeout,
159     "o|output=s" => \$output,
160     "format=s@" => \@formats,
161 davidg 2692 "define=s" => \%directives,
162 davidg 1758 "v|verbose+" => \$verbosity,
163     "h|help+" => \$help,
164 davidg 2646 "V|version+" => \$showversion,
165 davidg 1758 "q|quiet+" => \$quiet,
166     "d|debug+" => \$debuglevel,
167     "p|parallelism=i" => \$parallelism,
168 davidg 1878 "nosymlinks+" => \$nosymlinks,
169 davidg 1758 "a|agingtolerance=i" => \$agingtolerance,
170     "r|randomwait=i" => \$randomwait,
171 davidg 2604 "inet6glue+" => \$inet6glue,
172 davidg 1758 ) or &help and exit(1);
173    
174     $help and &help and exit(0);
175 davidg 2646 $showversion and &showversion and exit(0);
176 davidg 1758
177 davidg 1878 $configfile ||= ( -e "/etc/fetch-crl.conf" and "/etc/fetch-crl.conf" );
178 davidg 1758 $configfile ||= ( -e "/etc/fetch-crl.cnf" and "/etc/fetch-crl.cnf" );
179    
180     $cnf = ConfigTiny->new();
181     $configfile and
182     $cnf->read($configfile) || die "Invalid config file $configfile:\n " .
183     $cnf->errstr . "\n";
184    
185 davidg 2084 ( defined $cnf->{_}->{cfgdir} and $cfgdir = $cnf->{_}->{cfgdir} )
186     unless defined $cfgdir;
187     $cfgdir ||= "/etc/fetch-crl.d";
188     if ( defined $cfgdir and -d $cfgdir and opendir(my $dh,$cfgdir) ) {
189     while ( my $fn = readdir $dh ) {
190     -f "$cfgdir/$fn" and -r "$cfgdir/$fn" and $cnf->read("$cfgdir/$fn");
191     }
192     close $dh;
193     }
194    
195 davidg 2692 # add defined from the command line to the configuration, to the
196     # main section _ thereof unless there is a colon in the key
197     foreach my $k ( keys %directives ) {
198     my $section ="_";
199     my $dvalue = $directives{$k};
200     if ( $k =~ m/(\w+):(.*)/ ) {
201     $section = $1;
202     $k=$2;
203     }
204     $cnf->{$section}->{$k} = $dvalue;
205     }
206    
207 davidg 1758 # command-line option overrides
208     $cnf->{_}->{agingtolerance} = $agingtolerance if defined $agingtolerance;
209     $cnf->{_}->{infodir} = $infodir if defined $infodir;
210     $cnf->{_}->{cadir} = $cadir if defined $cadir;
211     $cnf->{_}->{statedir} = $statedir if defined $statedir;
212     $cnf->{_}->{httptimeout} = $httptimeout if defined $httptimeout;
213     $cnf->{_}->{verbosity} = $verbosity if defined $verbosity;
214     $cnf->{_}->{debuglevel} = $debuglevel if defined $debuglevel;
215     $cnf->{_}->{output} = $output if defined $output;
216 davidg 2305 $cnf->{_}->{formats} = join "\001",@formats if @formats;
217 davidg 1758 $cnf->{_}->{parallelism} = $parallelism if $parallelism;
218     $cnf->{_}->{randomwait} = $randomwait if defined $randomwait;
219 davidg 1878 $cnf->{_}->{nosymlinks} = $nosymlinks if defined $nosymlinks;
220 davidg 2604 $cnf->{_}->{inet6glue} = $inet6glue if $inet6glue;
221 davidg 1758
222 davidg 2597 # deal with interaction of verbosity in logfile and quiet option
223     # since a noquiet config option can cancel it
224     if ( not defined $cnf->{_}->{noquiet} ) {
225     if ( $quiet == 1) { $cnf->{_}->{verbosity} = -1; }
226     } else {
227     if ( $quiet >= 2) { $cnf->{_}->{verbosity} = -1; }
228     }
229    
230 davidg 1758 # key default values
231     defined $cnf->{_}->{version} or $cnf->{_}->{version} = "3+";
232     defined $cnf->{_}->{packager} or $cnf->{_}->{packager} = "EUGridPMA";
233     defined $cnf->{_}->{openssl} or $cnf->{_}->{openssl} = "openssl";
234     defined $cnf->{_}->{agingtolerance} or $cnf->{_}->{agingtolerance} ||= 24;
235     defined $cnf->{_}->{infodir} or $cnf->{_}->{infodir} = '/etc/grid-security/certificates';
236     defined $cnf->{_}->{output} or $cnf->{_}->{output} = $cnf->{_}->{infodir};
237     defined $cnf->{_}->{cadir} or $cnf->{_}->{cadir} = $cnf->{_}->{infodir};
238     defined $cnf->{_}->{statedir} or $cnf->{_}->{statedir} = "/var/cache/fetch-crl" if -d "/var/cache/fetch-crl" and -w "/var/cache/fetch-crl";
239     defined $cnf->{_}->{formats} or $cnf->{_}->{formats} = "openssl";
240     defined $cnf->{_}->{opensslmode} or $cnf->{_}->{opensslmode} = "dual";
241     defined $cnf->{_}->{httptimeout} or $cnf->{_}->{httptimeout} = 120;
242 davidg 2783 defined $cnf->{_}->{expirestolerance} or $cnf->{_}->{expirestolerance} = (7*60*60); # at least 7 hrs should nextUpdate be beyond the cache FreshUntil
243     defined $cnf->{_}->{maxcachetime} or $cnf->{_}->{maxcachetime} = (4*24*60*60); # arbitrarily set it at 4 days
244 davidg 1758 defined $cnf->{_}->{nametemplate_der} or
245     $cnf->{_}->{nametemplate_der} = "\@ANCHORNAME\@.\@R\@.crl";
246     defined $cnf->{_}->{nametemplate_pem} or
247     $cnf->{_}->{nametemplate_pem} = "\@ANCHORNAME\@.\@R\@.crl.pem";
248     defined $cnf->{_}->{catemplate} or
249 davidg 2305 $cnf->{_}->{catemplate} = "\@ALIAS\@.pem\001".
250     "\@ALIAS\@.\@R\@\001\@ANCHORNAME\@.\@R\@";
251 davidg 1758
252     $cnf->{_}->{nonssverify} ||= 0;
253     $cnf->{_}->{nocache} ||= 0;
254 davidg 1878 $cnf->{_}->{nosymlinks} ||= 0;
255 davidg 1758 $cnf->{_}->{verbosity} ||= 0;
256     $cnf->{_}->{debuglevel} ||= 0;
257 davidg 2604 $cnf->{_}->{inet6glue} ||= 0;
258 davidg 1758
259     $cnf->{_}->{stateless} and delete $cnf->{_}->{statedir};
260    
261     # expand array keys in config
262     defined $cnf->{_}->{formats} and
263 davidg 2305 @{$cnf->{_}->{formats_}} = split(/[\001;,\s]+/,$cnf->{_}->{formats});
264 davidg 1758
265     # sanity check on configuration
266     $cnf->{_}->{statedir} and ! -d $cnf->{_}->{statedir} and
267     die "Invalid state directory " . $cnf->{_}->{statedir} . "\n";
268     $cnf->{_}->{infodir} and ! -d $cnf->{_}->{infodir} and
269     die "Invalid meta-data directory ".$cnf->{_}->{infodir}."\n";
270    
271     # initialize logging
272     $log->flush;
273     $cnf->{_}->{logmode} and $log->destremove("qualified") and do {
274 davidg 2305 foreach ( split(/[,\001]+/,$cnf->{_}->{logmode}) ) {
275 davidg 1758 if ( /^syslog$/ ) { $log->destadd($_,$cnf->{_}->{syslogfacility}); }
276     elsif ( /^(direct|qualified|cache)$/ ) { $log->destadd($_); }
277     else { die "Invalid log destination $_, exiting.\n"; }
278     }
279     };
280     $log->setverbose($cnf->{_}->{verbosity});
281     $log->setdebug($cnf->{_}->{debuglevel});
282    
283     return ($cnf,$log);
284     }
285    
286     # ###########################################################################
287     #
288     #
289 davidg 2646 sub showversion() {
290     (my $name = $0) =~ s/.*\///;
291     print "$name version @VERSION@\n";
292     return 1;
293     }
294    
295 davidg 1758 sub help() {
296     (my $name = $0) =~ s/.*\///;
297     print <<EOHELP;
298     The fetch-crl utility will retrieve certificate revocation lists (CRLs) for
299     a set of installed trust anchors, based on crl_url files or IGTF-style info
300     files. It will install these for use with OpenSSL, NSS or third-party tools.
301    
302     Usage: $name [-c|--config configfile] [-l|--infodir path]
303     [--cadir path] [-s|--statedir path] [-o|--output path] [--format \@formats]
304 davidg 1878 [-T|--httptimeout seconds] [-p|--parallelism n] [--nosymlinks]
305 davidg 1758 [-a|--agingtolerance hours] [-r|--randomwait seconds]
306     [-v|--verbose] [-h|--help] [-q|--quiet] [-d|--debug level]
307    
308     Options:
309     -c | --config path
310 davidg 1878 Read configuration data from path, default: /etc/fetch-crl.conf
311 davidg 1758 -l | --infodir path
312     Location of the trust anchor meta-data files (crl_url or info),
313     default: /etc/grid-security/certificates
314     --cadir path
315     Location of the trust anchors (default to infodir)
316     -s | --statedir path
317     Location of the historic state data (for caching and delayed-warning)
318     -T | --httptimeout sec
319     Maximum time in seconds to wait for retrieval or a single URL
320     -o | --output path
321     Location of the CRLs written (global default, defaults to infodir
322     --format \@formats
323     Format(s) in which the CRLs will be written (openssl, pem, der, nss)
324 davidg 1878 --nosymlinks
325     Do not include meta-data files that are symlinks
326 davidg 1758 -v | --verbose
327     Become more talkative
328     -q | --quiet
329     Become really quiet (overrides verbosity)
330     -p | --parallelism n
331     Run up to n parallel trust anchor retrieval processes
332     -a | --agingtolerance hours
333     Be quiet for up to hours hours before raising an error. Until
334     the tolerance has passed, only warnings are raised
335     -r | --randomwait seconds
336     Introduce a random delay of up to seconds seconds before starting
337     any retrieval processes
338     -h | --help
339     This help text
340    
341 davidg 2646 Version: @VERSION@
342 davidg 1758 EOHELP
343    
344     return 1;
345     }
346    
347     # ###########################################################################
348     #
349     #
350     sub process_metafiles(@) {
351     my @metafiles = @_;
352    
353     foreach my $f ( @metafiles ) {
354     my $ta = TrustAnchor->new();
355     $cnf->{_}->{"infodir"} and $ta->setInfodir($cnf->{_}->{"infodir"});
356     $ta->loadAnchor($f) or next;
357     $ta->saveLogMode() and $ta->setLogMode();
358     $ta->loadState() or next;
359 davidg 2421
360     # using the HASH in the CA filename templates requires the CRL
361     # is retrieved first to determinte the hash
362     if ( $cnf->{_}->{"catemplate"} =~ /\@HASH\@/ ) {
363     $ta->retrieve or next;
364     $ta->loadCAfiles() or next;
365     } else {
366     $ta->loadCAfiles() or next;
367     $ta->retrieve or next;
368     }
369    
370 davidg 1758 $ta->verifyAndConvertCRLs or next;
371    
372     my $writer = CRLWriter->new($ta);
373     $writer->writeall() or next;
374     $ta->saveState() or next;
375 davidg 3107
376     if ( $::cnf->{$ta->{"alias"}}->{"postexec"} ) {
377     my @args = ( $::cnf->{$ta->{"alias"}}->{"postexec"},
378     "v1", "ta",
379     $ta->{"alias"}, $ta->{"filename"}, $::cnf->{_}->{"cadir"}, $::cnf->{_}->{"output"} );
380     $::log->verb(2,"Executing postscript for ".$ta->{"alias"}.": @args");
381     my $postrc = system(@args);
382     if ( $postrc == -1 ) {
383     $::log->err("Cannot execute postexec program for".$ta->{"alias"}.": $!");
384     } elsif ( $postrc > 0 ) {
385     $::log->err("postexec program for ".$ta->{"alias"}." returned error code ".($? >> 8));
386     }
387     }
388 davidg 1758 $ta->restoreLogMode();
389     }
390    
391     return 1;
392     }
393    
394     sub parallel_metafiles($@) {
395     my $parallelism = shift;
396     my @metafiles = @_;
397    
398     my %pids = (); # file handle by processID
399     my %metafile_by_fh = (); # reverse map
400     my $readset = new IO::Select();
401     my %logoutput = ();
402    
403     $| = 1;
404    
405     $::log->verb(2,"starting up to $parallelism worker processes");
406    
407     while ( @metafiles or scalar keys %pids ) {
408     # loop until we have started all possible retrievals AND have
409     # collected all possible output
410    
411     ( @metafiles and (scalar keys %pids < $parallelism) ) and do {
412     # we have metafiles left, and have spare process slots
413     my $metafile = shift @metafiles;
414    
415    
416     $logoutput{$metafile} = "";
417    
418     my $cout;
419     my $cpid = open $cout, "-|";
420     defined $cpid and defined $cout or
421     $::log->err("Cannot fork ($metafile): $!") and next;
422    
423     $::log->verb(5,"LOOP: starting process $cpid for $metafile");
424    
425     if ( $cpid == 0 ) { # I'm the child that should care for $metafile
426     $0 = "fetch-crl worker $metafile";
427     $::log->cleanse();
428     $::log->destadd("qualified");
429     &process_metafiles($metafile);
430     $::log->flush;
431     exit($::log->exitstatus);
432     } else { # parent
433     $pids{$cpid} = $cout;
434     $readset->add($cout);
435     $metafile_by_fh{$cout} = $metafile;
436     }
437     };
438    
439     # do a select loop over the outstanding requests to collect messages
440     # if we are in the process of starting more processes, we just
441     # briefly poll out pending output so as not to have blocking
442     # children, but if we have started as many children as we ought to
443     # we put in a longer timeout -- any output on a handle will
444     # get us out of the select and into flushing mode again
445     my $timeout = (@metafiles && (scalar keys %pids < $parallelism) ? 0.1:1);
446    
447     $::log->verb(6,"PLOOP: select with timeout $timeout");
448     my ( $rh_set ) = IO::Select->select($readset, undef, undef, $timeout);
449    
450     foreach my $fh ( @$rh_set ) {
451     my $metafile = $metafile_by_fh{$fh};
452     # we know there is at least one byte to read, but also that
453     # any client sends complete
454     while (1) {
455     my $char;
456     my $length = sysread $fh, $char, 1;
457     if ( $length ) {
458     $logoutput{$metafile} .= $char;
459     $char eq "\n" and last;
460     } else {
461     #expected a char but got eof
462     $readset->remove($fh);
463     close($fh);
464     map {
465     $pids{$_} == $fh and
466     waitpid($_,WNOHANG) and
467     delete $pids{$_} and
468     $::log->verb(5,"Collected pid $_ (rc=$?),",
469     length($logoutput{$metafile}),"bytes log output");
470     } keys %pids;
471     last;
472     }
473     }
474     }
475     }
476    
477     # log out all collected log data from our children
478     foreach my $metafile ( sort keys %logoutput ) {
479     foreach my $line ( split(/\n/,$logoutput{$metafile}) ) {
480     $line =~ /^ERROR\s+(.*)$/ and $::log->err($1);
481     $line =~ /^WARN\s+(.*)$/ and $::log->warn($1);
482     $line =~ /^VERBOSE\((\d+)\)\s+(.*)$/ and $::log->verb($1,$2);
483     $line =~ /^DEBUG\((\d+)\)\s+(.*)$/ and $::log->debug($1,$2);
484     }
485     }
486    
487     return 1;
488     }

Properties

Name Value
svn:executable *
svn:keywords Date Author Revision Id

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28