/[pdpsoft]/nl.nikhef.pdp.fetchcrl/trunk/fetch-crl3.pl.cin
ViewVC logotype

Annotation of /nl.nikhef.pdp.fetchcrl/trunk/fetch-crl3.pl.cin

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2605 - (hide annotations) (download) (as text)
Mon Feb 25 15:27:32 2013 UTC (8 years, 9 months ago) by davidg
Original Path: nl.nikhef.pdp.fetchcrl/trunk/fetch-crl3.pl
File MIME type: text/x-prolog
File size: 14977 byte(s)
Use only INET6 socket glue

1 davidg 1758 #! /usr/bin/perl -w
2     #
3     # @(#)$Id$
4     #
5     # Copyright 2010 David Groep, Nationaal instituut voor
6     # subatomaire fysica NIKHEF
7     #
8     # Licensed under the Apache License, Version 2.0 (the "License");
9     # you may not use this file except in compliance with the License.
10     # You may obtain a copy of the License at
11     #
12     # http://www.apache.org/licenses/LICENSE-2.0
13     #
14     # Unless required by applicable law or agreed to in writing, software
15     # distributed under the License is distributed on an "AS IS" BASIS,
16     # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17     # See the License for the specific language governing permissions and
18     # limitations under the License.
19     #
20     #
21     package main;
22    
23     use strict;
24     use Getopt::Long qw(:config no_ignore_case bundling);
25     use POSIX;
26     eval { require LWP or die; }; $@ and die "Please install libwww-perl (LWP)\n";
27    
28     # import modules that are needed but still external
29     # (the installed version may have these packages embedded in-line)
30     #
31     require ConfigTiny and import ConfigTiny unless defined &ConfigTiny::new;
32     require TrustAnchor and import TrustAnchor unless defined &TrustAnchor::new;
33     require CRLWriter and import CRLWriter unless defined &CRLWriter::new;
34     require FCLog and import FCLog unless defined &FCLog::new;
35     require OSSL and import OSSL unless defined &OSSL::new;
36     require CRL and import CRL unless defined &CRL::new;
37    
38     my $use_DataDumper = eval { require Data::Dumper; };
39     my $use_IOSelect = eval { require IO::Select; };
40    
41     use vars qw/ $log $cnf /;
42    
43    
44     # ###########################################################################
45     #
46     #
47     ($cnf,$log) = &init_configuration();
48    
49 davidg 2604 # use Net::INET6Glue if so requested (is not a default module)
50     if ( $cnf->{_}->{inet6glue} ) {
51 davidg 2605 eval { require Net::INET6Glue::INET_is_INET6 or die; };
52 davidg 2604 $@ and die "Please install Net::INET6Glue before enabling inet6glue config\n";
53     }
54    
55 davidg 1758 # verify local installation sanity for loaded modules
56     $::log->getverbose > 6 and ! $use_DataDumper and
57     $::log->err("Cannot set verbosity higher than 6 without Data::Dumper") and
58     exit(1);
59     $::cnf->{_}->{parallelism} and ! $use_IOSelect and
60     $::log->err("Cannot use parallel retrieval without IO::Select") and
61     exit(1);
62    
63     $use_DataDumper and $::log->verb(7,Data::Dumper::Dumper($cnf));
64    
65     # set safe path if so requested
66     $cnf->{_}->{path} and $ENV{"PATH"} = $cnf->{_}->{path} and
67     $::log->verb(5,"Set PATH to",$ENV{"PATH"});
68    
69     # wait up to randomwait seconds to spread download load
70     $cnf->{_}->{randomwait} and do {
71     my $wtime = int(rand($cnf->{_}->{randomwait}));
72     $::log->verb(2,"Sleeping $wtime seconds before continuing");
73     sleep($wtime);
74     };
75    
76    
77     # the list of trust anchors to process comes from the command line and
78     # all files in the infodir that are metadata or crl urls
79     # in the next phase, the suffix will be stripped and the info file
80     # when present preferred over the crlurl
81     #
82     my @metafiles = @ARGV;
83     $::cnf->{_}->{"infodir"} and do {
84     foreach my $fn (
85     map { glob ( $::cnf->{_}->{"infodir"} . "/$_" ); } "*.info", "*.crl_url"
86     ) {
87 davidg 1878 next if $::cnf->{_}->{nosymlinks} and -l $fn;
88 davidg 1758 $fn =~ /.*\/([^\/]+)(\.crl_url|\.info)$/;
89 davidg 2188 push @metafiles, $1 unless grep /^$1$/,@metafiles or not defined $1;
90 davidg 1758 }
91     };
92    
93     @metafiles or
94     $log->err("No trust anchors to process") and exit($log->exitstatus);
95    
96     if ( $::cnf->{_}->{parallelism} ) {
97     &parallel_metafiles($::cnf->{_}->{parallelism}, @metafiles);
98     } else {
99     &process_metafiles( @metafiles );
100     }
101    
102     $log->flush;
103     exit($log->exitstatus);
104    
105    
106     # ###########################################################################
107     #
108     #
109     sub init_configuration() {
110     my ($cnf,$log);
111    
112     my ($configfile,$agingtolerance,$infodir,$statedir,$cadir,$httptimeout);
113     my ($output);
114     my @formats;
115     my $verbosity;
116     my $quiet=0;
117     my $help=0;
118     my $debuglevel;
119     my $parallelism=0;
120     my $randomwait;
121 davidg 1878 my $nosymlinks;
122 davidg 2084 my $cfgdir;
123 davidg 2604 my $inet6glue=0;
124 davidg 1758
125     $log = FCLog->new("qualified");
126    
127     &GetOptions(
128     "c|config=s" => \$configfile,
129     "l|infodir=s" => \$infodir,
130     "cadir=s" => \$cadir,
131     "s|statedir=s" => \$statedir,
132 davidg 2084 "cfgdir=s" => \$cfgdir,
133 davidg 1758 "T|httptimeout=i" => \$httptimeout,
134     "o|output=s" => \$output,
135     "format=s@" => \@formats,
136     "v|verbose+" => \$verbosity,
137     "h|help+" => \$help,
138     "q|quiet+" => \$quiet,
139     "d|debug+" => \$debuglevel,
140     "p|parallelism=i" => \$parallelism,
141 davidg 1878 "nosymlinks+" => \$nosymlinks,
142 davidg 1758 "a|agingtolerance=i" => \$agingtolerance,
143     "r|randomwait=i" => \$randomwait,
144 davidg 2604 "inet6glue+" => \$inet6glue,
145 davidg 1758 ) or &help and exit(1);
146    
147     $help and &help and exit(0);
148    
149 davidg 1878 $configfile ||= ( -e "/etc/fetch-crl.conf" and "/etc/fetch-crl.conf" );
150 davidg 1758 $configfile ||= ( -e "/etc/fetch-crl.cnf" and "/etc/fetch-crl.cnf" );
151    
152     $cnf = ConfigTiny->new();
153     $configfile and
154     $cnf->read($configfile) || die "Invalid config file $configfile:\n " .
155     $cnf->errstr . "\n";
156    
157 davidg 2084 ( defined $cnf->{_}->{cfgdir} and $cfgdir = $cnf->{_}->{cfgdir} )
158     unless defined $cfgdir;
159     $cfgdir ||= "/etc/fetch-crl.d";
160     if ( defined $cfgdir and -d $cfgdir and opendir(my $dh,$cfgdir) ) {
161     while ( my $fn = readdir $dh ) {
162     -f "$cfgdir/$fn" and -r "$cfgdir/$fn" and $cnf->read("$cfgdir/$fn");
163     }
164     close $dh;
165     }
166    
167 davidg 1758 # command-line option overrides
168     $cnf->{_}->{agingtolerance} = $agingtolerance if defined $agingtolerance;
169     $cnf->{_}->{infodir} = $infodir if defined $infodir;
170     $cnf->{_}->{cadir} = $cadir if defined $cadir;
171     $cnf->{_}->{statedir} = $statedir if defined $statedir;
172     $cnf->{_}->{httptimeout} = $httptimeout if defined $httptimeout;
173     $cnf->{_}->{verbosity} = $verbosity if defined $verbosity;
174     $cnf->{_}->{debuglevel} = $debuglevel if defined $debuglevel;
175     $cnf->{_}->{output} = $output if defined $output;
176 davidg 2305 $cnf->{_}->{formats} = join "\001",@formats if @formats;
177 davidg 1758 $cnf->{_}->{parallelism} = $parallelism if $parallelism;
178     $cnf->{_}->{randomwait} = $randomwait if defined $randomwait;
179 davidg 1878 $cnf->{_}->{nosymlinks} = $nosymlinks if defined $nosymlinks;
180 davidg 2604 $cnf->{_}->{inet6glue} = $inet6glue if $inet6glue;
181 davidg 1758
182 davidg 2597 # deal with interaction of verbosity in logfile and quiet option
183     # since a noquiet config option can cancel it
184     if ( not defined $cnf->{_}->{noquiet} ) {
185     if ( $quiet == 1) { $cnf->{_}->{verbosity} = -1; }
186     } else {
187     if ( $quiet >= 2) { $cnf->{_}->{verbosity} = -1; }
188     }
189    
190 davidg 1758 # key default values
191     defined $cnf->{_}->{version} or $cnf->{_}->{version} = "3+";
192     defined $cnf->{_}->{packager} or $cnf->{_}->{packager} = "EUGridPMA";
193     defined $cnf->{_}->{openssl} or $cnf->{_}->{openssl} = "openssl";
194     defined $cnf->{_}->{agingtolerance} or $cnf->{_}->{agingtolerance} ||= 24;
195     defined $cnf->{_}->{infodir} or $cnf->{_}->{infodir} = '/etc/grid-security/certificates';
196     defined $cnf->{_}->{output} or $cnf->{_}->{output} = $cnf->{_}->{infodir};
197     defined $cnf->{_}->{cadir} or $cnf->{_}->{cadir} = $cnf->{_}->{infodir};
198     defined $cnf->{_}->{statedir} or $cnf->{_}->{statedir} = "/var/cache/fetch-crl" if -d "/var/cache/fetch-crl" and -w "/var/cache/fetch-crl";
199     defined $cnf->{_}->{formats} or $cnf->{_}->{formats} = "openssl";
200     defined $cnf->{_}->{opensslmode} or $cnf->{_}->{opensslmode} = "dual";
201     defined $cnf->{_}->{httptimeout} or $cnf->{_}->{httptimeout} = 120;
202     defined $cnf->{_}->{nametemplate_der} or
203     $cnf->{_}->{nametemplate_der} = "\@ANCHORNAME\@.\@R\@.crl";
204     defined $cnf->{_}->{nametemplate_pem} or
205     $cnf->{_}->{nametemplate_pem} = "\@ANCHORNAME\@.\@R\@.crl.pem";
206     defined $cnf->{_}->{catemplate} or
207 davidg 2305 $cnf->{_}->{catemplate} = "\@ALIAS\@.pem\001".
208     "\@ALIAS\@.\@R\@\001\@ANCHORNAME\@.\@R\@";
209 davidg 1758
210     $cnf->{_}->{nonssverify} ||= 0;
211     $cnf->{_}->{nocache} ||= 0;
212 davidg 1878 $cnf->{_}->{nosymlinks} ||= 0;
213 davidg 1758 $cnf->{_}->{verbosity} ||= 0;
214     $cnf->{_}->{debuglevel} ||= 0;
215 davidg 2604 $cnf->{_}->{inet6glue} ||= 0;
216 davidg 1758
217     $cnf->{_}->{stateless} and delete $cnf->{_}->{statedir};
218    
219     # expand array keys in config
220     defined $cnf->{_}->{formats} and
221 davidg 2305 @{$cnf->{_}->{formats_}} = split(/[\001;,\s]+/,$cnf->{_}->{formats});
222 davidg 1758
223     # sanity check on configuration
224     $cnf->{_}->{statedir} and ! -d $cnf->{_}->{statedir} and
225     die "Invalid state directory " . $cnf->{_}->{statedir} . "\n";
226     $cnf->{_}->{infodir} and ! -d $cnf->{_}->{infodir} and
227     die "Invalid meta-data directory ".$cnf->{_}->{infodir}."\n";
228    
229     # initialize logging
230     $log->flush;
231     $cnf->{_}->{logmode} and $log->destremove("qualified") and do {
232 davidg 2305 foreach ( split(/[,\001]+/,$cnf->{_}->{logmode}) ) {
233 davidg 1758 if ( /^syslog$/ ) { $log->destadd($_,$cnf->{_}->{syslogfacility}); }
234     elsif ( /^(direct|qualified|cache)$/ ) { $log->destadd($_); }
235     else { die "Invalid log destination $_, exiting.\n"; }
236     }
237     };
238     $log->setverbose($cnf->{_}->{verbosity});
239     $log->setdebug($cnf->{_}->{debuglevel});
240    
241     return ($cnf,$log);
242     }
243    
244     # ###########################################################################
245     #
246     #
247     sub help() {
248     (my $name = $0) =~ s/.*\///;
249     print <<EOHELP;
250     The fetch-crl utility will retrieve certificate revocation lists (CRLs) for
251     a set of installed trust anchors, based on crl_url files or IGTF-style info
252     files. It will install these for use with OpenSSL, NSS or third-party tools.
253    
254     Usage: $name [-c|--config configfile] [-l|--infodir path]
255     [--cadir path] [-s|--statedir path] [-o|--output path] [--format \@formats]
256 davidg 1878 [-T|--httptimeout seconds] [-p|--parallelism n] [--nosymlinks]
257 davidg 1758 [-a|--agingtolerance hours] [-r|--randomwait seconds]
258     [-v|--verbose] [-h|--help] [-q|--quiet] [-d|--debug level]
259    
260     Options:
261     -c | --config path
262 davidg 1878 Read configuration data from path, default: /etc/fetch-crl.conf
263 davidg 1758 -l | --infodir path
264     Location of the trust anchor meta-data files (crl_url or info),
265     default: /etc/grid-security/certificates
266     --cadir path
267     Location of the trust anchors (default to infodir)
268     -s | --statedir path
269     Location of the historic state data (for caching and delayed-warning)
270     -T | --httptimeout sec
271     Maximum time in seconds to wait for retrieval or a single URL
272     -o | --output path
273     Location of the CRLs written (global default, defaults to infodir
274     --format \@formats
275     Format(s) in which the CRLs will be written (openssl, pem, der, nss)
276 davidg 1878 --nosymlinks
277     Do not include meta-data files that are symlinks
278 davidg 1758 -v | --verbose
279     Become more talkative
280     -q | --quiet
281     Become really quiet (overrides verbosity)
282     -p | --parallelism n
283     Run up to n parallel trust anchor retrieval processes
284     -a | --agingtolerance hours
285     Be quiet for up to hours hours before raising an error. Until
286     the tolerance has passed, only warnings are raised
287     -r | --randomwait seconds
288     Introduce a random delay of up to seconds seconds before starting
289     any retrieval processes
290     -h | --help
291     This help text
292    
293     EOHELP
294    
295     return 1;
296     }
297    
298     # ###########################################################################
299     #
300     #
301     sub process_metafiles(@) {
302     my @metafiles = @_;
303    
304     foreach my $f ( @metafiles ) {
305     my $ta = TrustAnchor->new();
306     $cnf->{_}->{"infodir"} and $ta->setInfodir($cnf->{_}->{"infodir"});
307     $ta->loadAnchor($f) or next;
308     $ta->saveLogMode() and $ta->setLogMode();
309     $ta->loadState() or next;
310 davidg 2421
311     # using the HASH in the CA filename templates requires the CRL
312     # is retrieved first to determinte the hash
313     if ( $cnf->{_}->{"catemplate"} =~ /\@HASH\@/ ) {
314     $ta->retrieve or next;
315     $ta->loadCAfiles() or next;
316     } else {
317     $ta->loadCAfiles() or next;
318     $ta->retrieve or next;
319     }
320    
321 davidg 1758 $ta->verifyAndConvertCRLs or next;
322    
323     my $writer = CRLWriter->new($ta);
324     $writer->writeall() or next;
325     $ta->saveState() or next;
326     $ta->restoreLogMode();
327     }
328    
329     return 1;
330     }
331    
332     sub parallel_metafiles($@) {
333     my $parallelism = shift;
334     my @metafiles = @_;
335    
336     my %pids = (); # file handle by processID
337     my %metafile_by_fh = (); # reverse map
338     my $readset = new IO::Select();
339     my %logoutput = ();
340    
341     $| = 1;
342    
343     $::log->verb(2,"starting up to $parallelism worker processes");
344    
345     while ( @metafiles or scalar keys %pids ) {
346     # loop until we have started all possible retrievals AND have
347     # collected all possible output
348    
349     ( @metafiles and (scalar keys %pids < $parallelism) ) and do {
350     # we have metafiles left, and have spare process slots
351     my $metafile = shift @metafiles;
352    
353    
354     $logoutput{$metafile} = "";
355    
356     my $cout;
357     my $cpid = open $cout, "-|";
358     defined $cpid and defined $cout or
359     $::log->err("Cannot fork ($metafile): $!") and next;
360    
361     $::log->verb(5,"LOOP: starting process $cpid for $metafile");
362    
363     if ( $cpid == 0 ) { # I'm the child that should care for $metafile
364     $0 = "fetch-crl worker $metafile";
365     $::log->cleanse();
366     $::log->destadd("qualified");
367     &process_metafiles($metafile);
368     $::log->flush;
369     exit($::log->exitstatus);
370     } else { # parent
371     $pids{$cpid} = $cout;
372     $readset->add($cout);
373     $metafile_by_fh{$cout} = $metafile;
374     }
375     };
376    
377     # do a select loop over the outstanding requests to collect messages
378     # if we are in the process of starting more processes, we just
379     # briefly poll out pending output so as not to have blocking
380     # children, but if we have started as many children as we ought to
381     # we put in a longer timeout -- any output on a handle will
382     # get us out of the select and into flushing mode again
383     my $timeout = (@metafiles && (scalar keys %pids < $parallelism) ? 0.1:1);
384    
385     $::log->verb(6,"PLOOP: select with timeout $timeout");
386     my ( $rh_set ) = IO::Select->select($readset, undef, undef, $timeout);
387    
388     foreach my $fh ( @$rh_set ) {
389     my $metafile = $metafile_by_fh{$fh};
390     # we know there is at least one byte to read, but also that
391     # any client sends complete
392     while (1) {
393     my $char;
394     my $length = sysread $fh, $char, 1;
395     if ( $length ) {
396     $logoutput{$metafile} .= $char;
397     $char eq "\n" and last;
398     } else {
399     #expected a char but got eof
400     $readset->remove($fh);
401     close($fh);
402     map {
403     $pids{$_} == $fh and
404     waitpid($_,WNOHANG) and
405     delete $pids{$_} and
406     $::log->verb(5,"Collected pid $_ (rc=$?),",
407     length($logoutput{$metafile}),"bytes log output");
408     } keys %pids;
409     last;
410     }
411     }
412     }
413     }
414    
415     # log out all collected log data from our children
416     foreach my $metafile ( sort keys %logoutput ) {
417     foreach my $line ( split(/\n/,$logoutput{$metafile}) ) {
418     $line =~ /^ERROR\s+(.*)$/ and $::log->err($1);
419     $line =~ /^WARN\s+(.*)$/ and $::log->warn($1);
420     $line =~ /^VERBOSE\((\d+)\)\s+(.*)$/ and $::log->verb($1,$2);
421     $line =~ /^DEBUG\((\d+)\)\s+(.*)$/ and $::log->debug($1,$2);
422     }
423     }
424    
425     return 1;
426     }

Properties

Name Value
svn:executable *

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28