/[pdpsoft]/nl.nikhef.pdp.fetchcrl/tags/fetch-crl-3.0.19/fetch-crl3.pl.cin
ViewVC logotype

Annotation of /nl.nikhef.pdp.fetchcrl/tags/fetch-crl-3.0.19/fetch-crl3.pl.cin

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1759 - (hide annotations) (download) (as text)
Fri Jun 11 15:41:40 2010 UTC (11 years, 5 months ago) by davidg
Original Path: trunk/fetchcrl/fetch-crl3.pl
File MIME type: text/x-prolog
File size: 13340 byte(s)
Moved fetch-crl to a tagger part of the repo

1 davidg 1758 #! /usr/bin/perl -w
2     #
3     # @(#)$Id$
4     #
5     # Copyright 2010 David Groep, Nationaal instituut voor
6     # subatomaire fysica NIKHEF
7     #
8     # Licensed under the Apache License, Version 2.0 (the "License");
9     # you may not use this file except in compliance with the License.
10     # You may obtain a copy of the License at
11     #
12     # http://www.apache.org/licenses/LICENSE-2.0
13     #
14     # Unless required by applicable law or agreed to in writing, software
15     # distributed under the License is distributed on an "AS IS" BASIS,
16     # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17     # See the License for the specific language governing permissions and
18     # limitations under the License.
19     #
20     #
21     package main;
22    
23     use strict;
24     use Getopt::Long qw(:config no_ignore_case bundling);
25     use POSIX;
26     eval { require LWP or die; }; $@ and die "Please install libwww-perl (LWP)\n";
27    
28     # import modules that are needed but still external
29     # (the installed version may have these packages embedded in-line)
30     #
31     require ConfigTiny and import ConfigTiny unless defined &ConfigTiny::new;
32     require TrustAnchor and import TrustAnchor unless defined &TrustAnchor::new;
33     require CRLWriter and import CRLWriter unless defined &CRLWriter::new;
34     require FCLog and import FCLog unless defined &FCLog::new;
35     require OSSL and import OSSL unless defined &OSSL::new;
36     require CRL and import CRL unless defined &CRL::new;
37    
38     my $use_DataDumper = eval { require Data::Dumper; };
39     my $use_IOSelect = eval { require IO::Select; };
40    
41     use vars qw/ $log $cnf /;
42    
43    
44     # ###########################################################################
45     #
46     #
47     ($cnf,$log) = &init_configuration();
48    
49     # verify local installation sanity for loaded modules
50     $::log->getverbose > 6 and ! $use_DataDumper and
51     $::log->err("Cannot set verbosity higher than 6 without Data::Dumper") and
52     exit(1);
53     $::cnf->{_}->{parallelism} and ! $use_IOSelect and
54     $::log->err("Cannot use parallel retrieval without IO::Select") and
55     exit(1);
56    
57     $use_DataDumper and $::log->verb(7,Data::Dumper::Dumper($cnf));
58    
59     # set safe path if so requested
60     $cnf->{_}->{path} and $ENV{"PATH"} = $cnf->{_}->{path} and
61     $::log->verb(5,"Set PATH to",$ENV{"PATH"});
62    
63     # wait up to randomwait seconds to spread download load
64     $cnf->{_}->{randomwait} and do {
65     my $wtime = int(rand($cnf->{_}->{randomwait}));
66     $::log->verb(2,"Sleeping $wtime seconds before continuing");
67     sleep($wtime);
68     };
69    
70    
71     # the list of trust anchors to process comes from the command line and
72     # all files in the infodir that are metadata or crl urls
73     # in the next phase, the suffix will be stripped and the info file
74     # when present preferred over the crlurl
75     #
76     my @metafiles = @ARGV;
77     $::cnf->{_}->{"infodir"} and do {
78     foreach my $fn (
79     map { glob ( $::cnf->{_}->{"infodir"} . "/$_" ); } "*.info", "*.crl_url"
80     ) {
81     $fn =~ /.*\/([^\/]+)(\.crl_url|\.info)$/;
82     push @metafiles, $1 unless grep /$1/,@metafiles or not defined $1;
83     }
84     };
85    
86     @metafiles or
87     $log->err("No trust anchors to process") and exit($log->exitstatus);
88    
89     if ( $::cnf->{_}->{parallelism} ) {
90     &parallel_metafiles($::cnf->{_}->{parallelism}, @metafiles);
91     } else {
92     &process_metafiles( @metafiles );
93     }
94    
95     $log->flush;
96     exit($log->exitstatus);
97    
98    
99     # ###########################################################################
100     #
101     #
102     sub init_configuration() {
103     my ($cnf,$log);
104    
105     my ($configfile,$agingtolerance,$infodir,$statedir,$cadir,$httptimeout);
106     my ($output);
107     my @formats;
108     my $verbosity;
109     my $quiet=0;
110     my $help=0;
111     my $debuglevel;
112     my $parallelism=0;
113     my $randomwait;
114    
115     $log = FCLog->new("qualified");
116    
117     &GetOptions(
118     "c|config=s" => \$configfile,
119     "l|infodir=s" => \$infodir,
120     "cadir=s" => \$cadir,
121     "s|statedir=s" => \$statedir,
122     "T|httptimeout=i" => \$httptimeout,
123     "o|output=s" => \$output,
124     "format=s@" => \@formats,
125     "v|verbose+" => \$verbosity,
126     "h|help+" => \$help,
127     "q|quiet+" => \$quiet,
128     "d|debug+" => \$debuglevel,
129     "p|parallelism=i" => \$parallelism,
130     "a|agingtolerance=i" => \$agingtolerance,
131     "r|randomwait=i" => \$randomwait,
132     ) or &help and exit(1);
133    
134     $help and &help and exit(0);
135    
136     $configfile ||= ( -e "/etc/fetch-crl.cnf" and "/etc/fetch-crl.cnf" );
137     ($quiet > 0) and $verbosity = -$quiet;
138    
139     $cnf = ConfigTiny->new();
140     $configfile and
141     $cnf->read($configfile) || die "Invalid config file $configfile:\n " .
142     $cnf->errstr . "\n";
143    
144     # command-line option overrides
145     $cnf->{_}->{agingtolerance} = $agingtolerance if defined $agingtolerance;
146     $cnf->{_}->{infodir} = $infodir if defined $infodir;
147     $cnf->{_}->{cadir} = $cadir if defined $cadir;
148     $cnf->{_}->{statedir} = $statedir if defined $statedir;
149     $cnf->{_}->{httptimeout} = $httptimeout if defined $httptimeout;
150     $cnf->{_}->{verbosity} = $verbosity if defined $verbosity;
151     $cnf->{_}->{debuglevel} = $debuglevel if defined $debuglevel;
152     $cnf->{_}->{output} = $output if defined $output;
153     $cnf->{_}->{formats} = join "",@formats if @formats;
154     $cnf->{_}->{parallelism} = $parallelism if $parallelism;
155     $cnf->{_}->{randomwait} = $randomwait if defined $randomwait;
156    
157     # key default values
158     defined $cnf->{_}->{version} or $cnf->{_}->{version} = "3+";
159     defined $cnf->{_}->{packager} or $cnf->{_}->{packager} = "EUGridPMA";
160     defined $cnf->{_}->{openssl} or $cnf->{_}->{openssl} = "openssl";
161     defined $cnf->{_}->{agingtolerance} or $cnf->{_}->{agingtolerance} ||= 24;
162     defined $cnf->{_}->{infodir} or $cnf->{_}->{infodir} = '/etc/grid-security/certificates';
163     defined $cnf->{_}->{output} or $cnf->{_}->{output} = $cnf->{_}->{infodir};
164     defined $cnf->{_}->{cadir} or $cnf->{_}->{cadir} = $cnf->{_}->{infodir};
165     defined $cnf->{_}->{statedir} or $cnf->{_}->{statedir} = "/var/cache/fetch-crl" if -d "/var/cache/fetch-crl" and -w "/var/cache/fetch-crl";
166     defined $cnf->{_}->{formats} or $cnf->{_}->{formats} = "openssl";
167     defined $cnf->{_}->{opensslmode} or $cnf->{_}->{opensslmode} = "dual";
168     defined $cnf->{_}->{httptimeout} or $cnf->{_}->{httptimeout} = 120;
169     defined $cnf->{_}->{nametemplate_der} or
170     $cnf->{_}->{nametemplate_der} = "\@ANCHORNAME\@.\@R\@.crl";
171     defined $cnf->{_}->{nametemplate_pem} or
172     $cnf->{_}->{nametemplate_pem} = "\@ANCHORNAME\@.\@R\@.crl.pem";
173     defined $cnf->{_}->{catemplate} or
174     $cnf->{_}->{catemplate} = "\@ALIAS\@.pem".
175     "\@ALIAS\@.\@R\@\@ANCHORNAME\@.\@R\@";
176    
177     $cnf->{_}->{nonssverify} ||= 0;
178     $cnf->{_}->{nocache} ||= 0;
179     $cnf->{_}->{verbosity} ||= 0;
180     $cnf->{_}->{debuglevel} ||= 0;
181    
182     $cnf->{_}->{stateless} and delete $cnf->{_}->{statedir};
183    
184     # expand array keys in config
185     defined $cnf->{_}->{formats} and
186     @{$cnf->{_}->{formats_}} = split(/[;,\s]+/,$cnf->{_}->{formats});
187    
188     # sanity check on configuration
189     $cnf->{_}->{statedir} and ! -d $cnf->{_}->{statedir} and
190     die "Invalid state directory " . $cnf->{_}->{statedir} . "\n";
191     $cnf->{_}->{cadir} ||= ".";
192     $cnf->{_}->{infodir} and ! -d $cnf->{_}->{infodir} and
193     die "Invalid meta-data directory ".$cnf->{_}->{infodir}."\n";
194    
195     # initialize logging
196     $log->flush;
197     $cnf->{_}->{logmode} and $log->destremove("qualified") and do {
198     foreach ( split(/[,]+/,$cnf->{_}->{logmode}) ) {
199     if ( /^syslog$/ ) { $log->destadd($_,$cnf->{_}->{syslogfacility}); }
200     elsif ( /^(direct|qualified|cache)$/ ) { $log->destadd($_); }
201     else { die "Invalid log destination $_, exiting.\n"; }
202     }
203     };
204     $log->setverbose($cnf->{_}->{verbosity});
205     $log->setdebug($cnf->{_}->{debuglevel});
206    
207     return ($cnf,$log);
208     }
209    
210     # ###########################################################################
211     #
212     #
213     sub help() {
214     (my $name = $0) =~ s/.*\///;
215     print <<EOHELP;
216     The fetch-crl utility will retrieve certificate revocation lists (CRLs) for
217     a set of installed trust anchors, based on crl_url files or IGTF-style info
218     files. It will install these for use with OpenSSL, NSS or third-party tools.
219    
220     Usage: $name [-c|--config configfile] [-l|--infodir path]
221     [--cadir path] [-s|--statedir path] [-o|--output path] [--format \@formats]
222     [-T|--httptimeout seconds] [-p|--parallelism n]
223     [-a|--agingtolerance hours] [-r|--randomwait seconds]
224     [-v|--verbose] [-h|--help] [-q|--quiet] [-d|--debug level]
225    
226     Options:
227     -c | --config path
228     Read configuration data from path, default: /etc/fetch-crl.cnf
229     -l | --infodir path
230     Location of the trust anchor meta-data files (crl_url or info),
231     default: /etc/grid-security/certificates
232     --cadir path
233     Location of the trust anchors (default to infodir)
234     -s | --statedir path
235     Location of the historic state data (for caching and delayed-warning)
236     -T | --httptimeout sec
237     Maximum time in seconds to wait for retrieval or a single URL
238     -o | --output path
239     Location of the CRLs written (global default, defaults to infodir
240     --format \@formats
241     Format(s) in which the CRLs will be written (openssl, pem, der, nss)
242     -v | --verbose
243     Become more talkative
244     -q | --quiet
245     Become really quiet (overrides verbosity)
246     -p | --parallelism n
247     Run up to n parallel trust anchor retrieval processes
248     -a | --agingtolerance hours
249     Be quiet for up to hours hours before raising an error. Until
250     the tolerance has passed, only warnings are raised
251     -r | --randomwait seconds
252     Introduce a random delay of up to seconds seconds before starting
253     any retrieval processes
254     -h | --help
255     This help text
256    
257     EOHELP
258    
259     return 1;
260     }
261    
262     # ###########################################################################
263     #
264     #
265     sub process_metafiles(@) {
266     my @metafiles = @_;
267    
268     foreach my $f ( @metafiles ) {
269     my $ta = TrustAnchor->new();
270     $cnf->{_}->{"infodir"} and $ta->setInfodir($cnf->{_}->{"infodir"});
271     $ta->loadAnchor($f) or next;
272     $ta->saveLogMode() and $ta->setLogMode();
273     $ta->loadCAfiles() or next;
274     $ta->loadState() or next;
275     $ta->retrieve or next;
276     $ta->verifyAndConvertCRLs or next;
277    
278     my $writer = CRLWriter->new($ta);
279     $writer->writeall() or next;
280     $ta->saveState() or next;
281     $ta->restoreLogMode();
282     }
283    
284     return 1;
285     }
286    
287     sub parallel_metafiles($@) {
288     my $parallelism = shift;
289     my @metafiles = @_;
290    
291     my %pids = (); # file handle by processID
292     my %metafile_by_fh = (); # reverse map
293     my $readset = new IO::Select();
294     my %logoutput = ();
295    
296     $| = 1;
297    
298     $::log->verb(2,"starting up to $parallelism worker processes");
299    
300     while ( @metafiles or scalar keys %pids ) {
301     # loop until we have started all possible retrievals AND have
302     # collected all possible output
303    
304     ( @metafiles and (scalar keys %pids < $parallelism) ) and do {
305     # we have metafiles left, and have spare process slots
306     my $metafile = shift @metafiles;
307    
308    
309     $logoutput{$metafile} = "";
310    
311     my $cout;
312     my $cpid = open $cout, "-|";
313     defined $cpid and defined $cout or
314     $::log->err("Cannot fork ($metafile): $!") and next;
315    
316     $::log->verb(5,"LOOP: starting process $cpid for $metafile");
317    
318     if ( $cpid == 0 ) { # I'm the child that should care for $metafile
319     $0 = "fetch-crl worker $metafile";
320     $::log->cleanse();
321     $::log->destadd("qualified");
322     &process_metafiles($metafile);
323     $::log->flush;
324     exit($::log->exitstatus);
325     } else { # parent
326     $pids{$cpid} = $cout;
327     $readset->add($cout);
328     $metafile_by_fh{$cout} = $metafile;
329     }
330     };
331    
332     # do a select loop over the outstanding requests to collect messages
333     # if we are in the process of starting more processes, we just
334     # briefly poll out pending output so as not to have blocking
335     # children, but if we have started as many children as we ought to
336     # we put in a longer timeout -- any output on a handle will
337     # get us out of the select and into flushing mode again
338     my $timeout = (@metafiles && (scalar keys %pids < $parallelism) ? 0.1:1);
339    
340     $::log->verb(6,"PLOOP: select with timeout $timeout");
341     my ( $rh_set ) = IO::Select->select($readset, undef, undef, $timeout);
342    
343     foreach my $fh ( @$rh_set ) {
344     my $metafile = $metafile_by_fh{$fh};
345     # we know there is at least one byte to read, but also that
346     # any client sends complete
347     while (1) {
348     my $char;
349     my $length = sysread $fh, $char, 1;
350     if ( $length ) {
351     $logoutput{$metafile} .= $char;
352     $char eq "\n" and last;
353     } else {
354     #expected a char but got eof
355     $readset->remove($fh);
356     close($fh);
357     map {
358     $pids{$_} == $fh and
359     waitpid($_,WNOHANG) and
360     delete $pids{$_} and
361     $::log->verb(5,"Collected pid $_ (rc=$?),",
362     length($logoutput{$metafile}),"bytes log output");
363     } keys %pids;
364     last;
365     }
366     }
367     }
368     }
369    
370     # log out all collected log data from our children
371     foreach my $metafile ( sort keys %logoutput ) {
372     foreach my $line ( split(/\n/,$logoutput{$metafile}) ) {
373     $line =~ /^ERROR\s+(.*)$/ and $::log->err($1);
374     $line =~ /^WARN\s+(.*)$/ and $::log->warn($1);
375     $line =~ /^VERBOSE\((\d+)\)\s+(.*)$/ and $::log->verb($1,$2);
376     $line =~ /^DEBUG\((\d+)\)\s+(.*)$/ and $::log->debug($1,$2);
377     }
378     }
379    
380     return 1;
381     }

Properties

Name Value
svn:executable *

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28