/[pdpsoft]/nl.nikhef.pdp.fetchcrl/tags/fetch-crl-3.0.17-2/fetch-crl3.pl.cin
ViewVC logotype

Contents of /nl.nikhef.pdp.fetchcrl/tags/fetch-crl-3.0.17-2/fetch-crl3.pl.cin

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2819 - (show annotations) (download)
Wed Dec 23 16:02:37 2015 UTC (5 years, 11 months ago) by davidg
File size: 16163 byte(s)
SUSE build patches

1 #! /usr/bin/perl -w
2 #
3 # @(#)$Id$
4 #
5 # Copyright 2010 David Groep, Nationaal instituut voor
6 # subatomaire fysica NIKHEF
7 #
8 # Licensed under the Apache License, Version 2.0 (the "License");
9 # you may not use this file except in compliance with the License.
10 # You may obtain a copy of the License at
11 #
12 # http://www.apache.org/licenses/LICENSE-2.0
13 #
14 # Unless required by applicable law or agreed to in writing, software
15 # distributed under the License is distributed on an "AS IS" BASIS,
16 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 # See the License for the specific language governing permissions and
18 # limitations under the License.
19 #
20 #
21 package main;
22
23 use strict;
24 use Getopt::Long qw(:config no_ignore_case bundling);
25 use POSIX;
26 eval { require LWP or die; }; $@ and die "Please install libwww-perl (LWP)\n";
27
28 my $sccsid = '@(#)$Id$';
29
30 # import modules that are needed but still external
31 # (the installed version may have these packages embedded in-line)
32 #
33 require ConfigTiny and import ConfigTiny unless defined &ConfigTiny::new;
34 require TrustAnchor and import TrustAnchor unless defined &TrustAnchor::new;
35 require CRLWriter and import CRLWriter unless defined &CRLWriter::new;
36 require FCLog and import FCLog unless defined &FCLog::new;
37 require OSSL and import OSSL unless defined &OSSL::new;
38 require CRL and import CRL unless defined &CRL::new;
39
40 my $use_DataDumper = eval { require Data::Dumper; };
41 my $use_IOSelect = eval { require IO::Select; };
42
43 use vars qw/ $log $cnf /;
44
45
46 # ###########################################################################
47 #
48 #
49 ($cnf,$log) = &init_configuration();
50
51 # use Net::INET6Glue if so requested (is not a default module)
52 if ( $cnf->{_}->{inet6glue} ) {
53 eval { require Net::INET6Glue::INET_is_INET6 or die; };
54 $@ and die "Please install Net::INET6Glue before enabling inet6glue config\n";
55 }
56
57 # verify local installation sanity for loaded modules
58 $::log->getverbose > 6 and ! $use_DataDumper and
59 $::log->err("Cannot set verbosity higher than 6 without Data::Dumper") and
60 exit(1);
61 $::cnf->{_}->{parallelism} and ! $use_IOSelect and
62 $::log->err("Cannot use parallel retrieval without IO::Select") and
63 exit(1);
64
65 $use_DataDumper and $::log->verb(7,Data::Dumper::Dumper($cnf));
66
67 # set safe path if so requested
68 $cnf->{_}->{path} and $ENV{"PATH"} = $cnf->{_}->{path} and
69 $::log->verb(5,"Set PATH to",$ENV{"PATH"});
70
71 # set rcmode if present in config
72 defined $cnf->{_}->{rcmode} and do {
73 $::log->verb(4,"Setting exit status mode to ".$cnf->{_}->{rcmode});
74 $::log->setrcmode($cnf->{_}->{rcmode}) or exit($log->exitstatus);
75 $::log->verb(2,"Exit status mode is set to ".$cnf->{_}->{rcmode});
76 };
77
78 # wait up to randomwait seconds to spread download load
79 $cnf->{_}->{randomwait} and do {
80 my $wtime = int(rand($cnf->{_}->{randomwait}));
81 $::log->verb(2,"Sleeping $wtime seconds before continuing");
82 sleep($wtime);
83 };
84
85
86 # the list of trust anchors to process comes from the command line and
87 # all files in the infodir that are metadata or crl urls
88 # in the next phase, the suffix will be stripped and the info file
89 # when present preferred over the crlurl
90 #
91 my @metafiles = @ARGV;
92 $::cnf->{_}->{"infodir"} and do {
93 foreach my $fn (
94 map { glob ( $::cnf->{_}->{"infodir"} . "/$_" ); } "*.info", "*.crl_url"
95 ) {
96 next if $::cnf->{_}->{nosymlinks} and -l $fn;
97 $fn =~ /.*\/([^\/]+)(\.crl_url|\.info)$/;
98 push @metafiles, $1 unless grep /^$1$/,@metafiles or not defined $1;
99 }
100 };
101
102 @metafiles or
103 $log->warn("No trust anchors to process") and exit($log->exitstatus);
104
105 if ( $::cnf->{_}->{parallelism} ) {
106 &parallel_metafiles($::cnf->{_}->{parallelism}, @metafiles);
107 } else {
108 &process_metafiles( @metafiles );
109 }
110
111 $log->flush;
112 exit($log->exitstatus);
113
114
115 # ###########################################################################
116 #
117 #
118 sub init_configuration() {
119 my ($cnf,$log);
120
121 my ($configfile,$agingtolerance,$infodir,$statedir,$cadir,$httptimeout);
122 my ($output);
123 my @formats;
124 my $verbosity;
125 my $quiet=0;
126 my $help=0;
127 my $showversion=0;
128 my $debuglevel;
129 my $parallelism=0;
130 my $randomwait;
131 my $nosymlinks;
132 my $cfgdir;
133 my $inet6glue=0;
134 my %directives;
135
136 $log = FCLog->new("qualified");
137
138 &GetOptions(
139 "c|config=s" => \$configfile,
140 "l|infodir=s" => \$infodir,
141 "cadir=s" => \$cadir,
142 "s|statedir=s" => \$statedir,
143 "cfgdir=s" => \$cfgdir,
144 "T|httptimeout=i" => \$httptimeout,
145 "o|output=s" => \$output,
146 "format=s@" => \@formats,
147 "define=s" => \%directives,
148 "v|verbose+" => \$verbosity,
149 "h|help+" => \$help,
150 "V|version+" => \$showversion,
151 "q|quiet+" => \$quiet,
152 "d|debug+" => \$debuglevel,
153 "p|parallelism=i" => \$parallelism,
154 "nosymlinks+" => \$nosymlinks,
155 "a|agingtolerance=i" => \$agingtolerance,
156 "r|randomwait=i" => \$randomwait,
157 "inet6glue+" => \$inet6glue,
158 ) or &help and exit(1);
159
160 $help and &help and exit(0);
161 $showversion and &showversion and exit(0);
162
163 $configfile ||= ( -e "/etc/fetch-crl.conf" and "/etc/fetch-crl.conf" );
164 $configfile ||= ( -e "/etc/fetch-crl.cnf" and "/etc/fetch-crl.cnf" );
165
166 $cnf = ConfigTiny->new();
167 $configfile and
168 $cnf->read($configfile) || die "Invalid config file $configfile:\n " .
169 $cnf->errstr . "\n";
170
171 ( defined $cnf->{_}->{cfgdir} and $cfgdir = $cnf->{_}->{cfgdir} )
172 unless defined $cfgdir;
173 $cfgdir ||= "/etc/fetch-crl.d";
174 if ( defined $cfgdir and -d $cfgdir and opendir(my $dh,$cfgdir) ) {
175 while ( my $fn = readdir $dh ) {
176 -f "$cfgdir/$fn" and -r "$cfgdir/$fn" and $cnf->read("$cfgdir/$fn");
177 }
178 close $dh;
179 }
180
181 # add defined from the command line to the configuration, to the
182 # main section _ thereof unless there is a colon in the key
183 foreach my $k ( keys %directives ) {
184 my $section ="_";
185 my $dvalue = $directives{$k};
186 if ( $k =~ m/(\w+):(.*)/ ) {
187 $section = $1;
188 $k=$2;
189 }
190 $cnf->{$section}->{$k} = $dvalue;
191 }
192
193 # command-line option overrides
194 $cnf->{_}->{agingtolerance} = $agingtolerance if defined $agingtolerance;
195 $cnf->{_}->{infodir} = $infodir if defined $infodir;
196 $cnf->{_}->{cadir} = $cadir if defined $cadir;
197 $cnf->{_}->{statedir} = $statedir if defined $statedir;
198 $cnf->{_}->{httptimeout} = $httptimeout if defined $httptimeout;
199 $cnf->{_}->{verbosity} = $verbosity if defined $verbosity;
200 $cnf->{_}->{debuglevel} = $debuglevel if defined $debuglevel;
201 $cnf->{_}->{output} = $output if defined $output;
202 $cnf->{_}->{formats} = join "\001",@formats if @formats;
203 $cnf->{_}->{parallelism} = $parallelism if $parallelism;
204 $cnf->{_}->{randomwait} = $randomwait if defined $randomwait;
205 $cnf->{_}->{nosymlinks} = $nosymlinks if defined $nosymlinks;
206 $cnf->{_}->{inet6glue} = $inet6glue if $inet6glue;
207
208 # deal with interaction of verbosity in logfile and quiet option
209 # since a noquiet config option can cancel it
210 if ( not defined $cnf->{_}->{noquiet} ) {
211 if ( $quiet == 1) { $cnf->{_}->{verbosity} = -1; }
212 } else {
213 if ( $quiet >= 2) { $cnf->{_}->{verbosity} = -1; }
214 }
215
216 # key default values
217 defined $cnf->{_}->{version} or $cnf->{_}->{version} = "3+";
218 defined $cnf->{_}->{packager} or $cnf->{_}->{packager} = "EUGridPMA";
219 defined $cnf->{_}->{openssl} or $cnf->{_}->{openssl} = "openssl";
220 defined $cnf->{_}->{agingtolerance} or $cnf->{_}->{agingtolerance} ||= 24;
221 defined $cnf->{_}->{infodir} or $cnf->{_}->{infodir} = '/etc/grid-security/certificates';
222 defined $cnf->{_}->{output} or $cnf->{_}->{output} = $cnf->{_}->{infodir};
223 defined $cnf->{_}->{cadir} or $cnf->{_}->{cadir} = $cnf->{_}->{infodir};
224 defined $cnf->{_}->{statedir} or $cnf->{_}->{statedir} = "/var/cache/fetch-crl" if -d "/var/cache/fetch-crl" and -w "/var/cache/fetch-crl";
225 defined $cnf->{_}->{formats} or $cnf->{_}->{formats} = "openssl";
226 defined $cnf->{_}->{opensslmode} or $cnf->{_}->{opensslmode} = "dual";
227 defined $cnf->{_}->{httptimeout} or $cnf->{_}->{httptimeout} = 120;
228 defined $cnf->{_}->{expirestolerance} or $cnf->{_}->{expirestolerance} = (7*60*60); # at least 7 hrs should nextUpdate be beyond the cache FreshUntil
229 defined $cnf->{_}->{maxcachetime} or $cnf->{_}->{maxcachetime} = (4*24*60*60); # arbitrarily set it at 4 days
230 defined $cnf->{_}->{nametemplate_der} or
231 $cnf->{_}->{nametemplate_der} = "\@ANCHORNAME\@.\@R\@.crl";
232 defined $cnf->{_}->{nametemplate_pem} or
233 $cnf->{_}->{nametemplate_pem} = "\@ANCHORNAME\@.\@R\@.crl.pem";
234 defined $cnf->{_}->{catemplate} or
235 $cnf->{_}->{catemplate} = "\@ALIAS\@.pem\001".
236 "\@ALIAS\@.\@R\@\001\@ANCHORNAME\@.\@R\@";
237
238 $cnf->{_}->{nonssverify} ||= 0;
239 $cnf->{_}->{nocache} ||= 0;
240 $cnf->{_}->{nosymlinks} ||= 0;
241 $cnf->{_}->{verbosity} ||= 0;
242 $cnf->{_}->{debuglevel} ||= 0;
243 $cnf->{_}->{inet6glue} ||= 0;
244
245 $cnf->{_}->{stateless} and delete $cnf->{_}->{statedir};
246
247 # expand array keys in config
248 defined $cnf->{_}->{formats} and
249 @{$cnf->{_}->{formats_}} = split(/[\001;,\s]+/,$cnf->{_}->{formats});
250
251 # sanity check on configuration
252 $cnf->{_}->{statedir} and ! -d $cnf->{_}->{statedir} and
253 die "Invalid state directory " . $cnf->{_}->{statedir} . "\n";
254 $cnf->{_}->{infodir} and ! -d $cnf->{_}->{infodir} and
255 die "Invalid meta-data directory ".$cnf->{_}->{infodir}."\n";
256
257 # initialize logging
258 $log->flush;
259 $cnf->{_}->{logmode} and $log->destremove("qualified") and do {
260 foreach ( split(/[,\001]+/,$cnf->{_}->{logmode}) ) {
261 if ( /^syslog$/ ) { $log->destadd($_,$cnf->{_}->{syslogfacility}); }
262 elsif ( /^(direct|qualified|cache)$/ ) { $log->destadd($_); }
263 else { die "Invalid log destination $_, exiting.\n"; }
264 }
265 };
266 $log->setverbose($cnf->{_}->{verbosity});
267 $log->setdebug($cnf->{_}->{debuglevel});
268
269 return ($cnf,$log);
270 }
271
272 # ###########################################################################
273 #
274 #
275 sub showversion() {
276 (my $name = $0) =~ s/.*\///;
277 print "$name version @VERSION@\n";
278 return 1;
279 }
280
281 sub help() {
282 (my $name = $0) =~ s/.*\///;
283 print <<EOHELP;
284 The fetch-crl utility will retrieve certificate revocation lists (CRLs) for
285 a set of installed trust anchors, based on crl_url files or IGTF-style info
286 files. It will install these for use with OpenSSL, NSS or third-party tools.
287
288 Usage: $name [-c|--config configfile] [-l|--infodir path]
289 [--cadir path] [-s|--statedir path] [-o|--output path] [--format \@formats]
290 [-T|--httptimeout seconds] [-p|--parallelism n] [--nosymlinks]
291 [-a|--agingtolerance hours] [-r|--randomwait seconds]
292 [-v|--verbose] [-h|--help] [-q|--quiet] [-d|--debug level]
293
294 Options:
295 -c | --config path
296 Read configuration data from path, default: /etc/fetch-crl.conf
297 -l | --infodir path
298 Location of the trust anchor meta-data files (crl_url or info),
299 default: /etc/grid-security/certificates
300 --cadir path
301 Location of the trust anchors (default to infodir)
302 -s | --statedir path
303 Location of the historic state data (for caching and delayed-warning)
304 -T | --httptimeout sec
305 Maximum time in seconds to wait for retrieval or a single URL
306 -o | --output path
307 Location of the CRLs written (global default, defaults to infodir
308 --format \@formats
309 Format(s) in which the CRLs will be written (openssl, pem, der, nss)
310 --nosymlinks
311 Do not include meta-data files that are symlinks
312 -v | --verbose
313 Become more talkative
314 -q | --quiet
315 Become really quiet (overrides verbosity)
316 -p | --parallelism n
317 Run up to n parallel trust anchor retrieval processes
318 -a | --agingtolerance hours
319 Be quiet for up to hours hours before raising an error. Until
320 the tolerance has passed, only warnings are raised
321 -r | --randomwait seconds
322 Introduce a random delay of up to seconds seconds before starting
323 any retrieval processes
324 -h | --help
325 This help text
326
327 Version: @VERSION@
328 EOHELP
329
330 return 1;
331 }
332
333 # ###########################################################################
334 #
335 #
336 sub process_metafiles(@) {
337 my @metafiles = @_;
338
339 foreach my $f ( @metafiles ) {
340 my $ta = TrustAnchor->new();
341 $cnf->{_}->{"infodir"} and $ta->setInfodir($cnf->{_}->{"infodir"});
342 $ta->loadAnchor($f) or next;
343 $ta->saveLogMode() and $ta->setLogMode();
344 $ta->loadState() or next;
345
346 # using the HASH in the CA filename templates requires the CRL
347 # is retrieved first to determinte the hash
348 if ( $cnf->{_}->{"catemplate"} =~ /\@HASH\@/ ) {
349 $ta->retrieve or next;
350 $ta->loadCAfiles() or next;
351 } else {
352 $ta->loadCAfiles() or next;
353 $ta->retrieve or next;
354 }
355
356 $ta->verifyAndConvertCRLs or next;
357
358 my $writer = CRLWriter->new($ta);
359 $writer->writeall() or next;
360 $ta->saveState() or next;
361 $ta->restoreLogMode();
362 }
363
364 return 1;
365 }
366
367 sub parallel_metafiles($@) {
368 my $parallelism = shift;
369 my @metafiles = @_;
370
371 my %pids = (); # file handle by processID
372 my %metafile_by_fh = (); # reverse map
373 my $readset = new IO::Select();
374 my %logoutput = ();
375
376 $| = 1;
377
378 $::log->verb(2,"starting up to $parallelism worker processes");
379
380 while ( @metafiles or scalar keys %pids ) {
381 # loop until we have started all possible retrievals AND have
382 # collected all possible output
383
384 ( @metafiles and (scalar keys %pids < $parallelism) ) and do {
385 # we have metafiles left, and have spare process slots
386 my $metafile = shift @metafiles;
387
388
389 $logoutput{$metafile} = "";
390
391 my $cout;
392 my $cpid = open $cout, "-|";
393 defined $cpid and defined $cout or
394 $::log->err("Cannot fork ($metafile): $!") and next;
395
396 $::log->verb(5,"LOOP: starting process $cpid for $metafile");
397
398 if ( $cpid == 0 ) { # I'm the child that should care for $metafile
399 $0 = "fetch-crl worker $metafile";
400 $::log->cleanse();
401 $::log->destadd("qualified");
402 &process_metafiles($metafile);
403 $::log->flush;
404 exit($::log->exitstatus);
405 } else { # parent
406 $pids{$cpid} = $cout;
407 $readset->add($cout);
408 $metafile_by_fh{$cout} = $metafile;
409 }
410 };
411
412 # do a select loop over the outstanding requests to collect messages
413 # if we are in the process of starting more processes, we just
414 # briefly poll out pending output so as not to have blocking
415 # children, but if we have started as many children as we ought to
416 # we put in a longer timeout -- any output on a handle will
417 # get us out of the select and into flushing mode again
418 my $timeout = (@metafiles && (scalar keys %pids < $parallelism) ? 0.1:1);
419
420 $::log->verb(6,"PLOOP: select with timeout $timeout");
421 my ( $rh_set ) = IO::Select->select($readset, undef, undef, $timeout);
422
423 foreach my $fh ( @$rh_set ) {
424 my $metafile = $metafile_by_fh{$fh};
425 # we know there is at least one byte to read, but also that
426 # any client sends complete
427 while (1) {
428 my $char;
429 my $length = sysread $fh, $char, 1;
430 if ( $length ) {
431 $logoutput{$metafile} .= $char;
432 $char eq "\n" and last;
433 } else {
434 #expected a char but got eof
435 $readset->remove($fh);
436 close($fh);
437 map {
438 $pids{$_} == $fh and
439 waitpid($_,WNOHANG) and
440 delete $pids{$_} and
441 $::log->verb(5,"Collected pid $_ (rc=$?),",
442 length($logoutput{$metafile}),"bytes log output");
443 } keys %pids;
444 last;
445 }
446 }
447 }
448 }
449
450 # log out all collected log data from our children
451 foreach my $metafile ( sort keys %logoutput ) {
452 foreach my $line ( split(/\n/,$logoutput{$metafile}) ) {
453 $line =~ /^ERROR\s+(.*)$/ and $::log->err($1);
454 $line =~ /^WARN\s+(.*)$/ and $::log->warn($1);
455 $line =~ /^VERBOSE\((\d+)\)\s+(.*)$/ and $::log->verb($1,$2);
456 $line =~ /^DEBUG\((\d+)\)\s+(.*)$/ and $::log->debug($1,$2);
457 }
458 }
459
460 return 1;
461 }

Properties

Name Value
svn:executable *
svn:keywords Date Author Revision Id

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28