Annotation of /trunk/nl.nikhef.ndpf.tools/ndpf-dpm-tools/bin/dpm-disk-consistency

#!/usr/bin/perl

#
# Tool to check the consistency between the DPM database
# and the files actually on disk.
#

use strict;

use Getopt::Long;
use Sys::Hostname;
use DBI;
use Digest::Adler32;
use Digest::MD5;
use DPM::Utils;

my $help = 0;
my $verbose = 0;
my $resolve = 0;
my $checksum_check = 0;

# if $dbfile is not empty, the DB data will be read from it
# instead of obtained by executing gridpp_dpm_disk
# my $dbfile = '/home/ronalds/tmp/DPM/dpm_db_dump.txt';
my $dbfile;

my $cfgfile = '/root/DPMINFO';

my $logfile;
##$logfile = "-";      # STDOUT for debugging
my $filesystem;

GetOptions(
    'help|h'            => \$help,
    'verbose|v+'        => \$verbose,
    'logfile|l:s'       => \$logfile,
    'dbfile:s'          => \$dbfile,
    'cfgfile:s'         => \$cfgfile,
    'resolve|r'         => \$resolve,
    'sum|s'             => \$checksum_check
);

$help and &usage;

my @now = localtime(time);
my $date = sprintf "%04d%02d%02d-%02d%02d%02d",
    $now[5]+1900, $now[4]+1, $now[3], $now[2], $now[1], $now[0];
$logfile ||= "dpm-consistency-check-$date";

open LOG, "> $logfile" or die "$logfile: $!\n";
($verbose > 1) and print LOG "Log file: $logfile\n";

my %replicas;
my %csumtype;
my %csumvalue;
my %count = ( db => 0, disk => 0, only_db => 0, only_disk => 0, db_and_disk => 0, unknown => 0 );
my $REPLICA_IN_DB = 1;
my $REPLICA_ON_DISK = 2;
my @only_db;
my @only_disk;

my ($db_user, $db_pw, $db_host, $dpns_db_name, $db_port, $dbh);

my $filesystem = shift;
$filesystem =~ s!/*$!!;
if ( ! $filesystem ) {
    warn "Missing mandatory argument 'filesystem'\n";
    &usage(1);
}
if ( ! -d $filesystem ) {
    warn "$!: $filesystem\n";
}

my $server = hostname;

($verbose > 0) and print LOG "server: $server filesystem: $filesystem\n";

# Initial design, unoptimized:
# 1. Determine all replicas registered in the DB that are stored 
#    on the current server for the given file system.
#    Store the replicas in hash %replicas with key replica 
#    and value $REPLICA_ON_DISK
# 2. Determine all files that are present 
#    on the current server for the given file system.
#    Store the files in hash %replicas with key file
#    and value $REPLICA_IN_DB
# 3. Compare the hashes, determining inconsistencies and 
#    counting statistics
#
# Later:
# -  Add an option to automatically remove entries from the
#    database for which the corresponding file is missing,
#    or remove the file from disk if there is no corresponding
#    entry in the database
# -  Memory consumption might be an issue for this script;
#    consider splitting the processing by date

my $dpm = DPM::Utils->new(conffile => $cfgfile, host => $server);


# Determine replicas registered in the database
&get_db_replicas(\%replicas, \%csumtype, \%csumvalue);

# Determine replicas on disk
&get_fs_replicas(\%replicas);

# Comparison
&compare_database_filesystem(\%replicas, \%csumtype, \%csumvalue);

# Results
&show_result;

close LOG;
exit 0;


sub usage {
    my $ret = shift;
    print STDERR <<EOH;
usage: $0 [options] <filesystem>
options:
    help, h                 Show this help text
    verbose, v              Increase output verbosity
    resolve, r              Resolve the path to a SURL
    sum, s                  Calculate and compare the checksum
EOH

    exit($ret);
}


sub get_db_replicas {
    my $ref = shift;
    my $csumtyperef = shift;
    my $csumvalueref = shift;

    # read the replicas registered in the database,
    # unless a file for debugging is defined
    if ( ! $dbfile ) {
        # $dbfile = '/tmp/dpm-db-${date}.dump';
        # execute the gridpp_* command
        # my $cmd = "gridpp_dpm_disk --server $server --fs $filesystem > $dbfile";
        # ($verbose > 0) and print LOG "Executing $cmd\n";

        my $sql = "SELECT r.sfn, m.csumtype, m.csumvalue FROM Cns_file_replica r, Cns_file_metadata m WHERE "
            . "r.fileid = m.fileid AND "
              . "r.status <> 'D' AND "
                . "r.host=" . $dpm->sql_quote($server) . " AND r.fs=" . $dpm->sql_quote($filesystem);
        my @results = $dpm->sql_query($sql);

        foreach my $row (@results) {
            my $replica = $$row{sfn};
            my $csumtype = $$row{csumtype};
            my $csumvalue = $$row{csumvalue};
            $replica =~ s!^$server:$filesystem[/]*!!;
            ($verbose > 2) and print LOG "$replica\n";
            $$ref{$replica} |= $REPLICA_IN_DB;
            $$csumtyperef{$replica} = $csumtype;
            $$csumvalueref{$replica} = $csumvalue;
            $count{db}++;
        }
    }

    ($verbose > 0) and print LOG "Found $count{db} replicas in the database under $filesystem\n";
}

sub get_fs_replicas {
    my $ref = shift;

    # find all files under the specified filesystem
    my $cmd = "find $filesystem -type f -print";

    open FS, "$cmd | " or die "$cmd: $!\n";
    while ( my $file = <FS> ) {
        $file =~ s!^$filesystem[/]*!!;
        chomp $file;
        ($verbose > 2) and print LOG "$file\n";
        $$ref{$file} |= $REPLICA_ON_DISK;
        $count{disk}++;
    }
    close FS;

    ($verbose > 0) and print LOG "Found $count{disk} files under $filesystem\n";
}


sub compare_database_filesystem {
    my $replica_ref = shift;
    my $csumtyperef = shift;
    my $csumvalueref = shift;

    my $db_and_disk = ($REPLICA_IN_DB | $REPLICA_ON_DISK);
    while ( my ($repl, $state) = each(%$replica_ref) ) {
        ($verbose > 1) and print "$repl $state\t";
        if ( $state == $REPLICA_IN_DB ) {
            $count{only_db}++;
            push(@only_db, $repl);
            ($verbose > 1) and print LOG "[only in DB]\n";
        }
        elsif ( $state == $REPLICA_ON_DISK ) {
            $count{only_disk}++;
            push(@only_disk, $repl);
            ($verbose > 1) and print LOG "[only on disk]\n";
        }
        elsif ( $state == $db_and_disk ) {
            $count{db_and_disk}++;
            if ($checksum_check) {
              check_checksum($repl, $csumtyperef, $csumvalueref);
            }
            if ($resolve) {
              check_resolve($repl);
            }
            ($verbose > 1) and print LOG "[OK]\n";
        }
        else {
            $count{unknown}++;
            ($verbose > 1) and print LOG "[Unexpected state]\n";
        }
    }
}


sub check_checksum {
  my $replica = shift;
  my $csumtyperef = shift;
  my $csumvalueref = shift;
  my $cksum;
  my $digest;

  if ( ! $$csumtyperef{$replica} ) {
    print LOG "cksum: $replica has no recorded checksum type.\n";
    return;
  }
  if ($$csumtyperef{$replica} == "AD") {
    $digest = Digest::Adler32->new;
  } elsif ($$csumtyperef{$replica} == "MD") {
    $digest = Digest::MD5->new;
  } else {
    print LOG "cksum: unknown checksum type '" . $$csumtyperef{$replica} . "'\n";
    return;
  }

  if (open(my $fh, "<", $filesystem . "/" . $replica)) {
    print STDERR "Calculating sum of $filesystem/$replica...\n";
    $digest->addfile($fh);
    $cksum = $digest->hexdigest;
    close $fh;
  } else {
    print LOG "cksum: could not open $replica\n";
    return;
  }
  if ($cksum != $$csumvalueref{$replica}) {
    print LOG "cksum: mismatch for $replica (current $cksum, database has " .
      $$csumvalueref{$replica} . ", type " . $$csumtyperef{$replica} . ")\n";
  } else {
    print LOG "cksum: ok for $replica ($cksum, type " . $$csumtyperef{$replica} . ")\n";
  }
}

sub check_resolve {
  my $replica = shift;
  # try a full path resolution
  my $surl = $dpm->replica2surl("$server:/$replica");

  if (defined $surl) {
    print LOG "resolve: $replica -> $surl\n";
  } else {
    print LOG "resolve: failed for $replica\n";
  }
  return;
}

sub show_result {
    print LOG "# replicas in DB: $count{db}\n";
    print LOG "# replicas on disk: $count{disk}\n";
    print LOG "# replicas in DB and on disk: $count{db_and_disk}\n";
    print LOG "# replicas only in the DB: $count{only_db}\n";
    print LOG "# replicas only on disk: $count{only_disk}\n";

    print LOG "\n# Replicas only in the database\n";
    foreach my $repl (sort @only_db) {
        print LOG "$server:$filesystem/$repl\n";
    }

    print LOG "\n# Replicas only on the fileystem\n";
    foreach my $repl (sort @only_disk) {
        print LOG "$server:$filesystem/$repl\n";
    }

    print LOG "\n# times: " . join("\t", times);
}

1	ronalds	1810	#!/usr/bin/perl
2
3			#
4			# Tool to check the consistency between the DPM database
5			# and the files actually on disk.
6			#
7
8			use strict;
9
10			use Getopt::Long;
11			use Sys::Hostname;
12			use DBI;
13	dennisvd	3182	use Digest::Adler32;
14			use Digest::MD5;
15			use DPM::Utils;
16	ronalds	1810
17			my $help = 0;
18			my $verbose = 0;
19	dennisvd	3182	my $resolve = 0;
20			my $checksum_check = 0;
21	ronalds	1810
22			# if $dbfile is not empty, the DB data will be read from it
23			# instead of obtained by executing gridpp_dpm_disk
24			# my $dbfile = '/home/ronalds/tmp/DPM/dpm_db_dump.txt';
25			my $dbfile;
26
27			my $cfgfile = '/root/DPMINFO';
28
29			my $logfile;
30			##$logfile = "-"; # STDOUT for debugging
31			my $filesystem;
32
33			GetOptions(
34			'help\|h' => \$help,
35			'verbose\|v+' => \$verbose,
36			'logfile\|l:s' => \$logfile,
37			'dbfile:s' => \$dbfile,
38			'cfgfile:s' => \$cfgfile,
39	dennisvd	3182	'resolve\|r' => \$resolve,
40			'sum\|s' => \$checksum_check
41	ronalds	1810	);
42
43			$help and &usage;
44
45			my @now = localtime(time);
46			my $date = sprintf "%04d%02d%02d-%02d%02d%02d",
47			$now[5]+1900, $now[4]+1, $now[3], $now[2], $now[1], $now[0];
48			$logfile \|\|= "dpm-consistency-check-$date";
49
50			open LOG, "> $logfile" or die "$logfile: $!\n";
51			($verbose > 1) and print LOG "Log file: $logfile\n";
52
53			my %replicas;
54	dennisvd	3182	my %csumtype;
55			my %csumvalue;
56	ronalds	1810	my %count = ( db => 0, disk => 0, only_db => 0, only_disk => 0, db_and_disk => 0, unknown => 0 );
57			my $REPLICA_IN_DB = 1;
58			my $REPLICA_ON_DISK = 2;
59			my @only_db;
60			my @only_disk;
61
62			my ($db_user, $db_pw, $db_host, $dpns_db_name, $db_port, $dbh);
63
64			my $filesystem = shift;
65			$filesystem =~ s!/*$!!;
66			if ( ! $filesystem ) {
67			warn "Missing mandatory argument 'filesystem'\n";
68			&usage(1);
69			}
70			if ( ! -d $filesystem ) {
71			warn "$!: $filesystem\n";
72			}
73
74			my $server = hostname;
75
76	dennisvd	3182	($verbose > 0) and print LOG "server: $server filesystem: $filesystem\n";
77	ronalds	1810
78			# Initial design, unoptimized:
79			# 1. Determine all replicas registered in the DB that are stored
80			# on the current server for the given file system.
81			# Store the replicas in hash %replicas with key replica
82	dennisvd	3182	# and value $REPLICA_ON_DISK
83	ronalds	1810	# 2. Determine all files that are present
84			# on the current server for the given file system.
85			# Store the files in hash %replicas with key file
86			# and value $REPLICA_IN_DB
87			# 3. Compare the hashes, determining inconsistencies and
88			# counting statistics
89			#
90			# Later:
91			# - Add an option to automatically remove entries from the
92			# database for which the corresponding file is missing,
93			# or remove the file from disk if there is no corresponding
94			# entry in the database
95			# - Memory consumption might be an issue for this script;
96			# consider splitting the processing by date
97
98	dennisvd	3182	my $dpm = DPM::Utils->new(conffile => $cfgfile, host => $server);
99	ronalds	1810
100
101			# Determine replicas registered in the database
102	dennisvd	3182	&get_db_replicas(\%replicas, \%csumtype, \%csumvalue);
103	ronalds	1810
104			# Determine replicas on disk
105			&get_fs_replicas(\%replicas);
106
107			# Comparison
108	dennisvd	3182	&compare_database_filesystem(\%replicas, \%csumtype, \%csumvalue);
109	ronalds	1810
110			# Results
111			&show_result;
112
113			close LOG;
114			exit 0;
115
116
117
118			sub usage {
119			my $ret = shift;
120			print STDERR <<EOH;
121			usage: $0 [options] <filesystem>
122			options:
123			help, h Show this help text
124			verbose, v Increase output verbosity
125	dennisvd	3182	resolve, r Resolve the path to a SURL
126			sum, s Calculate and compare the checksum
127	ronalds	1810	EOH
128
129			exit($ret);
130	dennisvd	3182	}
131	ronalds	1810
132
133			sub get_db_replicas {
134			my $ref = shift;
135	dennisvd	3182	my $csumtyperef = shift;
136			my $csumvalueref = shift;
137	ronalds	1810
138			# read the replicas registered in the database,
139			# unless a file for debugging is defined
140			if ( ! $dbfile ) {
141			# $dbfile = '/tmp/dpm-db-${date}.dump';
142			# execute the gridpp_* command
143			# my $cmd = "gridpp_dpm_disk --server $server --fs $filesystem > $dbfile";
144			# ($verbose > 0) and print LOG "Executing $cmd\n";
145
146	dennisvd	3182	my $sql = "SELECT r.sfn, m.csumtype, m.csumvalue FROM Cns_file_replica r, Cns_file_metadata m WHERE "
147			. "r.fileid = m.fileid AND "
148			. "r.status <> 'D' AND "
149			. "r.host=" . $dpm->sql_quote($server) . " AND r.fs=" . $dpm->sql_quote($filesystem);
150			my @results = $dpm->sql_query($sql);
151
152	ronalds	1810	foreach my $row (@results) {
153			my $replica = $$row{sfn};
154	dennisvd	3182	my $csumtype = $$row{csumtype};
155			my $csumvalue = $$row{csumvalue};
156	ronalds	1810	$replica =~ s!^$server:$filesystem[/]*!!;
157			($verbose > 2) and print LOG "$replica\n";
158			$$ref{$replica} \|= $REPLICA_IN_DB;
159	dennisvd	3182	$$csumtyperef{$replica} = $csumtype;
160			$$csumvalueref{$replica} = $csumvalue;
161	ronalds	1810	$count{db}++;
162			}
163			}
164
165			($verbose > 0) and print LOG "Found $count{db} replicas in the database under $filesystem\n";
166			}
167
168			sub get_fs_replicas {
169			my $ref = shift;
170
171			# find all files under the specified filesystem
172			my $cmd = "find $filesystem -type f -print";
173
174			open FS, "$cmd \| " or die "$cmd: $!\n";
175			while ( my $file = <FS> ) {
176			$file =~ s!^$filesystem[/]*!!;
177			chomp $file;
178			($verbose > 2) and print LOG "$file\n";
179			$$ref{$file} \|= $REPLICA_ON_DISK;
180			$count{disk}++;
181			}
182			close FS;
183
184			($verbose > 0) and print LOG "Found $count{disk} files under $filesystem\n";
185			}
186
187
188			sub compare_database_filesystem {
189	dennisvd	3182	my $replica_ref = shift;
190			my $csumtyperef = shift;
191			my $csumvalueref = shift;
192	ronalds	1810
193			my $db_and_disk = ($REPLICA_IN_DB \| $REPLICA_ON_DISK);
194			while ( my ($repl, $state) = each(%$replica_ref) ) {
195			($verbose > 1) and print "$repl $state\t";
196			if ( $state == $REPLICA_IN_DB ) {
197			$count{only_db}++;
198			push(@only_db, $repl);
199			($verbose > 1) and print LOG "[only in DB]\n";
200			}
201			elsif ( $state == $REPLICA_ON_DISK ) {
202			$count{only_disk}++;
203			push(@only_disk, $repl);
204			($verbose > 1) and print LOG "[only on disk]\n";
205			}
206			elsif ( $state == $db_and_disk ) {
207			$count{db_and_disk}++;
208	dennisvd	3182	if ($checksum_check) {
209			check_checksum($repl, $csumtyperef, $csumvalueref);
210			}
211			if ($resolve) {
212			check_resolve($repl);
213			}
214	ronalds	1810	($verbose > 1) and print LOG "[OK]\n";
215			}
216			else {
217			$count{unknown}++;
218			($verbose > 1) and print LOG "[Unexpected state]\n";
219			}
220			}
221			}
222
223
224	dennisvd	3182	sub check_checksum {
225			my $replica = shift;
226			my $csumtyperef = shift;
227			my $csumvalueref = shift;
228			my $cksum;
229			my $digest;
230	ronalds	1810
231	dennisvd	3182	if ( ! $$csumtyperef{$replica} ) {
232			print LOG "cksum: $replica has no recorded checksum type.\n";
233			return;
234			}
235			if ($$csumtyperef{$replica} == "AD") {
236			$digest = Digest::Adler32->new;
237			} elsif ($$csumtyperef{$replica} == "MD") {
238			$digest = Digest::MD5->new;
239			} else {
240			print LOG "cksum: unknown checksum type '" . $$csumtyperef{$replica} . "'\n";
241			return;
242			}
243
244			if (open(my $fh, "<", $filesystem . "/" . $replica)) {
245			print STDERR "Calculating sum of $filesystem/$replica...\n";
246			$digest->addfile($fh);
247			$cksum = $digest->hexdigest;
248			close $fh;
249			} else {
250			print LOG "cksum: could not open $replica\n";
251			return;
252			}
253			if ($cksum != $$csumvalueref{$replica}) {
254			print LOG "cksum: mismatch for $replica (current $cksum, database has " .
255			$$csumvalueref{$replica} . ", type " . $$csumtyperef{$replica} . ")\n";
256			} else {
257			print LOG "cksum: ok for $replica ($cksum, type " . $$csumtyperef{$replica} . ")\n";
258			}
259			}
260
261			sub check_resolve {
262			my $replica = shift;
263			# try a full path resolution
264			my $surl = $dpm->replica2surl("$server:/$replica");
265
266			if (defined $surl) {
267			print LOG "resolve: $replica -> $surl\n";
268			} else {
269			print LOG "resolve: failed for $replica\n";
270			}
271			return;
272			}
273
274	ronalds	1810	sub show_result {
275			print LOG "# replicas in DB: $count{db}\n";
276			print LOG "# replicas on disk: $count{disk}\n";
277			print LOG "# replicas in DB and on disk: $count{db_and_disk}\n";
278			print LOG "# replicas only in the DB: $count{only_db}\n";
279			print LOG "# replicas only on disk: $count{only_disk}\n";
280
281			print LOG "\n# Replicas only in the database\n";
282			foreach my $repl (sort @only_db) {
283			print LOG "$server:$filesystem/$repl\n";
284			}
285
286			print LOG "\n# Replicas only on the fileystem\n";
287			foreach my $repl (sort @only_disk) {
288			print LOG "$server:$filesystem/$repl\n";
289			}
290
291			print LOG "\n# times: " . join("\t", times);
292			}
293
grid.support@nikhef.nl	ViewVC Help
Powered by ViewVC 1.1.28