/[pdpsoft]/trunk/nl.nikhef.ndpf.tools/ndpf-dpm-tools/bin/dpm-disk-consistency
ViewVC logotype

Annotation of /trunk/nl.nikhef.ndpf.tools/ndpf-dpm-tools/bin/dpm-disk-consistency

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2478 - (hide annotations) (download)
Tue Jan 10 10:37:00 2012 UTC (10 years, 4 months ago) by ronalds
File size: 8279 byte(s)
fixed bug in Ganglia publisher for pool names with dots
1 ronalds 1810 #!/usr/bin/perl
2    
3     #
4     # Tool to check the consistency between the DPM database
5     # and the files actually on disk.
6     #
7    
8     use strict;
9    
10     use Getopt::Long;
11     use Sys::Hostname;
12     use DBI;
13    
14     my $help = 0;
15     my $verbose = 0;
16    
17     # if $dbfile is not empty, the DB data will be read from it
18     # instead of obtained by executing gridpp_dpm_disk
19     # my $dbfile = '/home/ronalds/tmp/DPM/dpm_db_dump.txt';
20     my $dbfile;
21    
22     my $cfgfile = '/root/DPMINFO';
23    
24     my $logfile;
25     ##$logfile = "-"; # STDOUT for debugging
26     my $filesystem;
27    
28     GetOptions(
29     'help|h' => \$help,
30     'verbose|v+' => \$verbose,
31     'logfile|l:s' => \$logfile,
32     'dbfile:s' => \$dbfile,
33     'cfgfile:s' => \$cfgfile,
34     );
35    
36     $help and &usage;
37    
38     my @now = localtime(time);
39     my $date = sprintf "%04d%02d%02d-%02d%02d%02d",
40     $now[5]+1900, $now[4]+1, $now[3], $now[2], $now[1], $now[0];
41     $logfile ||= "dpm-consistency-check-$date";
42    
43     open LOG, "> $logfile" or die "$logfile: $!\n";
44     ($verbose > 1) and print LOG "Log file: $logfile\n";
45    
46     my %replicas;
47     my %count = ( db => 0, disk => 0, only_db => 0, only_disk => 0, db_and_disk => 0, unknown => 0 );
48     my $REPLICA_IN_DB = 1;
49     my $REPLICA_ON_DISK = 2;
50     my @only_db;
51     my @only_disk;
52    
53     my ($db_user, $db_pw, $db_host, $dpns_db_name, $db_port, $dbh);
54    
55     my $filesystem = shift;
56     $filesystem =~ s!/*$!!;
57     if ( ! $filesystem ) {
58     warn "Missing mandatory argument 'filesystem'\n";
59     &usage(1);
60     }
61     if ( ! -d $filesystem ) {
62     warn "$!: $filesystem\n";
63     }
64    
65     my $server = hostname;
66    
67     ($verbose > 0) and print LOG "server: $server filesystem: $filesystem\n";
68    
69    
70     # Initial design, unoptimized:
71     # 1. Determine all replicas registered in the DB that are stored
72     # on the current server for the given file system.
73     # Store the replicas in hash %replicas with key replica
74     # and value $REPLICA_IN_DB
75     # 2. Determine all files that are present
76     # on the current server for the given file system.
77     # Store the files in hash %replicas with key file
78     # and value $REPLICA_IN_DB
79     # 3. Compare the hashes, determining inconsistencies and
80     # counting statistics
81     #
82     # Later:
83     # - Add an option to automatically remove entries from the
84     # database for which the corresponding file is missing,
85     # or remove the file from disk if there is no corresponding
86     # entry in the database
87     # - Memory consumption might be an issue for this script;
88     # consider splitting the processing by date
89    
90    
91     # initialize database connection
92     &parse_db_config($cfgfile);
93     &sql_init;
94    
95     # Determine replicas registered in the database
96     &get_db_replicas(\%replicas);
97    
98     # Determine replicas on disk
99     &get_fs_replicas(\%replicas);
100    
101     # Comparison
102     &compare_database_filesystem(\%replicas);
103    
104     # Results
105     &show_result;
106    
107     &sql_terminate;
108    
109     close LOG;
110     exit 0;
111    
112    
113    
114     sub usage {
115     my $ret = shift;
116     print STDERR <<EOH;
117     usage: $0 [options] <filesystem>
118     options:
119     help, h Show this help text
120     verbose, v Increase output verbosity
121     EOH
122    
123     exit($ret);
124     }
125    
126    
127     sub get_db_replicas {
128     my $ref = shift;
129    
130     # read the replicas registered in the database,
131     # unless a file for debugging is defined
132     if ( ! $dbfile ) {
133     # $dbfile = '/tmp/dpm-db-${date}.dump';
134     # execute the gridpp_* command
135     # my $cmd = "gridpp_dpm_disk --server $server --fs $filesystem > $dbfile";
136     # ($verbose > 0) and print LOG "Executing $cmd\n";
137    
138     my $sql = "SELECT sfn FROM Cns_file_replica WHERE "
139 ronalds 2478 . "status <> 'D' AND "
140 ronalds 1810 . "host=" . &sql_quote($server) . " AND fs=" . &sql_quote($filesystem);
141     my @results = &sql_query($sql);
142     foreach my $row (@results) {
143     my $replica = $$row{sfn};
144     $replica =~ s!^$server:$filesystem[/]*!!;
145     ($verbose > 2) and print LOG "$replica\n";
146     $$ref{$replica} |= $REPLICA_IN_DB;
147     $count{db}++;
148     }
149     }
150    
151     ($verbose > 0) and print LOG "Found $count{db} replicas in the database under $filesystem\n";
152     }
153    
154     sub get_fs_replicas {
155     my $ref = shift;
156    
157     # find all files under the specified filesystem
158     my $cmd = "find $filesystem -type f -print";
159    
160     open FS, "$cmd | " or die "$cmd: $!\n";
161     while ( my $file = <FS> ) {
162     $file =~ s!^$filesystem[/]*!!;
163     chomp $file;
164     ($verbose > 2) and print LOG "$file\n";
165     $$ref{$file} |= $REPLICA_ON_DISK;
166     $count{disk}++;
167     }
168     close FS;
169    
170     ($verbose > 0) and print LOG "Found $count{disk} files under $filesystem\n";
171     }
172    
173    
174     sub compare_database_filesystem {
175     my ($replica_ref) = shift;
176    
177     my $db_and_disk = ($REPLICA_IN_DB | $REPLICA_ON_DISK);
178     while ( my ($repl, $state) = each(%$replica_ref) ) {
179     ($verbose > 1) and print "$repl $state\t";
180     if ( $state == $REPLICA_IN_DB ) {
181     $count{only_db}++;
182     push(@only_db, $repl);
183     ($verbose > 1) and print LOG "[only in DB]\n";
184     }
185     elsif ( $state == $REPLICA_ON_DISK ) {
186     $count{only_disk}++;
187     push(@only_disk, $repl);
188     ($verbose > 1) and print LOG "[only on disk]\n";
189     }
190     elsif ( $state == $db_and_disk ) {
191     $count{db_and_disk}++;
192     ($verbose > 1) and print LOG "[OK]\n";
193     }
194     else {
195     $count{unknown}++;
196     ($verbose > 1) and print LOG "[Unexpected state]\n";
197     }
198     }
199     }
200    
201    
202    
203     sub show_result {
204     print LOG "# replicas in DB: $count{db}\n";
205     print LOG "# replicas on disk: $count{disk}\n";
206     print LOG "# replicas in DB and on disk: $count{db_and_disk}\n";
207     print LOG "# replicas only in the DB: $count{only_db}\n";
208     print LOG "# replicas only on disk: $count{only_disk}\n";
209    
210     print LOG "\n# Replicas only in the database\n";
211     foreach my $repl (sort @only_db) {
212     print LOG "$server:$filesystem/$repl\n";
213     }
214    
215     print LOG "\n# Replicas only on the fileystem\n";
216     foreach my $repl (sort @only_disk) {
217     print LOG "$server:$filesystem/$repl\n";
218     }
219    
220     print LOG "\n# times: " . join("\t", times);
221     }
222    
223    
224    
225    
226     #
227     # DB related
228     #
229     sub parse_db_config
230     {
231     my $conf_file = $_[0];
232     open(DB_CONF, $conf_file) ||
233     die "Database configuration file $conf_file cannot be read: $!\n";
234    
235     my $conf = <DB_CONF>;
236     chomp $conf;
237     # Optional match for db name
238     if ($conf =~ s/\/(\w+)$//) {
239     $dpns_db_name = $1;
240     } else {
241     $dpns_db_name = "cns_db";
242     }
243     # Optional port match
244     if ($conf =~ s/:(\d+)//) {
245     $db_port = $1;
246     } else {
247     $db_port = getservbyname("mysql", "tcp");
248     }
249     if ($conf =~ /^(\w+)\/([^@]+)@([A-Za-z0-9\.-]+)$/) {
250     $db_user = $1;
251     $db_pw = $2;
252     $db_host = $3;
253     } else {
254     die "Failed to interpret database configuration file. Format should\n",
255     "be a single line:\n",
256     "USER/PASSWORD\@HOST[:PORT][/DPNS_DB_NAME]\n",
257     "Port is optional (defaults to mysql port in services).\n",
258     "Database name is optional (defaults to cns_db).\n";
259     }
260     print STDERR "DB User: $db_user\nDB Host: $db_host\n",
261     "DPNS DB: $dpns_db_name\n\n" if ($verbose > 0);
262     close DB_CONF;
263     }
264    
265    
266     sub sql_init() {
267     my $dsn = "DBI:mysql:database=$dpns_db_name;host=$db_host;port=$db_port";
268     $dbh = DBI->connect($dsn, $db_user, $db_pw);
269     if (!$dbh) {
270     die "Failed to connect to MySQL server. Check username/password.\n";
271     }
272     return(0);
273     }
274    
275     sub sql_terminate() {
276     $dbh->disconnect;
277     }
278    
279     sub sql_quote() {
280     return ($dbh->quote($_[0]));
281     }
282    
283     sub sql_query() {
284     my $query = shift;
285    
286     print STDERR "QUERY: $query\n" if ($verbose > 0);
287    
288     my $sth = $dbh->prepare("$query");
289     if (!$sth) {
290     die "Failed to execute MySQL query: " . $sth->errstr . "\n";
291     }
292     if (!$sth->execute) {
293     die "Failed to execute MySQL query: " . $sth->errstr . "\n";
294     };
295     print STDERR "SQL Debug: query returned ", $sth->{'NUM_OF_FIELDS'}, " fields.\n" if ($verbose > 0);
296    
297     my $names = $sth->{'NAME'};
298     my $numFields = $sth->{'NUM_OF_FIELDS'};
299    
300     print STDERR "No matches!\n" if (($verbose > 0) and $sth->rows == 0);
301    
302     my @aref = ();
303     while (my $ref = $sth->fetchrow_arrayref) {
304     my %result;
305     for (my $i = 0; $i < $numFields; $i++) {
306     $result{$$names[$i]} = $$ref[$i];
307     }
308     push @aref, \%result;
309     }
310    
311     return(@aref);
312     }
313    

Properties

Name Value
svn:executable *
svn:keywords id

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28