/[pdpsoft]/trunk/nl.nikhef.ndpf.tools/ndpf-dpm-tools/bin/dpm-disk-consistency
ViewVC logotype

Contents of /trunk/nl.nikhef.ndpf.tools/ndpf-dpm-tools/bin/dpm-disk-consistency

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2478 - (show annotations) (download)
Tue Jan 10 10:37:00 2012 UTC (10 years, 4 months ago) by ronalds
File size: 8279 byte(s)
fixed bug in Ganglia publisher for pool names with dots
1 #!/usr/bin/perl
2
3 #
4 # Tool to check the consistency between the DPM database
5 # and the files actually on disk.
6 #
7
8 use strict;
9
10 use Getopt::Long;
11 use Sys::Hostname;
12 use DBI;
13
14 my $help = 0;
15 my $verbose = 0;
16
17 # if $dbfile is not empty, the DB data will be read from it
18 # instead of obtained by executing gridpp_dpm_disk
19 # my $dbfile = '/home/ronalds/tmp/DPM/dpm_db_dump.txt';
20 my $dbfile;
21
22 my $cfgfile = '/root/DPMINFO';
23
24 my $logfile;
25 ##$logfile = "-"; # STDOUT for debugging
26 my $filesystem;
27
28 GetOptions(
29 'help|h' => \$help,
30 'verbose|v+' => \$verbose,
31 'logfile|l:s' => \$logfile,
32 'dbfile:s' => \$dbfile,
33 'cfgfile:s' => \$cfgfile,
34 );
35
36 $help and &usage;
37
38 my @now = localtime(time);
39 my $date = sprintf "%04d%02d%02d-%02d%02d%02d",
40 $now[5]+1900, $now[4]+1, $now[3], $now[2], $now[1], $now[0];
41 $logfile ||= "dpm-consistency-check-$date";
42
43 open LOG, "> $logfile" or die "$logfile: $!\n";
44 ($verbose > 1) and print LOG "Log file: $logfile\n";
45
46 my %replicas;
47 my %count = ( db => 0, disk => 0, only_db => 0, only_disk => 0, db_and_disk => 0, unknown => 0 );
48 my $REPLICA_IN_DB = 1;
49 my $REPLICA_ON_DISK = 2;
50 my @only_db;
51 my @only_disk;
52
53 my ($db_user, $db_pw, $db_host, $dpns_db_name, $db_port, $dbh);
54
55 my $filesystem = shift;
56 $filesystem =~ s!/*$!!;
57 if ( ! $filesystem ) {
58 warn "Missing mandatory argument 'filesystem'\n";
59 &usage(1);
60 }
61 if ( ! -d $filesystem ) {
62 warn "$!: $filesystem\n";
63 }
64
65 my $server = hostname;
66
67 ($verbose > 0) and print LOG "server: $server filesystem: $filesystem\n";
68
69
70 # Initial design, unoptimized:
71 # 1. Determine all replicas registered in the DB that are stored
72 # on the current server for the given file system.
73 # Store the replicas in hash %replicas with key replica
74 # and value $REPLICA_IN_DB
75 # 2. Determine all files that are present
76 # on the current server for the given file system.
77 # Store the files in hash %replicas with key file
78 # and value $REPLICA_IN_DB
79 # 3. Compare the hashes, determining inconsistencies and
80 # counting statistics
81 #
82 # Later:
83 # - Add an option to automatically remove entries from the
84 # database for which the corresponding file is missing,
85 # or remove the file from disk if there is no corresponding
86 # entry in the database
87 # - Memory consumption might be an issue for this script;
88 # consider splitting the processing by date
89
90
91 # initialize database connection
92 &parse_db_config($cfgfile);
93 &sql_init;
94
95 # Determine replicas registered in the database
96 &get_db_replicas(\%replicas);
97
98 # Determine replicas on disk
99 &get_fs_replicas(\%replicas);
100
101 # Comparison
102 &compare_database_filesystem(\%replicas);
103
104 # Results
105 &show_result;
106
107 &sql_terminate;
108
109 close LOG;
110 exit 0;
111
112
113
114 sub usage {
115 my $ret = shift;
116 print STDERR <<EOH;
117 usage: $0 [options] <filesystem>
118 options:
119 help, h Show this help text
120 verbose, v Increase output verbosity
121 EOH
122
123 exit($ret);
124 }
125
126
127 sub get_db_replicas {
128 my $ref = shift;
129
130 # read the replicas registered in the database,
131 # unless a file for debugging is defined
132 if ( ! $dbfile ) {
133 # $dbfile = '/tmp/dpm-db-${date}.dump';
134 # execute the gridpp_* command
135 # my $cmd = "gridpp_dpm_disk --server $server --fs $filesystem > $dbfile";
136 # ($verbose > 0) and print LOG "Executing $cmd\n";
137
138 my $sql = "SELECT sfn FROM Cns_file_replica WHERE "
139 . "status <> 'D' AND "
140 . "host=" . &sql_quote($server) . " AND fs=" . &sql_quote($filesystem);
141 my @results = &sql_query($sql);
142 foreach my $row (@results) {
143 my $replica = $$row{sfn};
144 $replica =~ s!^$server:$filesystem[/]*!!;
145 ($verbose > 2) and print LOG "$replica\n";
146 $$ref{$replica} |= $REPLICA_IN_DB;
147 $count{db}++;
148 }
149 }
150
151 ($verbose > 0) and print LOG "Found $count{db} replicas in the database under $filesystem\n";
152 }
153
154 sub get_fs_replicas {
155 my $ref = shift;
156
157 # find all files under the specified filesystem
158 my $cmd = "find $filesystem -type f -print";
159
160 open FS, "$cmd | " or die "$cmd: $!\n";
161 while ( my $file = <FS> ) {
162 $file =~ s!^$filesystem[/]*!!;
163 chomp $file;
164 ($verbose > 2) and print LOG "$file\n";
165 $$ref{$file} |= $REPLICA_ON_DISK;
166 $count{disk}++;
167 }
168 close FS;
169
170 ($verbose > 0) and print LOG "Found $count{disk} files under $filesystem\n";
171 }
172
173
174 sub compare_database_filesystem {
175 my ($replica_ref) = shift;
176
177 my $db_and_disk = ($REPLICA_IN_DB | $REPLICA_ON_DISK);
178 while ( my ($repl, $state) = each(%$replica_ref) ) {
179 ($verbose > 1) and print "$repl $state\t";
180 if ( $state == $REPLICA_IN_DB ) {
181 $count{only_db}++;
182 push(@only_db, $repl);
183 ($verbose > 1) and print LOG "[only in DB]\n";
184 }
185 elsif ( $state == $REPLICA_ON_DISK ) {
186 $count{only_disk}++;
187 push(@only_disk, $repl);
188 ($verbose > 1) and print LOG "[only on disk]\n";
189 }
190 elsif ( $state == $db_and_disk ) {
191 $count{db_and_disk}++;
192 ($verbose > 1) and print LOG "[OK]\n";
193 }
194 else {
195 $count{unknown}++;
196 ($verbose > 1) and print LOG "[Unexpected state]\n";
197 }
198 }
199 }
200
201
202
203 sub show_result {
204 print LOG "# replicas in DB: $count{db}\n";
205 print LOG "# replicas on disk: $count{disk}\n";
206 print LOG "# replicas in DB and on disk: $count{db_and_disk}\n";
207 print LOG "# replicas only in the DB: $count{only_db}\n";
208 print LOG "# replicas only on disk: $count{only_disk}\n";
209
210 print LOG "\n# Replicas only in the database\n";
211 foreach my $repl (sort @only_db) {
212 print LOG "$server:$filesystem/$repl\n";
213 }
214
215 print LOG "\n# Replicas only on the fileystem\n";
216 foreach my $repl (sort @only_disk) {
217 print LOG "$server:$filesystem/$repl\n";
218 }
219
220 print LOG "\n# times: " . join("\t", times);
221 }
222
223
224
225
226 #
227 # DB related
228 #
229 sub parse_db_config
230 {
231 my $conf_file = $_[0];
232 open(DB_CONF, $conf_file) ||
233 die "Database configuration file $conf_file cannot be read: $!\n";
234
235 my $conf = <DB_CONF>;
236 chomp $conf;
237 # Optional match for db name
238 if ($conf =~ s/\/(\w+)$//) {
239 $dpns_db_name = $1;
240 } else {
241 $dpns_db_name = "cns_db";
242 }
243 # Optional port match
244 if ($conf =~ s/:(\d+)//) {
245 $db_port = $1;
246 } else {
247 $db_port = getservbyname("mysql", "tcp");
248 }
249 if ($conf =~ /^(\w+)\/([^@]+)@([A-Za-z0-9\.-]+)$/) {
250 $db_user = $1;
251 $db_pw = $2;
252 $db_host = $3;
253 } else {
254 die "Failed to interpret database configuration file. Format should\n",
255 "be a single line:\n",
256 "USER/PASSWORD\@HOST[:PORT][/DPNS_DB_NAME]\n",
257 "Port is optional (defaults to mysql port in services).\n",
258 "Database name is optional (defaults to cns_db).\n";
259 }
260 print STDERR "DB User: $db_user\nDB Host: $db_host\n",
261 "DPNS DB: $dpns_db_name\n\n" if ($verbose > 0);
262 close DB_CONF;
263 }
264
265
266 sub sql_init() {
267 my $dsn = "DBI:mysql:database=$dpns_db_name;host=$db_host;port=$db_port";
268 $dbh = DBI->connect($dsn, $db_user, $db_pw);
269 if (!$dbh) {
270 die "Failed to connect to MySQL server. Check username/password.\n";
271 }
272 return(0);
273 }
274
275 sub sql_terminate() {
276 $dbh->disconnect;
277 }
278
279 sub sql_quote() {
280 return ($dbh->quote($_[0]));
281 }
282
283 sub sql_query() {
284 my $query = shift;
285
286 print STDERR "QUERY: $query\n" if ($verbose > 0);
287
288 my $sth = $dbh->prepare("$query");
289 if (!$sth) {
290 die "Failed to execute MySQL query: " . $sth->errstr . "\n";
291 }
292 if (!$sth->execute) {
293 die "Failed to execute MySQL query: " . $sth->errstr . "\n";
294 };
295 print STDERR "SQL Debug: query returned ", $sth->{'NUM_OF_FIELDS'}, " fields.\n" if ($verbose > 0);
296
297 my $names = $sth->{'NAME'};
298 my $numFields = $sth->{'NUM_OF_FIELDS'};
299
300 print STDERR "No matches!\n" if (($verbose > 0) and $sth->rows == 0);
301
302 my @aref = ();
303 while (my $ref = $sth->fetchrow_arrayref) {
304 my %result;
305 for (my $i = 0; $i < $numFields; $i++) {
306 $result{$$names[$i]} = $$ref[$i];
307 }
308 push @aref, \%result;
309 }
310
311 return(@aref);
312 }
313

Properties

Name Value
svn:executable *
svn:keywords id

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28