/[pdpsoft]/nl.nikhef.ndpf.groupviews/branches/RB-2.0.0/ndpf-gv-mkplots
ViewVC logotype

Contents of /nl.nikhef.ndpf.groupviews/branches/RB-2.0.0/ndpf-gv-mkplots

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2392 - (show annotations) (download)
Tue Aug 2 09:39:36 2011 UTC (11 years, 2 months ago) by templon
File size: 18119 byte(s)
change time range of hour plots from 3,5 hr to 4; ensures
that there are always two major ticks on the axis of the small
version.  with only one major tick, the time scale was unclear.

there is no real change to mkdbs.sh, just editing to get right
DB defined.

1 #! /usr/bin/env python
2 # $Id$
3 # Source: $URL$
4 # J. A. Templon, NIKHEF/PDP 2011
5
6 import os
7
8 NUMGROUPS=8
9 DATADIR=os.environ['HOME'] + '/ndpfdata/'
10 PLOTDIR=os.environ['HOME'] + '/public_html/'
11 # 8 class qualitative paired color scheme
12
13 colors = [ "#A6CEE3", "#1F77B4", "#B2DF8A", "#33A02C",
14 "#FB9A99", "#E31A1C", "#FDBF6F", "#FF7F00" ]
15
16 colors.reverse()
17
18 timeargs = { 'hr' : [ '-s', 'n-4h', '-e', 'n' ],
19 'day' : [ '-s', 'n-29h', '-e', 'n' ],
20 'week' : [ '-s', 'n-8d', '-e', 'n' ],
21 'month' : [ '-s', 'n-35d', '-e', 'n' ]
22 }
23
24 # for reference : resolutions of the RRAs
25 # RRA:AVERAGE:0.5:1:2880 \
26 # RRA:AVERAGE:0.5:5:4032 \
27 # RRA:AVERAGE:0.5:60:1440 \
28 # RRA:AVERAGE:0.5:1440:3650
29 # 60s (1 min) x 2880 points = 48 hr
30 # 300s (5 min) x 4032 points = 14 days
31 # 3600s (1 hr) x 1440 points = 60 days
32 # 86400s (1 day) x 3650 points = 10 years
33
34 import rrdtool
35
36 # first need to find four "top eight" lists
37 # one for the past day, which will be used for the hour / day page
38 # one for the past week
39 # one for the past month
40 # one for the past year
41 # base it on running jobs
42
43 import time
44 now=int(time.mktime(time.localtime()))
45 end = (now / 60) * 60
46 start = end - (3600 * 24) + 60
47
48 import glob # number of running jobs
49
50 ### block finding 'top N' group list ###
51
52 day_groups = dict() # structure day_group[groupname] = total of hourly average
53
54 running_files = glob.glob(DATADIR+'*.running.rrd')
55 for db in running_files:
56 group = db[len(DATADIR):db.find('.running.rrd')]
57 if group == "total":
58 continue
59 tup = rrdtool.fetch(db,'AVERAGE','-r',"60",
60 '-s', repr(start), '-e', repr(end))
61 vallist = [0] # start with zero, in case no vals returned, get zero as answer
62 for tup2 in tup[2]:
63 val = tup2[0]
64 if val:
65 vallist.append(val)
66 day_groups[group] = sum(vallist)
67
68 day_groups_sorted = sorted(day_groups, key=day_groups.get, reverse=False)
69 topgroups_day=day_groups_sorted[-NUMGROUPS:]
70
71 ### end block 'top N' group list ###
72
73 ### block generating plots ###
74
75 commonargs = ['--imgformat', 'PNG',
76 '--legend-position=east', '--legend-direction=bottomup']
77
78 sizeargs = { 'small' : [ '--width', '195', '--height', '125' ],
79 'large' : [ '--width', '850', '--height', '500' ]
80 }
81 def doplot(grouplist, dbtype, psize, timetag):
82
83 defs = list()
84 plots = list()
85
86 data_defs = list()
87 plot_defs = list()
88
89 for group in (grouplist + ['total']):
90 data_defs.append('DEF:'+group+'='+DATADIR+group+'.'+dbtype+'.rrd:'+dbtype+':AVERAGE')
91
92 for idx in range(len(grouplist)):
93 group = grouplist[idx]
94 pdefstr = 'AREA' ':' + group + colors[idx] + ':' + group
95 pdefstr = pdefstr + '\\n'
96 if idx > 0:
97 pdefstr = pdefstr + ':STACK'
98 plot_defs.append(pdefstr)
99
100 plot_defs.append('LINE:total#000000:total')
101
102 pargs = [ PLOTDIR + dbtype + '-' + timetag + '-' + psize + '.png'] + \
103 commonargs + sizeargs[psize] + timeargs[timetag] + \
104 data_defs + plot_defs
105 rrdtool.graph( *pargs )
106
107 def doplot_wait(grouplist, dbtype, psize, timetag):
108
109 defs = list()
110 plots = list()
111
112 data_defs = list()
113 plot_defs = list()
114
115 for group in (grouplist + ['rollover']):
116 data_defs.append('DEF:'+group+'='+DATADIR+group+'.'+dbtype+'.rrd:'+dbtype+':AVERAGE')
117
118 for idx in range(len(grouplist)):
119 group = grouplist[idx]
120 pdefstr = 'LINE3' ':' + group + colors[idx] + ':' + group
121 pdefstr = pdefstr + '\\n'
122 plot_defs.append(pdefstr)
123
124 plot_defs.append('LINE2:rollover#000000:rollover')
125
126 pargs = [ PLOTDIR + dbtype + '-' + timetag + '-' + psize + '.png'] + \
127 ['--slope-mode', '-o'] + \
128 commonargs + sizeargs[psize] + timeargs[timetag] + \
129 data_defs + plot_defs
130 rrdtool.graph( *pargs )
131
132 for dbtype in ['queued', 'running', 'waittime']:
133 for psize in ['small', 'large']:
134 for timetag in ['hr', 'day']:
135 if dbtype == 'waittime':
136 doplot_wait(topgroups_day, dbtype, psize, timetag)
137 else:
138 doplot(topgroups_day, dbtype, psize, timetag)
139
140 import time
141 now=int(time.mktime(time.localtime()))
142 end = (now / 3600 ) * 3600
143 start = end - (86400*30) + 3600
144
145 ### block finding 'top N' group list ###
146
147 month_groups = dict() # structure day_group[groupname] = total of hourly average
148
149 for db in running_files:
150 group = db[len(DATADIR):db.find('.running.rrd')]
151 if group == "total":
152 continue
153 tup = rrdtool.fetch(db,'AVERAGE','-r',"3600",
154 '-s', repr(start), '-e', repr(end))
155 vallist = [0] # start with zero, in case no vals returned, get zero as answer
156 for tup2 in tup[2]:
157 val = tup2[0]
158 if val:
159 vallist.append(val)
160 month_groups[group] = sum(vallist)
161
162 month_groups_sorted = sorted(month_groups, key=month_groups.get, reverse=False)
163 topgroups_month=month_groups_sorted[-NUMGROUPS:]
164
165 ### end block 'top N' group list ###
166
167 for dbtype in ['queued', 'running', 'waittime']:
168 for psize in ['small', 'large']:
169 for timetag in ['week', 'month']:
170 if dbtype == 'waittime':
171 doplot_wait(topgroups_month, dbtype, psize, timetag)
172 else:
173 doplot(topgroups_month, dbtype, psize, timetag)
174 import sys
175 sys.exit(0)
176
177 pfnam = os.environ['HOME']+'/tmp/run.plt'
178
179 infh = open(os.environ['HOME']+'/tmp/groups.rdata','r')
180 datablocks = infh.read().split('\n\n')
181
182 countslist = list() # holds one rundict per timestamp
183 timestringlist = list() # same order as countslist
184
185 for block in datablocks[:-1]:
186 lines = block.split('\n')
187 timestr = lines[0] ; timestringlist.append(timestr)
188 rundict = dict() # format rundict['vo'] = number_running
189 for l in lines[1:]:
190 vo, num = eval(l, {}, {})
191 rundict[vo] = num
192 countslist.append(rundict)
193
194 def avgdata(vals, tstamps, step):
195 # step must be odd
196 # lengths of vals and tstamps must be identical, and
197 # also must be integer multiple of step
198
199 newvals = list()
200 newstamps = list()
201 if len(vals) % step != 0:
202 print "length of input arrays must be multiple of step size"
203 sys.exit(0)
204 optr = 0
205 while optr < len(vals):
206 print optr
207 print step
208 print vals[optr:optr+step]
209
210 newvals.append(sum(vals[optr:optr+step]))
211 newstamps.append(tstamps[optr + (fact/2)])
212 optr += fact
213 return newvals, newstamps
214
215 sl1 = countslist[-75:]
216 sl2 = timestringlist[-75:]
217
218 hrtuple_small = (
219 sl1, sl2,
220 [
221 # 'set terminal x11 size 1000,535\n',
222 'set terminal png size 365,201 font "arial,8"\n',
223 'set key autotitle columnheader\n',
224 'set xdata time\n',
225 'set timefmt "%b-%d-%Y-%H:%M:%S"\n',
226 'set output "/user/templon/public_html/run-hr-small.png"\n',
227 '# time range must be in same format as data file\n',
228 'set grid xtics mxtics ytics mytics front\n',
229 'set yrange [0 : *]\n',
230 'set auto fix\n'
231 'set key outside samplen 1\n',
232 'set format x "%H:%M"\n',
233 'set xtics 3600\n',
234 'set mxtics 6\n',
235 'set mytics 2\n'
236 ],
237 os.environ['HOME']+'/tmp/myd_hr_s'
238 )
239
240 hrtuple_large = (
241 sl1, sl2,
242 [
243 'set terminal png size 1000,535 font "arial,10"\n',
244 'set output "/user/templon/public_html/run-hr-large.png"\n',
245 'set format x "%H:%M"\n',
246 'set xtics 3600\n',
247 'set mxtics 6\n',
248 'set mytics 4\n'
249 'set key autotitle columnheader\n',
250 'set xdata time\n',
251 'set timefmt "%b-%d-%Y-%H:%M:%S"\n',
252 '# time range must be in same format as data file\n',
253 'set grid xtics mxtics ytics mytics front\n',
254 'set yrange [0 : *]\n',
255 'set auto fix\n'
256 'set key outside samplen 1\n',
257 ],
258 os.environ['HOME']+'/tmp/myd_hr_l'
259 )
260
261 sl3 = countslist[-720:]
262 sl4 = timestringlist[-720:]
263
264 daytuple_small = (
265 sl3, sl4,
266 [
267 'set terminal png size 365,201 font "arial,8"\n',
268 'set output "/user/templon/public_html/run-day-small.png"\n',
269 'set format x "%H:%M"\n',
270 'set xtics 21600\n',
271 'set mxtics 3\n'
272 ],
273 os.environ['HOME']+'/tmp/myd_day_s'
274 )
275
276 daytuple_large = (
277 sl3, sl4,
278 [
279 'set terminal png size 1000,535 font "arial,10"\n',
280 'set output "/user/templon/public_html/run-day-large.png"\n',
281 'set format x "%H:%M"\n',
282 'set xtics 7200\n',
283 'set mxtics 4\n'
284 ],
285 os.environ['HOME']+'/tmp/myd_day_l'
286 )
287
288 sl5 = countslist[-5760:]
289 sl6 = timestringlist[-5760:]
290
291 weektuple_small = (
292 sl5[::5], sl6[::5],
293 [
294 'set terminal png size 365,201 font "arial,8"\n',
295 'set output "/user/templon/public_html/run-week-small.png"\n',
296 'set format x "%a"\n',
297 'set xtics 86400\n',
298 'set nomxtics \n'
299 ],
300 os.environ['HOME']+'/tmp/myd_week_s'
301 )
302
303 # sl7, sl8 = avgdata(sl5, sl6, 5)
304
305 weektuple_large = (
306 sl5[::2], sl6[::2],
307 [
308 'set terminal png size 1000,535 font "arial,10"\n',
309 'set output "/user/templon/public_html/run-week-large.png"\n',
310 'set format x "%a"\n',
311 'set xtics 86400\n',
312 'set mxtics 3\n'
313 ],
314 os.environ['HOME']+'/tmp/myd_week_l'
315 )
316
317 sl10 = countslist[-22300:]
318 sl11 = timestringlist[-22300:]
319
320 # sl12, sl13 = avgdata(sl10, sl11, 100)
321
322 monthtuple_small = (
323 sl10[::100], sl11[::100],
324 [
325 'set terminal png size 365,201 font "arial,8"\n',
326 'set output "/user/templon/public_html/run-month-small.png"\n',
327 'set format x "%m/%d"\n',
328 'set xtics 604800\n',
329 'set nomxtics \n'
330 ],
331 os.environ['HOME']+'/tmp/myd_month_s'
332 )
333
334 # sl14, sl15 = avgdata(sl10, sl11, 20)
335
336 monthtuple_large = (
337 sl10[::20], sl11[::20],
338 [
339 'set terminal png size 1000,535 font "arial,10"\n',
340 'set output "/user/templon/public_html/run-month-large.png"\n',
341 'set format x "%m/%d"\n',
342 'set xtics 604800\n',
343 'set mxtics 7\n'
344 ],
345 os.environ['HOME']+'/tmp/myd_month_l'
346 )
347
348
349 # do it again but for queued jobs
350
351 infh = open(os.environ['HOME']+'/tmp/groups.qdata','r')
352 datablocks = infh.read().split('\n\n')
353
354 countslist = list() # holds one rundict per timestamp
355 timestringlist = list() # same order as countslist
356
357 for block in datablocks[:-1]:
358 lines = block.split('\n')
359 timestr = lines[0] ; timestringlist.append(timestr)
360 rundict = dict() # format rundict['vo'] = number_running
361 for l in lines[1:]:
362 vo, num = eval(l, {}, {})
363 rundict[vo] = num
364 countslist.append(rundict)
365
366 qhrtuple_small = (
367 countslist[-75:],
368 timestringlist[-75:],
369 [
370 # 'set terminal x11 size 1000,535\n',
371 'set terminal png size 365,201 font "arial,8"\n',
372 'set key autotitle columnheader\n',
373 'set xdata time\n',
374 'set timefmt "%b-%d-%Y-%H:%M:%S"\n',
375 'set output "/user/templon/public_html/queued-hr-small.png"\n',
376 '# time range must be in same format as data file\n',
377 'set grid xtics mxtics ytics mytics front\n',
378 'set yrange [0 : *]\n',
379 'set auto fix\n'
380 'set key outside samplen 1\n',
381 'set format x "%H:%M"\n',
382 'set xtics 3600\n',
383 'set mxtics 6\n',
384 'set mytics 3\n'
385 ],
386 os.environ['HOME']+'/tmp/myqd_hr'
387 )
388
389 qhrtuple_large = (
390 countslist[-75:],
391 timestringlist[-75:],
392 [
393 'set terminal png size 1000,535 font "arial,10"\n',
394 'set output "/user/templon/public_html/queued-hr-large.png"\n',
395 'set format x "%H:%M"\n',
396 'set xtics 3600\n',
397 'set mxtics 6\n',
398 'set mytics 3\n'
399 'set key autotitle columnheader\n',
400 'set xdata time\n',
401 'set timefmt "%b-%d-%Y-%H:%M:%S"\n',
402 '# time range must be in same format as data file\n',
403 'set grid xtics mxtics ytics mytics front\n',
404 # 'set yrange [0 : 2600]\n',
405 'set auto fix\n'
406 'set key outside samplen 1\n',
407 ],
408 os.environ['HOME']+'/tmp/myqd_hr'
409 )
410
411 qdaytuple_small = (
412 countslist[-720:],
413 timestringlist[-720:],
414 [
415 'set terminal png size 365,201 font "arial,8"\n',
416 'set output "/user/templon/public_html/queued-day-small.png"\n',
417 'set format x "%H:%M"\n',
418 'set xtics 21600\n',
419 'set mxtics 3\n'
420 ],
421 os.environ['HOME']+'/tmp/myqd_day'
422 )
423
424 qdaytuple_large = (
425 countslist[-720:],
426 timestringlist[-720:],
427 [
428 'set terminal png size 1000,535 font "arial,10"\n',
429 'set output "/user/templon/public_html/queued-day-large.png"\n',
430 'set format x "%H:%M"\n',
431 'set xtics 7200\n',
432 'set mxtics 4\n'
433 ],
434 os.environ['HOME']+'/tmp/myqd_day'
435 )
436
437
438 plottuple_list = [ hrtuple_small, hrtuple_large, daytuple_small,
439 daytuple_large, weektuple_small, weektuple_large,
440 monthtuple_small, monthtuple_large,
441 qhrtuple_small, qhrtuple_large,
442 qdaytuple_small, qdaytuple_large
443 ]
444
445 plotfileh = open(pfnam,'w')
446
447 def emit_plotfile(pfh, plotslist):
448 # pfh is handle to plot file object
449 # first make the right data structure: rundata['vo']['timestamp'] = number
450 rundata = dict() # format rundata['vo']['timestamp'] = number
451 tsl = plotslist[1]
452 ctl = plotslist[0]
453 for i in range(len(tsl)):
454 for vo in ctl[i].keys():
455 if vo not in rundata.keys():
456 rundata[vo] = dict()
457 rundata[vo][tsl[i]] = ctl[i][vo]
458
459 sums = dict()
460 for k in rundata.keys():
461 sums[k] = sum(rundata[k].values())
462
463 vos_by_jobs = sorted(sums, key=sums.get, reverse=False)
464
465 topvos = vos_by_jobs[-NUMGROUPS:]
466
467 dfh = open(plotslist[3], 'w')
468 dfh.write('timestamp ')
469 for k in topvos:
470 dfh.write("%8s " % k)
471 dfh.write('\n')
472 for t in tsl:
473 dfh.write(t+ " ")
474 for k in topvos:
475 if t in rundata[k].keys():
476 val = rundata[k][t]
477 else:
478 val = 0
479 dfh.write("%4d " % val)
480 dfh.write('\n')
481 dfh.close()
482
483 import glob
484 import os
485
486 gplines = plotslist[2]
487
488 # gplines.append('set terminal png size 335,201\n')
489
490 ctr = 1
491 for vo in reversed(topvos):
492 if ctr == 1:
493 pstr = 'plot '
494 datafnam = plotslist[3]
495 pstr += '"' + datafnam + '" using 1:($' + repr(topvos.index(vo)+2)
496 for k2 in range(topvos.index(vo)+1, 1, -1):
497 pstr += '+$' + repr(k2)
498 pstr += ') t "' + vo + '"'
499 if ctr < len(topvos):
500 pstr += ' w boxes fs solid,'
501 ctr += 1
502 else:
503 pstr += ' w boxes fs solid'
504 gplines.append(pstr + '\n')
505
506 for l in gplines:
507 pfh.write(l)
508
509 # gplines.append('set terminal png size 1000,535\n')
510 # gplines.append('set output "/user/templon/public_html/wait-hr-large.png"\n')
511 # gplines.append('set format x "%H:%M"\n')
512 # gplines.append('set format y "%.0te+%S"\n')
513 # gplines.append('set xtics 3600\n')
514
515 # ctr = 1
516 # for vo in volist:
517 # if ctr == 1:
518 # pstr = 'plot '
519 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
520 # pstr += '"< tail -75 ' + datafnam + '" using 1:3' + \
521 # ' title "' + vo + '"'
522 # if ctr < len(volist):
523 # pstr += ' with lines lw 3,'
524 # ctr += 1
525 # else:
526 # pstr += ' with lines lw 1'
527 # gplines.append(pstr + '\n')
528
529 # gplines.append('set terminal png size 335,201\n')
530 # gplines.append('set output "/user/templon/public_html/wait-day-small.png"\n')
531 # gplines.append('set format x "%H:%M"\n')
532 # gplines.append('set format y "%.0te+%S"\n')
533 # gplines.append('set xtics 21600\n')
534 # gplines.append('set mxtics 3\n')
535
536 # ctr = 1
537 # for vo in volist:
538 # if ctr == 1:
539 # pstr = 'plot '
540 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
541 # pstr += '"< tail -720 ' + datafnam + '" using 1:3' + \
542 # ' title "' + vo + '"'
543 # if ctr < len(volist):
544 # pstr += ' with lines lw 3,'
545 # ctr += 1
546 # else:
547 # pstr += ' with lines lw 1'
548 # gplines.append(pstr + '\n')
549
550 # gplines.append('set terminal png size 1000,535\n')
551 # gplines.append('set output "/user/templon/public_html/wait-day-large.png"\n')
552 # gplines.append('set format x "%H:%M"\n')
553 # gplines.append('set format y "%.0te+%S"\n')
554 # gplines.append('set xtics 7200\n')
555 # gplines.append('set mxtics 4\n')
556
557 # ctr = 1
558 # for vo in volist:
559 # if ctr == 1:
560 # pstr = 'plot '
561 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
562 # pstr += '"< tail -720 ' + datafnam + '" using 1:3' + \
563 # ' title "' + vo + '"'
564 # if ctr < len(volist):
565 # pstr += ' with lines lw 3,'
566 # ctr += 1
567 # else:
568 # pstr += ' with lines lw 1'
569 # gplines.append(pstr + '\n')
570
571 # gplines.append('set terminal png size 335,201\n')
572 # gplines.append('set output "/user/templon/public_html/wait-week-small.png"\n')
573 # gplines.append('set format x "%a"\n')
574 # gplines.append('set format y "%.0te+%S"\n')
575 # gplines.append('set xtics 86400\n')
576 # gplines.append('set mxtics 3\n')
577
578 # ctr = 1
579 # for vo in volist:
580 # if ctr == 1:
581 # pstr = 'plot '
582 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
583 # pstr += '"< tail -5760 ' + datafnam + '" using 1:3' + \
584 # ' title "' + vo + '"'
585 # if ctr < len(volist):
586 # pstr += ' with lines lw 3,'
587 # ctr += 1
588 # else:
589 # pstr += ' with lines lw 1'
590 # gplines.append(pstr + '\n')
591
592 # gplines.append('set terminal png size 1000,535\n')
593 # gplines.append('set output "/user/templon/public_html/wait-week-large.png"\n')
594 # gplines.append('set format x "%a %H:%M"\n')
595 # gplines.append('set format y "%.0te+%S"\n')
596 # gplines.append('set xtics 86400\n')
597 # gplines.append('set mxtics 3\n')
598
599 # ctr = 1
600 # for vo in volist:
601 # if ctr == 1:
602 # pstr = 'plot '
603 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
604 # pstr += '"< tail -5760 ' + datafnam + '" using 1:3' + \
605 # ' title "' + vo + '"'
606 # if ctr < len(volist):
607 # pstr += ' with lines lw 3,'
608 # ctr += 1
609 # else:
610 # pstr += ' with lines lw 1'
611
612 for itm in plottuple_list:
613 emit_plotfile(plotfileh, itm)
614
615 plotfileh.close()
616 os.system(os.environ['HOME']+'/bin/gnuplot ' + pfnam)

Properties

Name Value
svn:executable *
svn:keywords Id URL

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28