/[pdpsoft]/nl.nikhef.ndpf.groupviews/tags/REL-2.0.0/ndpf-gv-mkplots
ViewVC logotype

Contents of /nl.nikhef.ndpf.groupviews/tags/REL-2.0.0/ndpf-gv-mkplots

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2395 - (show annotations) (download)
Mon Aug 8 10:31:46 2011 UTC (10 years, 10 months ago) by templon
File size: 18119 byte(s)
tag release 2.0.0
1 #! /usr/bin/env python
2 # $Id$
3 # Source: $URL$
4 # J. A. Templon, NIKHEF/PDP 2011
5
6 import os
7
8 NUMGROUPS=8
9 DATADIR=os.environ['HOME'] + '/ndpfdata/'
10 PLOTDIR=os.environ['HOME'] + '/public_html/'
11 # 8 class qualitative paired color scheme
12
13 colors = [ "#A6CEE3", "#1F77B4", "#B2DF8A", "#33A02C",
14 "#FB9A99", "#E31A1C", "#FDBF6F", "#FF7F00" ]
15
16 colors.reverse()
17
18 timeargs = { 'hr' : [ '-s', 'n-4h', '-e', 'n' ],
19 'day' : [ '-s', 'n-29h', '-e', 'n' ],
20 'week' : [ '-s', 'n-8d', '-e', 'n' ],
21 'month' : [ '-s', 'n-35d', '-e', 'n' ]
22 }
23
24 # for reference : resolutions of the RRAs
25 # RRA:AVERAGE:0.5:1:2880 \
26 # RRA:AVERAGE:0.5:5:4032 \
27 # RRA:AVERAGE:0.5:60:1440 \
28 # RRA:AVERAGE:0.5:1440:3650
29 # 60s (1 min) x 2880 points = 48 hr
30 # 300s (5 min) x 4032 points = 14 days
31 # 3600s (1 hr) x 1440 points = 60 days
32 # 86400s (1 day) x 3650 points = 10 years
33
34 import rrdtool
35
36 # first need to find four "top eight" lists
37 # one for the past day, which will be used for the hour / day page
38 # one for the past week
39 # one for the past month
40 # one for the past year
41 # base it on running jobs
42
43 import time
44 now=int(time.mktime(time.localtime()))
45 end = (now / 60) * 60
46 start = end - (3600 * 24) + 60
47
48 import glob # number of running jobs
49
50 ### block finding 'top N' group list ###
51
52 day_groups = dict() # structure day_group[groupname] = total of hourly average
53
54 running_files = glob.glob(DATADIR+'*.running.rrd')
55 for db in running_files:
56 group = db[len(DATADIR):db.find('.running.rrd')]
57 if group == "total":
58 continue
59 tup = rrdtool.fetch(db,'AVERAGE','-r',"60",
60 '-s', repr(start), '-e', repr(end))
61 vallist = [0] # start with zero, in case no vals returned, get zero as answer
62 for tup2 in tup[2]:
63 val = tup2[0]
64 if val:
65 vallist.append(val)
66 day_groups[group] = sum(vallist)
67
68 day_groups_sorted = sorted(day_groups, key=day_groups.get, reverse=False)
69 topgroups_day=day_groups_sorted[-NUMGROUPS:]
70
71 ### end block 'top N' group list ###
72
73 ### block generating plots ###
74
75 commonargs = ['--imgformat', 'PNG',
76 '--legend-position=east', '--legend-direction=bottomup']
77
78 sizeargs = { 'small' : [ '--width', '195', '--height', '125' ],
79 'large' : [ '--width', '850', '--height', '500' ]
80 }
81 def doplot(grouplist, dbtype, psize, timetag):
82
83 defs = list()
84 plots = list()
85
86 data_defs = list()
87 plot_defs = list()
88
89 for group in (grouplist + ['total']):
90 data_defs.append('DEF:'+group+'='+DATADIR+group+'.'+dbtype+'.rrd:'+dbtype+':AVERAGE')
91
92 for idx in range(len(grouplist)):
93 group = grouplist[idx]
94 pdefstr = 'AREA' ':' + group + colors[idx] + ':' + group
95 pdefstr = pdefstr + '\\n'
96 if idx > 0:
97 pdefstr = pdefstr + ':STACK'
98 plot_defs.append(pdefstr)
99
100 plot_defs.append('LINE:total#000000:total')
101
102 pargs = [ PLOTDIR + dbtype + '-' + timetag + '-' + psize + '.png'] + \
103 commonargs + sizeargs[psize] + timeargs[timetag] + \
104 data_defs + plot_defs
105 rrdtool.graph( *pargs )
106
107 def doplot_wait(grouplist, dbtype, psize, timetag):
108
109 defs = list()
110 plots = list()
111
112 data_defs = list()
113 plot_defs = list()
114
115 for group in (grouplist + ['rollover']):
116 data_defs.append('DEF:'+group+'='+DATADIR+group+'.'+dbtype+'.rrd:'+dbtype+':AVERAGE')
117
118 for idx in range(len(grouplist)):
119 group = grouplist[idx]
120 pdefstr = 'LINE3' ':' + group + colors[idx] + ':' + group
121 pdefstr = pdefstr + '\\n'
122 plot_defs.append(pdefstr)
123
124 plot_defs.append('LINE2:rollover#000000:rollover')
125
126 pargs = [ PLOTDIR + dbtype + '-' + timetag + '-' + psize + '.png'] + \
127 ['--slope-mode', '-o'] + \
128 commonargs + sizeargs[psize] + timeargs[timetag] + \
129 data_defs + plot_defs
130 rrdtool.graph( *pargs )
131
132 for dbtype in ['queued', 'running', 'waittime']:
133 for psize in ['small', 'large']:
134 for timetag in ['hr', 'day']:
135 if dbtype == 'waittime':
136 doplot_wait(topgroups_day, dbtype, psize, timetag)
137 else:
138 doplot(topgroups_day, dbtype, psize, timetag)
139
140 import time
141 now=int(time.mktime(time.localtime()))
142 end = (now / 3600 ) * 3600
143 start = end - (86400*30) + 3600
144
145 ### block finding 'top N' group list ###
146
147 month_groups = dict() # structure day_group[groupname] = total of hourly average
148
149 for db in running_files:
150 group = db[len(DATADIR):db.find('.running.rrd')]
151 if group == "total":
152 continue
153 tup = rrdtool.fetch(db,'AVERAGE','-r',"3600",
154 '-s', repr(start), '-e', repr(end))
155 vallist = [0] # start with zero, in case no vals returned, get zero as answer
156 for tup2 in tup[2]:
157 val = tup2[0]
158 if val:
159 vallist.append(val)
160 month_groups[group] = sum(vallist)
161
162 month_groups_sorted = sorted(month_groups, key=month_groups.get, reverse=False)
163 topgroups_month=month_groups_sorted[-NUMGROUPS:]
164
165 ### end block 'top N' group list ###
166
167 for dbtype in ['queued', 'running', 'waittime']:
168 for psize in ['small', 'large']:
169 for timetag in ['week', 'month']:
170 if dbtype == 'waittime':
171 doplot_wait(topgroups_month, dbtype, psize, timetag)
172 else:
173 doplot(topgroups_month, dbtype, psize, timetag)
174 import sys
175 sys.exit(0)
176
177 pfnam = os.environ['HOME']+'/tmp/run.plt'
178
179 infh = open(os.environ['HOME']+'/tmp/groups.rdata','r')
180 datablocks = infh.read().split('\n\n')
181
182 countslist = list() # holds one rundict per timestamp
183 timestringlist = list() # same order as countslist
184
185 for block in datablocks[:-1]:
186 lines = block.split('\n')
187 timestr = lines[0] ; timestringlist.append(timestr)
188 rundict = dict() # format rundict['vo'] = number_running
189 for l in lines[1:]:
190 vo, num = eval(l, {}, {})
191 rundict[vo] = num
192 countslist.append(rundict)
193
194 def avgdata(vals, tstamps, step):
195 # step must be odd
196 # lengths of vals and tstamps must be identical, and
197 # also must be integer multiple of step
198
199 newvals = list()
200 newstamps = list()
201 if len(vals) % step != 0:
202 print "length of input arrays must be multiple of step size"
203 sys.exit(0)
204 optr = 0
205 while optr < len(vals):
206 print optr
207 print step
208 print vals[optr:optr+step]
209
210 newvals.append(sum(vals[optr:optr+step]))
211 newstamps.append(tstamps[optr + (fact/2)])
212 optr += fact
213 return newvals, newstamps
214
215 sl1 = countslist[-75:]
216 sl2 = timestringlist[-75:]
217
218 hrtuple_small = (
219 sl1, sl2,
220 [
221 # 'set terminal x11 size 1000,535\n',
222 'set terminal png size 365,201 font "arial,8"\n',
223 'set key autotitle columnheader\n',
224 'set xdata time\n',
225 'set timefmt "%b-%d-%Y-%H:%M:%S"\n',
226 'set output "/user/templon/public_html/run-hr-small.png"\n',
227 '# time range must be in same format as data file\n',
228 'set grid xtics mxtics ytics mytics front\n',
229 'set yrange [0 : *]\n',
230 'set auto fix\n'
231 'set key outside samplen 1\n',
232 'set format x "%H:%M"\n',
233 'set xtics 3600\n',
234 'set mxtics 6\n',
235 'set mytics 2\n'
236 ],
237 os.environ['HOME']+'/tmp/myd_hr_s'
238 )
239
240 hrtuple_large = (
241 sl1, sl2,
242 [
243 'set terminal png size 1000,535 font "arial,10"\n',
244 'set output "/user/templon/public_html/run-hr-large.png"\n',
245 'set format x "%H:%M"\n',
246 'set xtics 3600\n',
247 'set mxtics 6\n',
248 'set mytics 4\n'
249 'set key autotitle columnheader\n',
250 'set xdata time\n',
251 'set timefmt "%b-%d-%Y-%H:%M:%S"\n',
252 '# time range must be in same format as data file\n',
253 'set grid xtics mxtics ytics mytics front\n',
254 'set yrange [0 : *]\n',
255 'set auto fix\n'
256 'set key outside samplen 1\n',
257 ],
258 os.environ['HOME']+'/tmp/myd_hr_l'
259 )
260
261 sl3 = countslist[-720:]
262 sl4 = timestringlist[-720:]
263
264 daytuple_small = (
265 sl3, sl4,
266 [
267 'set terminal png size 365,201 font "arial,8"\n',
268 'set output "/user/templon/public_html/run-day-small.png"\n',
269 'set format x "%H:%M"\n',
270 'set xtics 21600\n',
271 'set mxtics 3\n'
272 ],
273 os.environ['HOME']+'/tmp/myd_day_s'
274 )
275
276 daytuple_large = (
277 sl3, sl4,
278 [
279 'set terminal png size 1000,535 font "arial,10"\n',
280 'set output "/user/templon/public_html/run-day-large.png"\n',
281 'set format x "%H:%M"\n',
282 'set xtics 7200\n',
283 'set mxtics 4\n'
284 ],
285 os.environ['HOME']+'/tmp/myd_day_l'
286 )
287
288 sl5 = countslist[-5760:]
289 sl6 = timestringlist[-5760:]
290
291 weektuple_small = (
292 sl5[::5], sl6[::5],
293 [
294 'set terminal png size 365,201 font "arial,8"\n',
295 'set output "/user/templon/public_html/run-week-small.png"\n',
296 'set format x "%a"\n',
297 'set xtics 86400\n',
298 'set nomxtics \n'
299 ],
300 os.environ['HOME']+'/tmp/myd_week_s'
301 )
302
303 # sl7, sl8 = avgdata(sl5, sl6, 5)
304
305 weektuple_large = (
306 sl5[::2], sl6[::2],
307 [
308 'set terminal png size 1000,535 font "arial,10"\n',
309 'set output "/user/templon/public_html/run-week-large.png"\n',
310 'set format x "%a"\n',
311 'set xtics 86400\n',
312 'set mxtics 3\n'
313 ],
314 os.environ['HOME']+'/tmp/myd_week_l'
315 )
316
317 sl10 = countslist[-22300:]
318 sl11 = timestringlist[-22300:]
319
320 # sl12, sl13 = avgdata(sl10, sl11, 100)
321
322 monthtuple_small = (
323 sl10[::100], sl11[::100],
324 [
325 'set terminal png size 365,201 font "arial,8"\n',
326 'set output "/user/templon/public_html/run-month-small.png"\n',
327 'set format x "%m/%d"\n',
328 'set xtics 604800\n',
329 'set nomxtics \n'
330 ],
331 os.environ['HOME']+'/tmp/myd_month_s'
332 )
333
334 # sl14, sl15 = avgdata(sl10, sl11, 20)
335
336 monthtuple_large = (
337 sl10[::20], sl11[::20],
338 [
339 'set terminal png size 1000,535 font "arial,10"\n',
340 'set output "/user/templon/public_html/run-month-large.png"\n',
341 'set format x "%m/%d"\n',
342 'set xtics 604800\n',
343 'set mxtics 7\n'
344 ],
345 os.environ['HOME']+'/tmp/myd_month_l'
346 )
347
348
349 # do it again but for queued jobs
350
351 infh = open(os.environ['HOME']+'/tmp/groups.qdata','r')
352 datablocks = infh.read().split('\n\n')
353
354 countslist = list() # holds one rundict per timestamp
355 timestringlist = list() # same order as countslist
356
357 for block in datablocks[:-1]:
358 lines = block.split('\n')
359 timestr = lines[0] ; timestringlist.append(timestr)
360 rundict = dict() # format rundict['vo'] = number_running
361 for l in lines[1:]:
362 vo, num = eval(l, {}, {})
363 rundict[vo] = num
364 countslist.append(rundict)
365
366 qhrtuple_small = (
367 countslist[-75:],
368 timestringlist[-75:],
369 [
370 # 'set terminal x11 size 1000,535\n',
371 'set terminal png size 365,201 font "arial,8"\n',
372 'set key autotitle columnheader\n',
373 'set xdata time\n',
374 'set timefmt "%b-%d-%Y-%H:%M:%S"\n',
375 'set output "/user/templon/public_html/queued-hr-small.png"\n',
376 '# time range must be in same format as data file\n',
377 'set grid xtics mxtics ytics mytics front\n',
378 'set yrange [0 : *]\n',
379 'set auto fix\n'
380 'set key outside samplen 1\n',
381 'set format x "%H:%M"\n',
382 'set xtics 3600\n',
383 'set mxtics 6\n',
384 'set mytics 3\n'
385 ],
386 os.environ['HOME']+'/tmp/myqd_hr'
387 )
388
389 qhrtuple_large = (
390 countslist[-75:],
391 timestringlist[-75:],
392 [
393 'set terminal png size 1000,535 font "arial,10"\n',
394 'set output "/user/templon/public_html/queued-hr-large.png"\n',
395 'set format x "%H:%M"\n',
396 'set xtics 3600\n',
397 'set mxtics 6\n',
398 'set mytics 3\n'
399 'set key autotitle columnheader\n',
400 'set xdata time\n',
401 'set timefmt "%b-%d-%Y-%H:%M:%S"\n',
402 '# time range must be in same format as data file\n',
403 'set grid xtics mxtics ytics mytics front\n',
404 # 'set yrange [0 : 2600]\n',
405 'set auto fix\n'
406 'set key outside samplen 1\n',
407 ],
408 os.environ['HOME']+'/tmp/myqd_hr'
409 )
410
411 qdaytuple_small = (
412 countslist[-720:],
413 timestringlist[-720:],
414 [
415 'set terminal png size 365,201 font "arial,8"\n',
416 'set output "/user/templon/public_html/queued-day-small.png"\n',
417 'set format x "%H:%M"\n',
418 'set xtics 21600\n',
419 'set mxtics 3\n'
420 ],
421 os.environ['HOME']+'/tmp/myqd_day'
422 )
423
424 qdaytuple_large = (
425 countslist[-720:],
426 timestringlist[-720:],
427 [
428 'set terminal png size 1000,535 font "arial,10"\n',
429 'set output "/user/templon/public_html/queued-day-large.png"\n',
430 'set format x "%H:%M"\n',
431 'set xtics 7200\n',
432 'set mxtics 4\n'
433 ],
434 os.environ['HOME']+'/tmp/myqd_day'
435 )
436
437
438 plottuple_list = [ hrtuple_small, hrtuple_large, daytuple_small,
439 daytuple_large, weektuple_small, weektuple_large,
440 monthtuple_small, monthtuple_large,
441 qhrtuple_small, qhrtuple_large,
442 qdaytuple_small, qdaytuple_large
443 ]
444
445 plotfileh = open(pfnam,'w')
446
447 def emit_plotfile(pfh, plotslist):
448 # pfh is handle to plot file object
449 # first make the right data structure: rundata['vo']['timestamp'] = number
450 rundata = dict() # format rundata['vo']['timestamp'] = number
451 tsl = plotslist[1]
452 ctl = plotslist[0]
453 for i in range(len(tsl)):
454 for vo in ctl[i].keys():
455 if vo not in rundata.keys():
456 rundata[vo] = dict()
457 rundata[vo][tsl[i]] = ctl[i][vo]
458
459 sums = dict()
460 for k in rundata.keys():
461 sums[k] = sum(rundata[k].values())
462
463 vos_by_jobs = sorted(sums, key=sums.get, reverse=False)
464
465 topvos = vos_by_jobs[-NUMGROUPS:]
466
467 dfh = open(plotslist[3], 'w')
468 dfh.write('timestamp ')
469 for k in topvos:
470 dfh.write("%8s " % k)
471 dfh.write('\n')
472 for t in tsl:
473 dfh.write(t+ " ")
474 for k in topvos:
475 if t in rundata[k].keys():
476 val = rundata[k][t]
477 else:
478 val = 0
479 dfh.write("%4d " % val)
480 dfh.write('\n')
481 dfh.close()
482
483 import glob
484 import os
485
486 gplines = plotslist[2]
487
488 # gplines.append('set terminal png size 335,201\n')
489
490 ctr = 1
491 for vo in reversed(topvos):
492 if ctr == 1:
493 pstr = 'plot '
494 datafnam = plotslist[3]
495 pstr += '"' + datafnam + '" using 1:($' + repr(topvos.index(vo)+2)
496 for k2 in range(topvos.index(vo)+1, 1, -1):
497 pstr += '+$' + repr(k2)
498 pstr += ') t "' + vo + '"'
499 if ctr < len(topvos):
500 pstr += ' w boxes fs solid,'
501 ctr += 1
502 else:
503 pstr += ' w boxes fs solid'
504 gplines.append(pstr + '\n')
505
506 for l in gplines:
507 pfh.write(l)
508
509 # gplines.append('set terminal png size 1000,535\n')
510 # gplines.append('set output "/user/templon/public_html/wait-hr-large.png"\n')
511 # gplines.append('set format x "%H:%M"\n')
512 # gplines.append('set format y "%.0te+%S"\n')
513 # gplines.append('set xtics 3600\n')
514
515 # ctr = 1
516 # for vo in volist:
517 # if ctr == 1:
518 # pstr = 'plot '
519 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
520 # pstr += '"< tail -75 ' + datafnam + '" using 1:3' + \
521 # ' title "' + vo + '"'
522 # if ctr < len(volist):
523 # pstr += ' with lines lw 3,'
524 # ctr += 1
525 # else:
526 # pstr += ' with lines lw 1'
527 # gplines.append(pstr + '\n')
528
529 # gplines.append('set terminal png size 335,201\n')
530 # gplines.append('set output "/user/templon/public_html/wait-day-small.png"\n')
531 # gplines.append('set format x "%H:%M"\n')
532 # gplines.append('set format y "%.0te+%S"\n')
533 # gplines.append('set xtics 21600\n')
534 # gplines.append('set mxtics 3\n')
535
536 # ctr = 1
537 # for vo in volist:
538 # if ctr == 1:
539 # pstr = 'plot '
540 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
541 # pstr += '"< tail -720 ' + datafnam + '" using 1:3' + \
542 # ' title "' + vo + '"'
543 # if ctr < len(volist):
544 # pstr += ' with lines lw 3,'
545 # ctr += 1
546 # else:
547 # pstr += ' with lines lw 1'
548 # gplines.append(pstr + '\n')
549
550 # gplines.append('set terminal png size 1000,535\n')
551 # gplines.append('set output "/user/templon/public_html/wait-day-large.png"\n')
552 # gplines.append('set format x "%H:%M"\n')
553 # gplines.append('set format y "%.0te+%S"\n')
554 # gplines.append('set xtics 7200\n')
555 # gplines.append('set mxtics 4\n')
556
557 # ctr = 1
558 # for vo in volist:
559 # if ctr == 1:
560 # pstr = 'plot '
561 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
562 # pstr += '"< tail -720 ' + datafnam + '" using 1:3' + \
563 # ' title "' + vo + '"'
564 # if ctr < len(volist):
565 # pstr += ' with lines lw 3,'
566 # ctr += 1
567 # else:
568 # pstr += ' with lines lw 1'
569 # gplines.append(pstr + '\n')
570
571 # gplines.append('set terminal png size 335,201\n')
572 # gplines.append('set output "/user/templon/public_html/wait-week-small.png"\n')
573 # gplines.append('set format x "%a"\n')
574 # gplines.append('set format y "%.0te+%S"\n')
575 # gplines.append('set xtics 86400\n')
576 # gplines.append('set mxtics 3\n')
577
578 # ctr = 1
579 # for vo in volist:
580 # if ctr == 1:
581 # pstr = 'plot '
582 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
583 # pstr += '"< tail -5760 ' + datafnam + '" using 1:3' + \
584 # ' title "' + vo + '"'
585 # if ctr < len(volist):
586 # pstr += ' with lines lw 3,'
587 # ctr += 1
588 # else:
589 # pstr += ' with lines lw 1'
590 # gplines.append(pstr + '\n')
591
592 # gplines.append('set terminal png size 1000,535\n')
593 # gplines.append('set output "/user/templon/public_html/wait-week-large.png"\n')
594 # gplines.append('set format x "%a %H:%M"\n')
595 # gplines.append('set format y "%.0te+%S"\n')
596 # gplines.append('set xtics 86400\n')
597 # gplines.append('set mxtics 3\n')
598
599 # ctr = 1
600 # for vo in volist:
601 # if ctr == 1:
602 # pstr = 'plot '
603 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
604 # pstr += '"< tail -5760 ' + datafnam + '" using 1:3' + \
605 # ' title "' + vo + '"'
606 # if ctr < len(volist):
607 # pstr += ' with lines lw 3,'
608 # ctr += 1
609 # else:
610 # pstr += ' with lines lw 1'
611
612 for itm in plottuple_list:
613 emit_plotfile(plotfileh, itm)
614
615 plotfileh.close()
616 os.system(os.environ['HOME']+'/bin/gnuplot ' + pfnam)

Properties

Name Value
svn:executable *
svn:keywords Id URL

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28