/[pdpsoft]/nl.nikhef.ndpf.groupviews/trunk/ndpf-gv-mkplots
ViewVC logotype

Contents of /nl.nikhef.ndpf.groupviews/trunk/ndpf-gv-mkplots

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2376 - (show annotations) (download)
Sun Jul 24 15:09:30 2011 UTC (10 years, 10 months ago) by templon
File size: 18046 byte(s)
fix resolution bug in month plot

1 #! /usr/bin/env python
2
3 import os
4
5 NUMGROUPS=8
6 DATADIR=os.environ['HOME'] + '/ndpfdata/'
7 PLOTDIR=os.environ['HOME'] + '/public_html/'
8 # 8 class qualitative paired color scheme
9
10 colors = [ "#A6CEE3", "#1F77B4", "#B2DF8A", "#33A02C",
11 "#FB9A99", "#E31A1C", "#FDBF6F", "#FF7F00" ]
12
13 colors.reverse()
14
15 timeargs = { 'hr' : [ '-s', 'n-3h30min', '-e', 'n' ],
16 'day' : [ '-s', 'n-29h', '-e', 'n' ],
17 'week' : [ '-s', 'n-8d', '-e', 'n' ],
18 'month' : [ '-s', 'n-35d', '-e', 'n' ]
19 }
20
21 # for reference : resolutions of the RRAs
22 # 120s x 240 points = 8 hr
23 # 240s x 900 points = 60 hr
24 # 1800s x 768 points = 192 hr (16 days)
25 # 7200s (2 hrs) x 840 points = 70 days
26 # 86400s (1 day) x 3650 points = 10 years
27
28 import rrdtool
29
30 # first need to find four "top eight" lists
31 # one for the past day, which will be used for the hour / day page
32 # one for the past week
33 # one for the past month
34 # one for the past year
35 # base it on running jobs
36
37 import time
38 now=int(time.mktime(time.localtime()))
39 end = (now / 240) * 240
40 start = end - (3600 * 30) + 240
41
42 import glob # number of running jobs
43
44 ### block finding 'top N' group list ###
45
46 day_groups = dict() # structure day_group[groupname] = total of hourly average
47
48 running_files = glob.glob(DATADIR+'*.running.rrd')
49 for db in running_files:
50 group = db[len(DATADIR):db.find('.running.rrd')]
51 if group == "total":
52 continue
53 tup = rrdtool.fetch(db,'AVERAGE','-r',"240",
54 '-s', repr(start), '-e', repr(end))
55 vallist = [0] # start with zero, in case no vals returned, get zero as answer
56 for tup2 in tup[2]:
57 val = tup2[0]
58 if val:
59 vallist.append(val)
60 day_groups[group] = sum(vallist[:-1]) # chop weird last value from rrdtool
61
62 day_groups_sorted = sorted(day_groups, key=day_groups.get, reverse=False)
63 topgroups_day=day_groups_sorted[-NUMGROUPS:]
64
65 ### end block 'top N' group list ###
66
67 ### block generating plots ###
68
69 commonargs = ['--imgformat', 'PNG' ]
70 sizeargs = { 'small' : [ '--width', '285', '--height', '125' ],
71 'large' : [ '--width', '950', '--height', '500' ]
72 }
73 def doplot(grouplist, dbtype, psize, timetag):
74
75 defs = list()
76 plots = list()
77
78 data_defs = list()
79 plot_defs = list()
80
81 for group in (grouplist + ['total']):
82 data_defs.append('DEF:'+group+'='+DATADIR+group+'.'+dbtype+'.rrd:'+dbtype+':AVERAGE')
83
84 for idx in range(len(grouplist)):
85 group = grouplist[idx]
86 pdefstr = 'AREA' ':' + group + colors[idx] + ':' + group
87 if idx < 5 and psize == 'small':
88 pdefstr = pdefstr + '\g'
89 if idx > 0:
90 pdefstr = pdefstr + ':STACK'
91 plot_defs.append(pdefstr)
92
93 plot_defs.append('LINE:total#000000:total')
94
95 pargs = [ PLOTDIR + dbtype + '-' + timetag + '-' + psize + '.png'] + \
96 commonargs + sizeargs[psize] + timeargs[timetag] + \
97 data_defs + plot_defs
98 rrdtool.graph( *pargs )
99
100 def doplot_wait(grouplist, dbtype, psize, timetag):
101
102 defs = list()
103 plots = list()
104
105 data_defs = list()
106 plot_defs = list()
107
108 for group in (grouplist + ['rollover']):
109 data_defs.append('DEF:'+group+'='+DATADIR+group+'.'+dbtype+'.rrd:'+dbtype+':AVERAGE')
110
111 for idx in range(len(grouplist)):
112 group = grouplist[idx]
113 pdefstr = 'LINE3' ':' + group + colors[idx] + ':' + group
114 if idx < 5 and psize == 'small':
115 pdefstr = pdefstr + '\g'
116 plot_defs.append(pdefstr)
117
118 plot_defs.append('LINE2:rollover#000000:rollover')
119
120 pargs = [ PLOTDIR + dbtype + '-' + timetag + '-' + psize + '.png'] + \
121 ['--slope-mode', '-o'] + \
122 commonargs + sizeargs[psize] + timeargs[timetag] + \
123 data_defs + plot_defs
124 rrdtool.graph( *pargs )
125
126 for dbtype in ['queued', 'running', 'waittime']:
127 for psize in ['small', 'large']:
128 for timetag in ['hr', 'day']:
129 if dbtype == 'waittime':
130 doplot_wait(topgroups_day, dbtype, psize, timetag)
131 else:
132 doplot(topgroups_day, dbtype, psize, timetag)
133
134 import time
135 now=int(time.mktime(time.localtime()))
136 end = (now / 7200 ) * 7200
137 start = end - (86400*35) + 7200
138
139 ### block finding 'top N' group list ###
140
141 month_groups = dict() # structure day_group[groupname] = total of hourly average
142
143 for db in running_files:
144 group = db[len(DATADIR):db.find('.running.rrd')]
145 if group == "total":
146 continue
147 tup = rrdtool.fetch(db,'AVERAGE','-r',"7200",
148 '-s', repr(start), '-e', repr(end))
149 vallist = [0] # start with zero, in case no vals returned, get zero as answer
150 for tup2 in tup[2]:
151 val = tup2[0]
152 if val:
153 vallist.append(val)
154 month_groups[group] = sum(vallist[:-1]) # chop weird last value from rrdtool
155
156 month_groups_sorted = sorted(month_groups, key=month_groups.get, reverse=False)
157 topgroups_month=month_groups_sorted[-NUMGROUPS:]
158
159 ### end block 'top N' group list ###
160
161 for dbtype in ['queued', 'running', 'waittime']:
162 for psize in ['small', 'large']:
163 for timetag in ['week', 'month']:
164 if dbtype == 'waittime':
165 doplot_wait(topgroups_month, dbtype, psize, timetag)
166 else:
167 doplot(topgroups_month, dbtype, psize, timetag)
168 import sys
169 sys.exit(0)
170
171 pfnam = os.environ['HOME']+'/tmp/run.plt'
172
173 infh = open(os.environ['HOME']+'/tmp/groups.rdata','r')
174 datablocks = infh.read().split('\n\n')
175
176 countslist = list() # holds one rundict per timestamp
177 timestringlist = list() # same order as countslist
178
179 for block in datablocks[:-1]:
180 lines = block.split('\n')
181 timestr = lines[0] ; timestringlist.append(timestr)
182 rundict = dict() # format rundict['vo'] = number_running
183 for l in lines[1:]:
184 vo, num = eval(l, {}, {})
185 rundict[vo] = num
186 countslist.append(rundict)
187
188 def avgdata(vals, tstamps, step):
189 # step must be odd
190 # lengths of vals and tstamps must be identical, and
191 # also must be integer multiple of step
192
193 newvals = list()
194 newstamps = list()
195 if len(vals) % step != 0:
196 print "length of input arrays must be multiple of step size"
197 sys.exit(0)
198 optr = 0
199 while optr < len(vals):
200 print optr
201 print step
202 print vals[optr:optr+step]
203
204 newvals.append(sum(vals[optr:optr+step]))
205 newstamps.append(tstamps[optr + (fact/2)])
206 optr += fact
207 return newvals, newstamps
208
209 sl1 = countslist[-75:]
210 sl2 = timestringlist[-75:]
211
212 hrtuple_small = (
213 sl1, sl2,
214 [
215 # 'set terminal x11 size 1000,535\n',
216 'set terminal png size 365,201 font "arial,8"\n',
217 'set key autotitle columnheader\n',
218 'set xdata time\n',
219 'set timefmt "%b-%d-%Y-%H:%M:%S"\n',
220 'set output "/user/templon/public_html/run-hr-small.png"\n',
221 '# time range must be in same format as data file\n',
222 'set grid xtics mxtics ytics mytics front\n',
223 'set yrange [0 : *]\n',
224 'set auto fix\n'
225 'set key outside samplen 1\n',
226 'set format x "%H:%M"\n',
227 'set xtics 3600\n',
228 'set mxtics 6\n',
229 'set mytics 2\n'
230 ],
231 os.environ['HOME']+'/tmp/myd_hr_s'
232 )
233
234 hrtuple_large = (
235 sl1, sl2,
236 [
237 'set terminal png size 1000,535 font "arial,10"\n',
238 'set output "/user/templon/public_html/run-hr-large.png"\n',
239 'set format x "%H:%M"\n',
240 'set xtics 3600\n',
241 'set mxtics 6\n',
242 'set mytics 4\n'
243 'set key autotitle columnheader\n',
244 'set xdata time\n',
245 'set timefmt "%b-%d-%Y-%H:%M:%S"\n',
246 '# time range must be in same format as data file\n',
247 'set grid xtics mxtics ytics mytics front\n',
248 'set yrange [0 : *]\n',
249 'set auto fix\n'
250 'set key outside samplen 1\n',
251 ],
252 os.environ['HOME']+'/tmp/myd_hr_l'
253 )
254
255 sl3 = countslist[-720:]
256 sl4 = timestringlist[-720:]
257
258 daytuple_small = (
259 sl3, sl4,
260 [
261 'set terminal png size 365,201 font "arial,8"\n',
262 'set output "/user/templon/public_html/run-day-small.png"\n',
263 'set format x "%H:%M"\n',
264 'set xtics 21600\n',
265 'set mxtics 3\n'
266 ],
267 os.environ['HOME']+'/tmp/myd_day_s'
268 )
269
270 daytuple_large = (
271 sl3, sl4,
272 [
273 'set terminal png size 1000,535 font "arial,10"\n',
274 'set output "/user/templon/public_html/run-day-large.png"\n',
275 'set format x "%H:%M"\n',
276 'set xtics 7200\n',
277 'set mxtics 4\n'
278 ],
279 os.environ['HOME']+'/tmp/myd_day_l'
280 )
281
282 sl5 = countslist[-5760:]
283 sl6 = timestringlist[-5760:]
284
285 weektuple_small = (
286 sl5[::5], sl6[::5],
287 [
288 'set terminal png size 365,201 font "arial,8"\n',
289 'set output "/user/templon/public_html/run-week-small.png"\n',
290 'set format x "%a"\n',
291 'set xtics 86400\n',
292 'set nomxtics \n'
293 ],
294 os.environ['HOME']+'/tmp/myd_week_s'
295 )
296
297 # sl7, sl8 = avgdata(sl5, sl6, 5)
298
299 weektuple_large = (
300 sl5[::2], sl6[::2],
301 [
302 'set terminal png size 1000,535 font "arial,10"\n',
303 'set output "/user/templon/public_html/run-week-large.png"\n',
304 'set format x "%a"\n',
305 'set xtics 86400\n',
306 'set mxtics 3\n'
307 ],
308 os.environ['HOME']+'/tmp/myd_week_l'
309 )
310
311 sl10 = countslist[-22300:]
312 sl11 = timestringlist[-22300:]
313
314 # sl12, sl13 = avgdata(sl10, sl11, 100)
315
316 monthtuple_small = (
317 sl10[::100], sl11[::100],
318 [
319 'set terminal png size 365,201 font "arial,8"\n',
320 'set output "/user/templon/public_html/run-month-small.png"\n',
321 'set format x "%m/%d"\n',
322 'set xtics 604800\n',
323 'set nomxtics \n'
324 ],
325 os.environ['HOME']+'/tmp/myd_month_s'
326 )
327
328 # sl14, sl15 = avgdata(sl10, sl11, 20)
329
330 monthtuple_large = (
331 sl10[::20], sl11[::20],
332 [
333 'set terminal png size 1000,535 font "arial,10"\n',
334 'set output "/user/templon/public_html/run-month-large.png"\n',
335 'set format x "%m/%d"\n',
336 'set xtics 604800\n',
337 'set mxtics 7\n'
338 ],
339 os.environ['HOME']+'/tmp/myd_month_l'
340 )
341
342
343 # do it again but for queued jobs
344
345 infh = open(os.environ['HOME']+'/tmp/groups.qdata','r')
346 datablocks = infh.read().split('\n\n')
347
348 countslist = list() # holds one rundict per timestamp
349 timestringlist = list() # same order as countslist
350
351 for block in datablocks[:-1]:
352 lines = block.split('\n')
353 timestr = lines[0] ; timestringlist.append(timestr)
354 rundict = dict() # format rundict['vo'] = number_running
355 for l in lines[1:]:
356 vo, num = eval(l, {}, {})
357 rundict[vo] = num
358 countslist.append(rundict)
359
360 qhrtuple_small = (
361 countslist[-75:],
362 timestringlist[-75:],
363 [
364 # 'set terminal x11 size 1000,535\n',
365 'set terminal png size 365,201 font "arial,8"\n',
366 'set key autotitle columnheader\n',
367 'set xdata time\n',
368 'set timefmt "%b-%d-%Y-%H:%M:%S"\n',
369 'set output "/user/templon/public_html/queued-hr-small.png"\n',
370 '# time range must be in same format as data file\n',
371 'set grid xtics mxtics ytics mytics front\n',
372 'set yrange [0 : *]\n',
373 'set auto fix\n'
374 'set key outside samplen 1\n',
375 'set format x "%H:%M"\n',
376 'set xtics 3600\n',
377 'set mxtics 6\n',
378 'set mytics 3\n'
379 ],
380 os.environ['HOME']+'/tmp/myqd_hr'
381 )
382
383 qhrtuple_large = (
384 countslist[-75:],
385 timestringlist[-75:],
386 [
387 'set terminal png size 1000,535 font "arial,10"\n',
388 'set output "/user/templon/public_html/queued-hr-large.png"\n',
389 'set format x "%H:%M"\n',
390 'set xtics 3600\n',
391 'set mxtics 6\n',
392 'set mytics 3\n'
393 'set key autotitle columnheader\n',
394 'set xdata time\n',
395 'set timefmt "%b-%d-%Y-%H:%M:%S"\n',
396 '# time range must be in same format as data file\n',
397 'set grid xtics mxtics ytics mytics front\n',
398 # 'set yrange [0 : 2600]\n',
399 'set auto fix\n'
400 'set key outside samplen 1\n',
401 ],
402 os.environ['HOME']+'/tmp/myqd_hr'
403 )
404
405 qdaytuple_small = (
406 countslist[-720:],
407 timestringlist[-720:],
408 [
409 'set terminal png size 365,201 font "arial,8"\n',
410 'set output "/user/templon/public_html/queued-day-small.png"\n',
411 'set format x "%H:%M"\n',
412 'set xtics 21600\n',
413 'set mxtics 3\n'
414 ],
415 os.environ['HOME']+'/tmp/myqd_day'
416 )
417
418 qdaytuple_large = (
419 countslist[-720:],
420 timestringlist[-720:],
421 [
422 'set terminal png size 1000,535 font "arial,10"\n',
423 'set output "/user/templon/public_html/queued-day-large.png"\n',
424 'set format x "%H:%M"\n',
425 'set xtics 7200\n',
426 'set mxtics 4\n'
427 ],
428 os.environ['HOME']+'/tmp/myqd_day'
429 )
430
431
432 plottuple_list = [ hrtuple_small, hrtuple_large, daytuple_small,
433 daytuple_large, weektuple_small, weektuple_large,
434 monthtuple_small, monthtuple_large,
435 qhrtuple_small, qhrtuple_large,
436 qdaytuple_small, qdaytuple_large
437 ]
438
439 plotfileh = open(pfnam,'w')
440
441 def emit_plotfile(pfh, plotslist):
442 # pfh is handle to plot file object
443 # first make the right data structure: rundata['vo']['timestamp'] = number
444 rundata = dict() # format rundata['vo']['timestamp'] = number
445 tsl = plotslist[1]
446 ctl = plotslist[0]
447 for i in range(len(tsl)):
448 for vo in ctl[i].keys():
449 if vo not in rundata.keys():
450 rundata[vo] = dict()
451 rundata[vo][tsl[i]] = ctl[i][vo]
452
453 sums = dict()
454 for k in rundata.keys():
455 sums[k] = sum(rundata[k].values())
456
457 vos_by_jobs = sorted(sums, key=sums.get, reverse=False)
458
459 topvos = vos_by_jobs[-NUMGROUPS:]
460
461 dfh = open(plotslist[3], 'w')
462 dfh.write('timestamp ')
463 for k in topvos:
464 dfh.write("%8s " % k)
465 dfh.write('\n')
466 for t in tsl:
467 dfh.write(t+ " ")
468 for k in topvos:
469 if t in rundata[k].keys():
470 val = rundata[k][t]
471 else:
472 val = 0
473 dfh.write("%4d " % val)
474 dfh.write('\n')
475 dfh.close()
476
477 import glob
478 import os
479
480 gplines = plotslist[2]
481
482 # gplines.append('set terminal png size 335,201\n')
483
484 ctr = 1
485 for vo in reversed(topvos):
486 if ctr == 1:
487 pstr = 'plot '
488 datafnam = plotslist[3]
489 pstr += '"' + datafnam + '" using 1:($' + repr(topvos.index(vo)+2)
490 for k2 in range(topvos.index(vo)+1, 1, -1):
491 pstr += '+$' + repr(k2)
492 pstr += ') t "' + vo + '"'
493 if ctr < len(topvos):
494 pstr += ' w boxes fs solid,'
495 ctr += 1
496 else:
497 pstr += ' w boxes fs solid'
498 gplines.append(pstr + '\n')
499
500 for l in gplines:
501 pfh.write(l)
502
503 # gplines.append('set terminal png size 1000,535\n')
504 # gplines.append('set output "/user/templon/public_html/wait-hr-large.png"\n')
505 # gplines.append('set format x "%H:%M"\n')
506 # gplines.append('set format y "%.0te+%S"\n')
507 # gplines.append('set xtics 3600\n')
508
509 # ctr = 1
510 # for vo in volist:
511 # if ctr == 1:
512 # pstr = 'plot '
513 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
514 # pstr += '"< tail -75 ' + datafnam + '" using 1:3' + \
515 # ' title "' + vo + '"'
516 # if ctr < len(volist):
517 # pstr += ' with lines lw 3,'
518 # ctr += 1
519 # else:
520 # pstr += ' with lines lw 1'
521 # gplines.append(pstr + '\n')
522
523 # gplines.append('set terminal png size 335,201\n')
524 # gplines.append('set output "/user/templon/public_html/wait-day-small.png"\n')
525 # gplines.append('set format x "%H:%M"\n')
526 # gplines.append('set format y "%.0te+%S"\n')
527 # gplines.append('set xtics 21600\n')
528 # gplines.append('set mxtics 3\n')
529
530 # ctr = 1
531 # for vo in volist:
532 # if ctr == 1:
533 # pstr = 'plot '
534 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
535 # pstr += '"< tail -720 ' + datafnam + '" using 1:3' + \
536 # ' title "' + vo + '"'
537 # if ctr < len(volist):
538 # pstr += ' with lines lw 3,'
539 # ctr += 1
540 # else:
541 # pstr += ' with lines lw 1'
542 # gplines.append(pstr + '\n')
543
544 # gplines.append('set terminal png size 1000,535\n')
545 # gplines.append('set output "/user/templon/public_html/wait-day-large.png"\n')
546 # gplines.append('set format x "%H:%M"\n')
547 # gplines.append('set format y "%.0te+%S"\n')
548 # gplines.append('set xtics 7200\n')
549 # gplines.append('set mxtics 4\n')
550
551 # ctr = 1
552 # for vo in volist:
553 # if ctr == 1:
554 # pstr = 'plot '
555 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
556 # pstr += '"< tail -720 ' + datafnam + '" using 1:3' + \
557 # ' title "' + vo + '"'
558 # if ctr < len(volist):
559 # pstr += ' with lines lw 3,'
560 # ctr += 1
561 # else:
562 # pstr += ' with lines lw 1'
563 # gplines.append(pstr + '\n')
564
565 # gplines.append('set terminal png size 335,201\n')
566 # gplines.append('set output "/user/templon/public_html/wait-week-small.png"\n')
567 # gplines.append('set format x "%a"\n')
568 # gplines.append('set format y "%.0te+%S"\n')
569 # gplines.append('set xtics 86400\n')
570 # gplines.append('set mxtics 3\n')
571
572 # ctr = 1
573 # for vo in volist:
574 # if ctr == 1:
575 # pstr = 'plot '
576 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
577 # pstr += '"< tail -5760 ' + datafnam + '" using 1:3' + \
578 # ' title "' + vo + '"'
579 # if ctr < len(volist):
580 # pstr += ' with lines lw 3,'
581 # ctr += 1
582 # else:
583 # pstr += ' with lines lw 1'
584 # gplines.append(pstr + '\n')
585
586 # gplines.append('set terminal png size 1000,535\n')
587 # gplines.append('set output "/user/templon/public_html/wait-week-large.png"\n')
588 # gplines.append('set format x "%a %H:%M"\n')
589 # gplines.append('set format y "%.0te+%S"\n')
590 # gplines.append('set xtics 86400\n')
591 # gplines.append('set mxtics 3\n')
592
593 # ctr = 1
594 # for vo in volist:
595 # if ctr == 1:
596 # pstr = 'plot '
597 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
598 # pstr += '"< tail -5760 ' + datafnam + '" using 1:3' + \
599 # ' title "' + vo + '"'
600 # if ctr < len(volist):
601 # pstr += ' with lines lw 3,'
602 # ctr += 1
603 # else:
604 # pstr += ' with lines lw 1'
605
606 for itm in plottuple_list:
607 emit_plotfile(plotfileh, itm)
608
609 plotfileh.close()
610 os.system(os.environ['HOME']+'/bin/gnuplot ' + pfnam)

Properties

Name Value
svn:executable *
svn:keywords Id URL

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28