/[pdpsoft]/nl.nikhef.ndpf.groupviews/tags/REL-1.1.0/ndpf-gv-mkplots
ViewVC logotype

Contents of /nl.nikhef.ndpf.groupviews/tags/REL-1.1.0/ndpf-gv-mkplots

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2385 - (show annotations) (download)
Thu Jul 28 12:06:57 2011 UTC (11 years ago) by templon
File size: 17998 byte(s)
Tag Release 1.1; version in service week 30, 2011
1 #! /usr/bin/env python
2 # $Id$
3 # Source: $URL$
4 # J. A. Templon, NIKHEF/PDP 2011
5
6 import os
7
8 NUMGROUPS=8
9 DATADIR=os.environ['HOME'] + '/ndpfdata/'
10 PLOTDIR=os.environ['HOME'] + '/public_html/'
11 # 8 class qualitative paired color scheme
12
13 colors = [ "#A6CEE3", "#1F77B4", "#B2DF8A", "#33A02C",
14 "#FB9A99", "#E31A1C", "#FDBF6F", "#FF7F00" ]
15
16 colors.reverse()
17
18 timeargs = { 'hr' : [ '-s', 'n-3h30min', '-e', 'n' ],
19 'day' : [ '-s', 'n-29h', '-e', 'n' ],
20 'week' : [ '-s', 'n-8d', '-e', 'n' ],
21 'month' : [ '-s', 'n-35d', '-e', 'n' ]
22 }
23
24 # for reference : resolutions of the RRAs
25 # 120s x 240 points = 8 hr
26 # 240s x 900 points = 60 hr
27 # 1800s x 768 points = 192 hr (16 days)
28 # 7200s (2 hrs) x 840 points = 70 days
29 # 86400s (1 day) x 3650 points = 10 years
30
31 import rrdtool
32
33 # first need to find four "top eight" lists
34 # one for the past day, which will be used for the hour / day page
35 # one for the past week
36 # one for the past month
37 # one for the past year
38 # base it on running jobs
39
40 import time
41 now=int(time.mktime(time.localtime()))
42 end = (now / 240) * 240
43 start = end - (3600 * 30) + 240
44
45 import glob # number of running jobs
46
47 ### block finding 'top N' group list ###
48
49 day_groups = dict() # structure day_group[groupname] = total of hourly average
50
51 running_files = glob.glob(DATADIR+'*.running.rrd')
52 for db in running_files:
53 group = db[len(DATADIR):db.find('.running.rrd')]
54 if group == "total":
55 continue
56 tup = rrdtool.fetch(db,'AVERAGE','-r',"240",
57 '-s', repr(start), '-e', repr(end))
58 vallist = [0] # start with zero, in case no vals returned, get zero as answer
59 for tup2 in tup[2]:
60 val = tup2[0]
61 if val:
62 vallist.append(val)
63 day_groups[group] = sum(vallist)
64
65 day_groups_sorted = sorted(day_groups, key=day_groups.get, reverse=False)
66 topgroups_day=day_groups_sorted[-NUMGROUPS:]
67
68 ### end block 'top N' group list ###
69
70 ### block generating plots ###
71
72 commonargs = ['--imgformat', 'PNG',
73 '--legend-position=east', '--legend-direction=bottomup']
74
75 sizeargs = { 'small' : [ '--width', '195', '--height', '125' ],
76 'large' : [ '--width', '850', '--height', '500' ]
77 }
78 def doplot(grouplist, dbtype, psize, timetag):
79
80 defs = list()
81 plots = list()
82
83 data_defs = list()
84 plot_defs = list()
85
86 for group in (grouplist + ['total']):
87 data_defs.append('DEF:'+group+'='+DATADIR+group+'.'+dbtype+'.rrd:'+dbtype+':AVERAGE')
88
89 for idx in range(len(grouplist)):
90 group = grouplist[idx]
91 pdefstr = 'AREA' ':' + group + colors[idx] + ':' + group
92 pdefstr = pdefstr + '\\n'
93 if idx > 0:
94 pdefstr = pdefstr + ':STACK'
95 plot_defs.append(pdefstr)
96
97 plot_defs.append('LINE:total#000000:total')
98
99 pargs = [ PLOTDIR + dbtype + '-' + timetag + '-' + psize + '.png'] + \
100 commonargs + sizeargs[psize] + timeargs[timetag] + \
101 data_defs + plot_defs
102 rrdtool.graph( *pargs )
103
104 def doplot_wait(grouplist, dbtype, psize, timetag):
105
106 defs = list()
107 plots = list()
108
109 data_defs = list()
110 plot_defs = list()
111
112 for group in (grouplist + ['rollover']):
113 data_defs.append('DEF:'+group+'='+DATADIR+group+'.'+dbtype+'.rrd:'+dbtype+':AVERAGE')
114
115 for idx in range(len(grouplist)):
116 group = grouplist[idx]
117 pdefstr = 'LINE3' ':' + group + colors[idx] + ':' + group
118 pdefstr = pdefstr + '\\n'
119 plot_defs.append(pdefstr)
120
121 plot_defs.append('LINE2:rollover#000000:rollover')
122
123 pargs = [ PLOTDIR + dbtype + '-' + timetag + '-' + psize + '.png'] + \
124 ['--slope-mode', '-o'] + \
125 commonargs + sizeargs[psize] + timeargs[timetag] + \
126 data_defs + plot_defs
127 rrdtool.graph( *pargs )
128
129 for dbtype in ['queued', 'running', 'waittime']:
130 for psize in ['small', 'large']:
131 for timetag in ['hr', 'day']:
132 if dbtype == 'waittime':
133 doplot_wait(topgroups_day, dbtype, psize, timetag)
134 else:
135 doplot(topgroups_day, dbtype, psize, timetag)
136
137 import time
138 now=int(time.mktime(time.localtime()))
139 end = (now / 7200 ) * 7200
140 start = end - (86400*35) + 7200
141
142 ### block finding 'top N' group list ###
143
144 month_groups = dict() # structure day_group[groupname] = total of hourly average
145
146 for db in running_files:
147 group = db[len(DATADIR):db.find('.running.rrd')]
148 if group == "total":
149 continue
150 tup = rrdtool.fetch(db,'AVERAGE','-r',"7200",
151 '-s', repr(start), '-e', repr(end))
152 vallist = [0] # start with zero, in case no vals returned, get zero as answer
153 for tup2 in tup[2]:
154 val = tup2[0]
155 if val:
156 vallist.append(val)
157 month_groups[group] = sum(vallist)
158
159 month_groups_sorted = sorted(month_groups, key=month_groups.get, reverse=False)
160 topgroups_month=month_groups_sorted[-NUMGROUPS:]
161
162 ### end block 'top N' group list ###
163
164 for dbtype in ['queued', 'running', 'waittime']:
165 for psize in ['small', 'large']:
166 for timetag in ['week', 'month']:
167 if dbtype == 'waittime':
168 doplot_wait(topgroups_month, dbtype, psize, timetag)
169 else:
170 doplot(topgroups_month, dbtype, psize, timetag)
171 import sys
172 sys.exit(0)
173
174 pfnam = os.environ['HOME']+'/tmp/run.plt'
175
176 infh = open(os.environ['HOME']+'/tmp/groups.rdata','r')
177 datablocks = infh.read().split('\n\n')
178
179 countslist = list() # holds one rundict per timestamp
180 timestringlist = list() # same order as countslist
181
182 for block in datablocks[:-1]:
183 lines = block.split('\n')
184 timestr = lines[0] ; timestringlist.append(timestr)
185 rundict = dict() # format rundict['vo'] = number_running
186 for l in lines[1:]:
187 vo, num = eval(l, {}, {})
188 rundict[vo] = num
189 countslist.append(rundict)
190
191 def avgdata(vals, tstamps, step):
192 # step must be odd
193 # lengths of vals and tstamps must be identical, and
194 # also must be integer multiple of step
195
196 newvals = list()
197 newstamps = list()
198 if len(vals) % step != 0:
199 print "length of input arrays must be multiple of step size"
200 sys.exit(0)
201 optr = 0
202 while optr < len(vals):
203 print optr
204 print step
205 print vals[optr:optr+step]
206
207 newvals.append(sum(vals[optr:optr+step]))
208 newstamps.append(tstamps[optr + (fact/2)])
209 optr += fact
210 return newvals, newstamps
211
212 sl1 = countslist[-75:]
213 sl2 = timestringlist[-75:]
214
215 hrtuple_small = (
216 sl1, sl2,
217 [
218 # 'set terminal x11 size 1000,535\n',
219 'set terminal png size 365,201 font "arial,8"\n',
220 'set key autotitle columnheader\n',
221 'set xdata time\n',
222 'set timefmt "%b-%d-%Y-%H:%M:%S"\n',
223 'set output "/user/templon/public_html/run-hr-small.png"\n',
224 '# time range must be in same format as data file\n',
225 'set grid xtics mxtics ytics mytics front\n',
226 'set yrange [0 : *]\n',
227 'set auto fix\n'
228 'set key outside samplen 1\n',
229 'set format x "%H:%M"\n',
230 'set xtics 3600\n',
231 'set mxtics 6\n',
232 'set mytics 2\n'
233 ],
234 os.environ['HOME']+'/tmp/myd_hr_s'
235 )
236
237 hrtuple_large = (
238 sl1, sl2,
239 [
240 'set terminal png size 1000,535 font "arial,10"\n',
241 'set output "/user/templon/public_html/run-hr-large.png"\n',
242 'set format x "%H:%M"\n',
243 'set xtics 3600\n',
244 'set mxtics 6\n',
245 'set mytics 4\n'
246 'set key autotitle columnheader\n',
247 'set xdata time\n',
248 'set timefmt "%b-%d-%Y-%H:%M:%S"\n',
249 '# time range must be in same format as data file\n',
250 'set grid xtics mxtics ytics mytics front\n',
251 'set yrange [0 : *]\n',
252 'set auto fix\n'
253 'set key outside samplen 1\n',
254 ],
255 os.environ['HOME']+'/tmp/myd_hr_l'
256 )
257
258 sl3 = countslist[-720:]
259 sl4 = timestringlist[-720:]
260
261 daytuple_small = (
262 sl3, sl4,
263 [
264 'set terminal png size 365,201 font "arial,8"\n',
265 'set output "/user/templon/public_html/run-day-small.png"\n',
266 'set format x "%H:%M"\n',
267 'set xtics 21600\n',
268 'set mxtics 3\n'
269 ],
270 os.environ['HOME']+'/tmp/myd_day_s'
271 )
272
273 daytuple_large = (
274 sl3, sl4,
275 [
276 'set terminal png size 1000,535 font "arial,10"\n',
277 'set output "/user/templon/public_html/run-day-large.png"\n',
278 'set format x "%H:%M"\n',
279 'set xtics 7200\n',
280 'set mxtics 4\n'
281 ],
282 os.environ['HOME']+'/tmp/myd_day_l'
283 )
284
285 sl5 = countslist[-5760:]
286 sl6 = timestringlist[-5760:]
287
288 weektuple_small = (
289 sl5[::5], sl6[::5],
290 [
291 'set terminal png size 365,201 font "arial,8"\n',
292 'set output "/user/templon/public_html/run-week-small.png"\n',
293 'set format x "%a"\n',
294 'set xtics 86400\n',
295 'set nomxtics \n'
296 ],
297 os.environ['HOME']+'/tmp/myd_week_s'
298 )
299
300 # sl7, sl8 = avgdata(sl5, sl6, 5)
301
302 weektuple_large = (
303 sl5[::2], sl6[::2],
304 [
305 'set terminal png size 1000,535 font "arial,10"\n',
306 'set output "/user/templon/public_html/run-week-large.png"\n',
307 'set format x "%a"\n',
308 'set xtics 86400\n',
309 'set mxtics 3\n'
310 ],
311 os.environ['HOME']+'/tmp/myd_week_l'
312 )
313
314 sl10 = countslist[-22300:]
315 sl11 = timestringlist[-22300:]
316
317 # sl12, sl13 = avgdata(sl10, sl11, 100)
318
319 monthtuple_small = (
320 sl10[::100], sl11[::100],
321 [
322 'set terminal png size 365,201 font "arial,8"\n',
323 'set output "/user/templon/public_html/run-month-small.png"\n',
324 'set format x "%m/%d"\n',
325 'set xtics 604800\n',
326 'set nomxtics \n'
327 ],
328 os.environ['HOME']+'/tmp/myd_month_s'
329 )
330
331 # sl14, sl15 = avgdata(sl10, sl11, 20)
332
333 monthtuple_large = (
334 sl10[::20], sl11[::20],
335 [
336 'set terminal png size 1000,535 font "arial,10"\n',
337 'set output "/user/templon/public_html/run-month-large.png"\n',
338 'set format x "%m/%d"\n',
339 'set xtics 604800\n',
340 'set mxtics 7\n'
341 ],
342 os.environ['HOME']+'/tmp/myd_month_l'
343 )
344
345
346 # do it again but for queued jobs
347
348 infh = open(os.environ['HOME']+'/tmp/groups.qdata','r')
349 datablocks = infh.read().split('\n\n')
350
351 countslist = list() # holds one rundict per timestamp
352 timestringlist = list() # same order as countslist
353
354 for block in datablocks[:-1]:
355 lines = block.split('\n')
356 timestr = lines[0] ; timestringlist.append(timestr)
357 rundict = dict() # format rundict['vo'] = number_running
358 for l in lines[1:]:
359 vo, num = eval(l, {}, {})
360 rundict[vo] = num
361 countslist.append(rundict)
362
363 qhrtuple_small = (
364 countslist[-75:],
365 timestringlist[-75:],
366 [
367 # 'set terminal x11 size 1000,535\n',
368 'set terminal png size 365,201 font "arial,8"\n',
369 'set key autotitle columnheader\n',
370 'set xdata time\n',
371 'set timefmt "%b-%d-%Y-%H:%M:%S"\n',
372 'set output "/user/templon/public_html/queued-hr-small.png"\n',
373 '# time range must be in same format as data file\n',
374 'set grid xtics mxtics ytics mytics front\n',
375 'set yrange [0 : *]\n',
376 'set auto fix\n'
377 'set key outside samplen 1\n',
378 'set format x "%H:%M"\n',
379 'set xtics 3600\n',
380 'set mxtics 6\n',
381 'set mytics 3\n'
382 ],
383 os.environ['HOME']+'/tmp/myqd_hr'
384 )
385
386 qhrtuple_large = (
387 countslist[-75:],
388 timestringlist[-75:],
389 [
390 'set terminal png size 1000,535 font "arial,10"\n',
391 'set output "/user/templon/public_html/queued-hr-large.png"\n',
392 'set format x "%H:%M"\n',
393 'set xtics 3600\n',
394 'set mxtics 6\n',
395 'set mytics 3\n'
396 'set key autotitle columnheader\n',
397 'set xdata time\n',
398 'set timefmt "%b-%d-%Y-%H:%M:%S"\n',
399 '# time range must be in same format as data file\n',
400 'set grid xtics mxtics ytics mytics front\n',
401 # 'set yrange [0 : 2600]\n',
402 'set auto fix\n'
403 'set key outside samplen 1\n',
404 ],
405 os.environ['HOME']+'/tmp/myqd_hr'
406 )
407
408 qdaytuple_small = (
409 countslist[-720:],
410 timestringlist[-720:],
411 [
412 'set terminal png size 365,201 font "arial,8"\n',
413 'set output "/user/templon/public_html/queued-day-small.png"\n',
414 'set format x "%H:%M"\n',
415 'set xtics 21600\n',
416 'set mxtics 3\n'
417 ],
418 os.environ['HOME']+'/tmp/myqd_day'
419 )
420
421 qdaytuple_large = (
422 countslist[-720:],
423 timestringlist[-720:],
424 [
425 'set terminal png size 1000,535 font "arial,10"\n',
426 'set output "/user/templon/public_html/queued-day-large.png"\n',
427 'set format x "%H:%M"\n',
428 'set xtics 7200\n',
429 'set mxtics 4\n'
430 ],
431 os.environ['HOME']+'/tmp/myqd_day'
432 )
433
434
435 plottuple_list = [ hrtuple_small, hrtuple_large, daytuple_small,
436 daytuple_large, weektuple_small, weektuple_large,
437 monthtuple_small, monthtuple_large,
438 qhrtuple_small, qhrtuple_large,
439 qdaytuple_small, qdaytuple_large
440 ]
441
442 plotfileh = open(pfnam,'w')
443
444 def emit_plotfile(pfh, plotslist):
445 # pfh is handle to plot file object
446 # first make the right data structure: rundata['vo']['timestamp'] = number
447 rundata = dict() # format rundata['vo']['timestamp'] = number
448 tsl = plotslist[1]
449 ctl = plotslist[0]
450 for i in range(len(tsl)):
451 for vo in ctl[i].keys():
452 if vo not in rundata.keys():
453 rundata[vo] = dict()
454 rundata[vo][tsl[i]] = ctl[i][vo]
455
456 sums = dict()
457 for k in rundata.keys():
458 sums[k] = sum(rundata[k].values())
459
460 vos_by_jobs = sorted(sums, key=sums.get, reverse=False)
461
462 topvos = vos_by_jobs[-NUMGROUPS:]
463
464 dfh = open(plotslist[3], 'w')
465 dfh.write('timestamp ')
466 for k in topvos:
467 dfh.write("%8s " % k)
468 dfh.write('\n')
469 for t in tsl:
470 dfh.write(t+ " ")
471 for k in topvos:
472 if t in rundata[k].keys():
473 val = rundata[k][t]
474 else:
475 val = 0
476 dfh.write("%4d " % val)
477 dfh.write('\n')
478 dfh.close()
479
480 import glob
481 import os
482
483 gplines = plotslist[2]
484
485 # gplines.append('set terminal png size 335,201\n')
486
487 ctr = 1
488 for vo in reversed(topvos):
489 if ctr == 1:
490 pstr = 'plot '
491 datafnam = plotslist[3]
492 pstr += '"' + datafnam + '" using 1:($' + repr(topvos.index(vo)+2)
493 for k2 in range(topvos.index(vo)+1, 1, -1):
494 pstr += '+$' + repr(k2)
495 pstr += ') t "' + vo + '"'
496 if ctr < len(topvos):
497 pstr += ' w boxes fs solid,'
498 ctr += 1
499 else:
500 pstr += ' w boxes fs solid'
501 gplines.append(pstr + '\n')
502
503 for l in gplines:
504 pfh.write(l)
505
506 # gplines.append('set terminal png size 1000,535\n')
507 # gplines.append('set output "/user/templon/public_html/wait-hr-large.png"\n')
508 # gplines.append('set format x "%H:%M"\n')
509 # gplines.append('set format y "%.0te+%S"\n')
510 # gplines.append('set xtics 3600\n')
511
512 # ctr = 1
513 # for vo in volist:
514 # if ctr == 1:
515 # pstr = 'plot '
516 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
517 # pstr += '"< tail -75 ' + datafnam + '" using 1:3' + \
518 # ' title "' + vo + '"'
519 # if ctr < len(volist):
520 # pstr += ' with lines lw 3,'
521 # ctr += 1
522 # else:
523 # pstr += ' with lines lw 1'
524 # gplines.append(pstr + '\n')
525
526 # gplines.append('set terminal png size 335,201\n')
527 # gplines.append('set output "/user/templon/public_html/wait-day-small.png"\n')
528 # gplines.append('set format x "%H:%M"\n')
529 # gplines.append('set format y "%.0te+%S"\n')
530 # gplines.append('set xtics 21600\n')
531 # gplines.append('set mxtics 3\n')
532
533 # ctr = 1
534 # for vo in volist:
535 # if ctr == 1:
536 # pstr = 'plot '
537 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
538 # pstr += '"< tail -720 ' + datafnam + '" using 1:3' + \
539 # ' title "' + vo + '"'
540 # if ctr < len(volist):
541 # pstr += ' with lines lw 3,'
542 # ctr += 1
543 # else:
544 # pstr += ' with lines lw 1'
545 # gplines.append(pstr + '\n')
546
547 # gplines.append('set terminal png size 1000,535\n')
548 # gplines.append('set output "/user/templon/public_html/wait-day-large.png"\n')
549 # gplines.append('set format x "%H:%M"\n')
550 # gplines.append('set format y "%.0te+%S"\n')
551 # gplines.append('set xtics 7200\n')
552 # gplines.append('set mxtics 4\n')
553
554 # ctr = 1
555 # for vo in volist:
556 # if ctr == 1:
557 # pstr = 'plot '
558 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
559 # pstr += '"< tail -720 ' + datafnam + '" using 1:3' + \
560 # ' title "' + vo + '"'
561 # if ctr < len(volist):
562 # pstr += ' with lines lw 3,'
563 # ctr += 1
564 # else:
565 # pstr += ' with lines lw 1'
566 # gplines.append(pstr + '\n')
567
568 # gplines.append('set terminal png size 335,201\n')
569 # gplines.append('set output "/user/templon/public_html/wait-week-small.png"\n')
570 # gplines.append('set format x "%a"\n')
571 # gplines.append('set format y "%.0te+%S"\n')
572 # gplines.append('set xtics 86400\n')
573 # gplines.append('set mxtics 3\n')
574
575 # ctr = 1
576 # for vo in volist:
577 # if ctr == 1:
578 # pstr = 'plot '
579 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
580 # pstr += '"< tail -5760 ' + datafnam + '" using 1:3' + \
581 # ' title "' + vo + '"'
582 # if ctr < len(volist):
583 # pstr += ' with lines lw 3,'
584 # ctr += 1
585 # else:
586 # pstr += ' with lines lw 1'
587 # gplines.append(pstr + '\n')
588
589 # gplines.append('set terminal png size 1000,535\n')
590 # gplines.append('set output "/user/templon/public_html/wait-week-large.png"\n')
591 # gplines.append('set format x "%a %H:%M"\n')
592 # gplines.append('set format y "%.0te+%S"\n')
593 # gplines.append('set xtics 86400\n')
594 # gplines.append('set mxtics 3\n')
595
596 # ctr = 1
597 # for vo in volist:
598 # if ctr == 1:
599 # pstr = 'plot '
600 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
601 # pstr += '"< tail -5760 ' + datafnam + '" using 1:3' + \
602 # ' title "' + vo + '"'
603 # if ctr < len(volist):
604 # pstr += ' with lines lw 3,'
605 # ctr += 1
606 # else:
607 # pstr += ' with lines lw 1'
608
609 for itm in plottuple_list:
610 emit_plotfile(plotfileh, itm)
611
612 plotfileh.close()
613 os.system(os.environ['HOME']+'/bin/gnuplot ' + pfnam)

Properties

Name Value
svn:executable *
svn:keywords Id URL

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28