/[pdpsoft]/nl.nikhef.ndpf.groupviews/trunk/ndpf-gv-mkplots
ViewVC logotype

Contents of /nl.nikhef.ndpf.groupviews/trunk/ndpf-gv-mkplots

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2380 - (show annotations) (download)
Sun Jul 24 17:41:06 2011 UTC (10 years, 10 months ago) by templon
File size: 18001 byte(s)
change legend formatting for readability

1 #! /usr/bin/env python
2 # $Id$
3 # Source: $URL$
4 # J. A. Templon, NIKHEF/PDP 2011
5
6 import os
7
8 NUMGROUPS=8
9 DATADIR=os.environ['HOME'] + '/ndpfdata/'
10 PLOTDIR=os.environ['HOME'] + '/public_html/'
11 # 8 class qualitative paired color scheme
12
13 colors = [ "#A6CEE3", "#1F77B4", "#B2DF8A", "#33A02C",
14 "#FB9A99", "#E31A1C", "#FDBF6F", "#FF7F00" ]
15
16 colors.reverse()
17
18 timeargs = { 'hr' : [ '-s', 'n-3h30min', '-e', 'n' ],
19 'day' : [ '-s', 'n-29h', '-e', 'n' ],
20 'week' : [ '-s', 'n-8d', '-e', 'n' ],
21 'month' : [ '-s', 'n-35d', '-e', 'n' ]
22 }
23
24 # for reference : resolutions of the RRAs
25 # 120s x 240 points = 8 hr
26 # 240s x 900 points = 60 hr
27 # 1800s x 768 points = 192 hr (16 days)
28 # 7200s (2 hrs) x 840 points = 70 days
29 # 86400s (1 day) x 3650 points = 10 years
30
31 import rrdtool
32
33 # first need to find four "top eight" lists
34 # one for the past day, which will be used for the hour / day page
35 # one for the past week
36 # one for the past month
37 # one for the past year
38 # base it on running jobs
39
40 import time
41 now=int(time.mktime(time.localtime()))
42 end = (now / 240) * 240
43 start = end - (3600 * 30) + 240
44
45 import glob # number of running jobs
46
47 ### block finding 'top N' group list ###
48
49 day_groups = dict() # structure day_group[groupname] = total of hourly average
50
51 running_files = glob.glob(DATADIR+'*.running.rrd')
52 for db in running_files:
53 group = db[len(DATADIR):db.find('.running.rrd')]
54 if group == "total":
55 continue
56 tup = rrdtool.fetch(db,'AVERAGE','-r',"240",
57 '-s', repr(start), '-e', repr(end))
58 vallist = [0] # start with zero, in case no vals returned, get zero as answer
59 for tup2 in tup[2]:
60 val = tup2[0]
61 if val:
62 vallist.append(val)
63 day_groups[group] = sum(vallist)
64
65 day_groups_sorted = sorted(day_groups, key=day_groups.get, reverse=False)
66 topgroups_day=day_groups_sorted[-NUMGROUPS:]
67
68 ### end block 'top N' group list ###
69
70 ### block generating plots ###
71
72 commonargs = ['--imgformat', 'PNG',
73 '--legend-position=east', '--legend-direction=bottomup']
74 sizeargs = { 'small' : [ '--width', '200', '--height', '125' ],
75 'large' : [ '--width', '900', '--height', '500' ]
76 }
77 def doplot(grouplist, dbtype, psize, timetag):
78
79 defs = list()
80 plots = list()
81
82 data_defs = list()
83 plot_defs = list()
84
85 for group in (grouplist + ['total']):
86 data_defs.append('DEF:'+group+'='+DATADIR+group+'.'+dbtype+'.rrd:'+dbtype+':AVERAGE')
87
88 for idx in range(len(grouplist)):
89 group = grouplist[idx]
90 pdefstr = 'AREA' ':' + group + colors[idx] + ':' + group
91 pdefstr = pdefstr + '\\n'
92 if idx > 0:
93 pdefstr = pdefstr + ':STACK'
94 plot_defs.append(pdefstr)
95
96 plot_defs.append('LINE:total#000000:total')
97
98 pargs = [ PLOTDIR + dbtype + '-' + timetag + '-' + psize + '.png'] + \
99 commonargs + sizeargs[psize] + timeargs[timetag] + \
100 data_defs + plot_defs
101 rrdtool.graph( *pargs )
102
103 def doplot_wait(grouplist, dbtype, psize, timetag):
104
105 defs = list()
106 plots = list()
107
108 data_defs = list()
109 plot_defs = list()
110
111 for group in (grouplist + ['rollover']):
112 data_defs.append('DEF:'+group+'='+DATADIR+group+'.'+dbtype+'.rrd:'+dbtype+':AVERAGE')
113
114 for idx in range(len(grouplist)):
115 group = grouplist[idx]
116 pdefstr = 'LINE3' ':' + group + colors[idx] + ':' + group
117 pdefstr = pdefstr + '\\n'
118 plot_defs.append(pdefstr)
119
120 plot_defs.append('LINE2:rollover#000000:rollover')
121
122 pargs = [ PLOTDIR + dbtype + '-' + timetag + '-' + psize + '.png'] + \
123 ['--slope-mode', '-o'] + \
124 commonargs + sizeargs[psize] + timeargs[timetag] + \
125 data_defs + plot_defs
126 rrdtool.graph( *pargs )
127
128 for dbtype in ['queued', 'running', 'waittime']:
129 for psize in ['small', 'large']:
130 for timetag in ['hr', 'day']:
131 if dbtype == 'waittime':
132 doplot_wait(topgroups_day, dbtype, psize, timetag)
133 else:
134 doplot(topgroups_day, dbtype, psize, timetag)
135
136 import time
137 now=int(time.mktime(time.localtime()))
138 end = (now / 7200 ) * 7200
139 start = end - (86400*35) + 7200
140
141 ### block finding 'top N' group list ###
142
143 month_groups = dict() # structure day_group[groupname] = total of hourly average
144
145 for db in running_files:
146 group = db[len(DATADIR):db.find('.running.rrd')]
147 if group == "total":
148 continue
149 tup = rrdtool.fetch(db,'AVERAGE','-r',"7200",
150 '-s', repr(start), '-e', repr(end))
151 vallist = [0] # start with zero, in case no vals returned, get zero as answer
152 for tup2 in tup[2]:
153 val = tup2[0]
154 if val:
155 vallist.append(val)
156 month_groups[group] = sum(vallist)
157
158 month_groups_sorted = sorted(month_groups, key=month_groups.get, reverse=False)
159 topgroups_month=month_groups_sorted[-NUMGROUPS:]
160
161 ### end block 'top N' group list ###
162
163 for dbtype in ['queued', 'running', 'waittime']:
164 for psize in ['small', 'large']:
165 for timetag in ['week', 'month']:
166 if dbtype == 'waittime':
167 doplot_wait(topgroups_month, dbtype, psize, timetag)
168 else:
169 doplot(topgroups_month, dbtype, psize, timetag)
170 import sys
171 sys.exit(0)
172
173 pfnam = os.environ['HOME']+'/tmp/run.plt'
174
175 infh = open(os.environ['HOME']+'/tmp/groups.rdata','r')
176 datablocks = infh.read().split('\n\n')
177
178 countslist = list() # holds one rundict per timestamp
179 timestringlist = list() # same order as countslist
180
181 for block in datablocks[:-1]:
182 lines = block.split('\n')
183 timestr = lines[0] ; timestringlist.append(timestr)
184 rundict = dict() # format rundict['vo'] = number_running
185 for l in lines[1:]:
186 vo, num = eval(l, {}, {})
187 rundict[vo] = num
188 countslist.append(rundict)
189
190 def avgdata(vals, tstamps, step):
191 # step must be odd
192 # lengths of vals and tstamps must be identical, and
193 # also must be integer multiple of step
194
195 newvals = list()
196 newstamps = list()
197 if len(vals) % step != 0:
198 print "length of input arrays must be multiple of step size"
199 sys.exit(0)
200 optr = 0
201 while optr < len(vals):
202 print optr
203 print step
204 print vals[optr:optr+step]
205
206 newvals.append(sum(vals[optr:optr+step]))
207 newstamps.append(tstamps[optr + (fact/2)])
208 optr += fact
209 return newvals, newstamps
210
211 sl1 = countslist[-75:]
212 sl2 = timestringlist[-75:]
213
214 hrtuple_small = (
215 sl1, sl2,
216 [
217 # 'set terminal x11 size 1000,535\n',
218 'set terminal png size 365,201 font "arial,8"\n',
219 'set key autotitle columnheader\n',
220 'set xdata time\n',
221 'set timefmt "%b-%d-%Y-%H:%M:%S"\n',
222 'set output "/user/templon/public_html/run-hr-small.png"\n',
223 '# time range must be in same format as data file\n',
224 'set grid xtics mxtics ytics mytics front\n',
225 'set yrange [0 : *]\n',
226 'set auto fix\n'
227 'set key outside samplen 1\n',
228 'set format x "%H:%M"\n',
229 'set xtics 3600\n',
230 'set mxtics 6\n',
231 'set mytics 2\n'
232 ],
233 os.environ['HOME']+'/tmp/myd_hr_s'
234 )
235
236 hrtuple_large = (
237 sl1, sl2,
238 [
239 'set terminal png size 1000,535 font "arial,10"\n',
240 'set output "/user/templon/public_html/run-hr-large.png"\n',
241 'set format x "%H:%M"\n',
242 'set xtics 3600\n',
243 'set mxtics 6\n',
244 'set mytics 4\n'
245 'set key autotitle columnheader\n',
246 'set xdata time\n',
247 'set timefmt "%b-%d-%Y-%H:%M:%S"\n',
248 '# time range must be in same format as data file\n',
249 'set grid xtics mxtics ytics mytics front\n',
250 'set yrange [0 : *]\n',
251 'set auto fix\n'
252 'set key outside samplen 1\n',
253 ],
254 os.environ['HOME']+'/tmp/myd_hr_l'
255 )
256
257 sl3 = countslist[-720:]
258 sl4 = timestringlist[-720:]
259
260 daytuple_small = (
261 sl3, sl4,
262 [
263 'set terminal png size 365,201 font "arial,8"\n',
264 'set output "/user/templon/public_html/run-day-small.png"\n',
265 'set format x "%H:%M"\n',
266 'set xtics 21600\n',
267 'set mxtics 3\n'
268 ],
269 os.environ['HOME']+'/tmp/myd_day_s'
270 )
271
272 daytuple_large = (
273 sl3, sl4,
274 [
275 'set terminal png size 1000,535 font "arial,10"\n',
276 'set output "/user/templon/public_html/run-day-large.png"\n',
277 'set format x "%H:%M"\n',
278 'set xtics 7200\n',
279 'set mxtics 4\n'
280 ],
281 os.environ['HOME']+'/tmp/myd_day_l'
282 )
283
284 sl5 = countslist[-5760:]
285 sl6 = timestringlist[-5760:]
286
287 weektuple_small = (
288 sl5[::5], sl6[::5],
289 [
290 'set terminal png size 365,201 font "arial,8"\n',
291 'set output "/user/templon/public_html/run-week-small.png"\n',
292 'set format x "%a"\n',
293 'set xtics 86400\n',
294 'set nomxtics \n'
295 ],
296 os.environ['HOME']+'/tmp/myd_week_s'
297 )
298
299 # sl7, sl8 = avgdata(sl5, sl6, 5)
300
301 weektuple_large = (
302 sl5[::2], sl6[::2],
303 [
304 'set terminal png size 1000,535 font "arial,10"\n',
305 'set output "/user/templon/public_html/run-week-large.png"\n',
306 'set format x "%a"\n',
307 'set xtics 86400\n',
308 'set mxtics 3\n'
309 ],
310 os.environ['HOME']+'/tmp/myd_week_l'
311 )
312
313 sl10 = countslist[-22300:]
314 sl11 = timestringlist[-22300:]
315
316 # sl12, sl13 = avgdata(sl10, sl11, 100)
317
318 monthtuple_small = (
319 sl10[::100], sl11[::100],
320 [
321 'set terminal png size 365,201 font "arial,8"\n',
322 'set output "/user/templon/public_html/run-month-small.png"\n',
323 'set format x "%m/%d"\n',
324 'set xtics 604800\n',
325 'set nomxtics \n'
326 ],
327 os.environ['HOME']+'/tmp/myd_month_s'
328 )
329
330 # sl14, sl15 = avgdata(sl10, sl11, 20)
331
332 monthtuple_large = (
333 sl10[::20], sl11[::20],
334 [
335 'set terminal png size 1000,535 font "arial,10"\n',
336 'set output "/user/templon/public_html/run-month-large.png"\n',
337 'set format x "%m/%d"\n',
338 'set xtics 604800\n',
339 'set mxtics 7\n'
340 ],
341 os.environ['HOME']+'/tmp/myd_month_l'
342 )
343
344
345 # do it again but for queued jobs
346
347 infh = open(os.environ['HOME']+'/tmp/groups.qdata','r')
348 datablocks = infh.read().split('\n\n')
349
350 countslist = list() # holds one rundict per timestamp
351 timestringlist = list() # same order as countslist
352
353 for block in datablocks[:-1]:
354 lines = block.split('\n')
355 timestr = lines[0] ; timestringlist.append(timestr)
356 rundict = dict() # format rundict['vo'] = number_running
357 for l in lines[1:]:
358 vo, num = eval(l, {}, {})
359 rundict[vo] = num
360 countslist.append(rundict)
361
362 qhrtuple_small = (
363 countslist[-75:],
364 timestringlist[-75:],
365 [
366 # 'set terminal x11 size 1000,535\n',
367 'set terminal png size 365,201 font "arial,8"\n',
368 'set key autotitle columnheader\n',
369 'set xdata time\n',
370 'set timefmt "%b-%d-%Y-%H:%M:%S"\n',
371 'set output "/user/templon/public_html/queued-hr-small.png"\n',
372 '# time range must be in same format as data file\n',
373 'set grid xtics mxtics ytics mytics front\n',
374 'set yrange [0 : *]\n',
375 'set auto fix\n'
376 'set key outside samplen 1\n',
377 'set format x "%H:%M"\n',
378 'set xtics 3600\n',
379 'set mxtics 6\n',
380 'set mytics 3\n'
381 ],
382 os.environ['HOME']+'/tmp/myqd_hr'
383 )
384
385 qhrtuple_large = (
386 countslist[-75:],
387 timestringlist[-75:],
388 [
389 'set terminal png size 1000,535 font "arial,10"\n',
390 'set output "/user/templon/public_html/queued-hr-large.png"\n',
391 'set format x "%H:%M"\n',
392 'set xtics 3600\n',
393 'set mxtics 6\n',
394 'set mytics 3\n'
395 'set key autotitle columnheader\n',
396 'set xdata time\n',
397 'set timefmt "%b-%d-%Y-%H:%M:%S"\n',
398 '# time range must be in same format as data file\n',
399 'set grid xtics mxtics ytics mytics front\n',
400 # 'set yrange [0 : 2600]\n',
401 'set auto fix\n'
402 'set key outside samplen 1\n',
403 ],
404 os.environ['HOME']+'/tmp/myqd_hr'
405 )
406
407 qdaytuple_small = (
408 countslist[-720:],
409 timestringlist[-720:],
410 [
411 'set terminal png size 365,201 font "arial,8"\n',
412 'set output "/user/templon/public_html/queued-day-small.png"\n',
413 'set format x "%H:%M"\n',
414 'set xtics 21600\n',
415 'set mxtics 3\n'
416 ],
417 os.environ['HOME']+'/tmp/myqd_day'
418 )
419
420 qdaytuple_large = (
421 countslist[-720:],
422 timestringlist[-720:],
423 [
424 'set terminal png size 1000,535 font "arial,10"\n',
425 'set output "/user/templon/public_html/queued-day-large.png"\n',
426 'set format x "%H:%M"\n',
427 'set xtics 7200\n',
428 'set mxtics 4\n'
429 ],
430 os.environ['HOME']+'/tmp/myqd_day'
431 )
432
433
434 plottuple_list = [ hrtuple_small, hrtuple_large, daytuple_small,
435 daytuple_large, weektuple_small, weektuple_large,
436 monthtuple_small, monthtuple_large,
437 qhrtuple_small, qhrtuple_large,
438 qdaytuple_small, qdaytuple_large
439 ]
440
441 plotfileh = open(pfnam,'w')
442
443 def emit_plotfile(pfh, plotslist):
444 # pfh is handle to plot file object
445 # first make the right data structure: rundata['vo']['timestamp'] = number
446 rundata = dict() # format rundata['vo']['timestamp'] = number
447 tsl = plotslist[1]
448 ctl = plotslist[0]
449 for i in range(len(tsl)):
450 for vo in ctl[i].keys():
451 if vo not in rundata.keys():
452 rundata[vo] = dict()
453 rundata[vo][tsl[i]] = ctl[i][vo]
454
455 sums = dict()
456 for k in rundata.keys():
457 sums[k] = sum(rundata[k].values())
458
459 vos_by_jobs = sorted(sums, key=sums.get, reverse=False)
460
461 topvos = vos_by_jobs[-NUMGROUPS:]
462
463 dfh = open(plotslist[3], 'w')
464 dfh.write('timestamp ')
465 for k in topvos:
466 dfh.write("%8s " % k)
467 dfh.write('\n')
468 for t in tsl:
469 dfh.write(t+ " ")
470 for k in topvos:
471 if t in rundata[k].keys():
472 val = rundata[k][t]
473 else:
474 val = 0
475 dfh.write("%4d " % val)
476 dfh.write('\n')
477 dfh.close()
478
479 import glob
480 import os
481
482 gplines = plotslist[2]
483
484 # gplines.append('set terminal png size 335,201\n')
485
486 ctr = 1
487 for vo in reversed(topvos):
488 if ctr == 1:
489 pstr = 'plot '
490 datafnam = plotslist[3]
491 pstr += '"' + datafnam + '" using 1:($' + repr(topvos.index(vo)+2)
492 for k2 in range(topvos.index(vo)+1, 1, -1):
493 pstr += '+$' + repr(k2)
494 pstr += ') t "' + vo + '"'
495 if ctr < len(topvos):
496 pstr += ' w boxes fs solid,'
497 ctr += 1
498 else:
499 pstr += ' w boxes fs solid'
500 gplines.append(pstr + '\n')
501
502 for l in gplines:
503 pfh.write(l)
504
505 # gplines.append('set terminal png size 1000,535\n')
506 # gplines.append('set output "/user/templon/public_html/wait-hr-large.png"\n')
507 # gplines.append('set format x "%H:%M"\n')
508 # gplines.append('set format y "%.0te+%S"\n')
509 # gplines.append('set xtics 3600\n')
510
511 # ctr = 1
512 # for vo in volist:
513 # if ctr == 1:
514 # pstr = 'plot '
515 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
516 # pstr += '"< tail -75 ' + datafnam + '" using 1:3' + \
517 # ' title "' + vo + '"'
518 # if ctr < len(volist):
519 # pstr += ' with lines lw 3,'
520 # ctr += 1
521 # else:
522 # pstr += ' with lines lw 1'
523 # gplines.append(pstr + '\n')
524
525 # gplines.append('set terminal png size 335,201\n')
526 # gplines.append('set output "/user/templon/public_html/wait-day-small.png"\n')
527 # gplines.append('set format x "%H:%M"\n')
528 # gplines.append('set format y "%.0te+%S"\n')
529 # gplines.append('set xtics 21600\n')
530 # gplines.append('set mxtics 3\n')
531
532 # ctr = 1
533 # for vo in volist:
534 # if ctr == 1:
535 # pstr = 'plot '
536 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
537 # pstr += '"< tail -720 ' + datafnam + '" using 1:3' + \
538 # ' title "' + vo + '"'
539 # if ctr < len(volist):
540 # pstr += ' with lines lw 3,'
541 # ctr += 1
542 # else:
543 # pstr += ' with lines lw 1'
544 # gplines.append(pstr + '\n')
545
546 # gplines.append('set terminal png size 1000,535\n')
547 # gplines.append('set output "/user/templon/public_html/wait-day-large.png"\n')
548 # gplines.append('set format x "%H:%M"\n')
549 # gplines.append('set format y "%.0te+%S"\n')
550 # gplines.append('set xtics 7200\n')
551 # gplines.append('set mxtics 4\n')
552
553 # ctr = 1
554 # for vo in volist:
555 # if ctr == 1:
556 # pstr = 'plot '
557 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
558 # pstr += '"< tail -720 ' + datafnam + '" using 1:3' + \
559 # ' title "' + vo + '"'
560 # if ctr < len(volist):
561 # pstr += ' with lines lw 3,'
562 # ctr += 1
563 # else:
564 # pstr += ' with lines lw 1'
565 # gplines.append(pstr + '\n')
566
567 # gplines.append('set terminal png size 335,201\n')
568 # gplines.append('set output "/user/templon/public_html/wait-week-small.png"\n')
569 # gplines.append('set format x "%a"\n')
570 # gplines.append('set format y "%.0te+%S"\n')
571 # gplines.append('set xtics 86400\n')
572 # gplines.append('set mxtics 3\n')
573
574 # ctr = 1
575 # for vo in volist:
576 # if ctr == 1:
577 # pstr = 'plot '
578 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
579 # pstr += '"< tail -5760 ' + datafnam + '" using 1:3' + \
580 # ' title "' + vo + '"'
581 # if ctr < len(volist):
582 # pstr += ' with lines lw 3,'
583 # ctr += 1
584 # else:
585 # pstr += ' with lines lw 1'
586 # gplines.append(pstr + '\n')
587
588 # gplines.append('set terminal png size 1000,535\n')
589 # gplines.append('set output "/user/templon/public_html/wait-week-large.png"\n')
590 # gplines.append('set format x "%a %H:%M"\n')
591 # gplines.append('set format y "%.0te+%S"\n')
592 # gplines.append('set xtics 86400\n')
593 # gplines.append('set mxtics 3\n')
594
595 # ctr = 1
596 # for vo in volist:
597 # if ctr == 1:
598 # pstr = 'plot '
599 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
600 # pstr += '"< tail -5760 ' + datafnam + '" using 1:3' + \
601 # ' title "' + vo + '"'
602 # if ctr < len(volist):
603 # pstr += ' with lines lw 3,'
604 # ctr += 1
605 # else:
606 # pstr += ' with lines lw 1'
607
608 for itm in plottuple_list:
609 emit_plotfile(plotfileh, itm)
610
611 plotfileh.close()
612 os.system(os.environ['HOME']+'/bin/gnuplot ' + pfnam)

Properties

Name Value
svn:executable *
svn:keywords Id URL

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28