/[pdpsoft]/nl.nikhef.ndpf.groupviews/trunk/ndpf-gv-mkplots
ViewVC logotype

Annotation of /nl.nikhef.ndpf.groupviews/trunk/ndpf-gv-mkplots

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2377 - (hide annotations) (download)
Sun Jul 24 15:10:20 2011 UTC (10 years, 9 months ago) by templon
File size: 18102 byte(s)
put in Id and URL tags

1 templon 2355 #! /usr/bin/env python
2 templon 2377 # $Id$
3     # Source: $URL$
4     # J. A. Templon, NIKHEF/PDP 2011
5 templon 2355
6 templon 2357 import os
7    
8 templon 2355 NUMGROUPS=8
9 templon 2357 DATADIR=os.environ['HOME'] + '/ndpfdata/'
10 templon 2359 PLOTDIR=os.environ['HOME'] + '/public_html/'
11 templon 2358 # 8 class qualitative paired color scheme
12    
13     colors = [ "#A6CEE3", "#1F77B4", "#B2DF8A", "#33A02C",
14     "#FB9A99", "#E31A1C", "#FDBF6F", "#FF7F00" ]
15    
16     colors.reverse()
17    
18 templon 2372 timeargs = { 'hr' : [ '-s', 'n-3h30min', '-e', 'n' ],
19     'day' : [ '-s', 'n-29h', '-e', 'n' ],
20 templon 2366 'week' : [ '-s', 'n-8d', '-e', 'n' ],
21 templon 2370 'month' : [ '-s', 'n-35d', '-e', 'n' ]
22 templon 2366 }
23    
24 templon 2357 # for reference : resolutions of the RRAs
25 templon 2374 # 120s x 240 points = 8 hr
26     # 240s x 900 points = 60 hr
27     # 1800s x 768 points = 192 hr (16 days)
28     # 7200s (2 hrs) x 840 points = 70 days
29     # 86400s (1 day) x 3650 points = 10 years
30 templon 2355
31 templon 2356 import rrdtool
32    
33     # first need to find four "top eight" lists
34     # one for the past day, which will be used for the hour / day page
35     # one for the past week
36     # one for the past month
37     # one for the past year
38     # base it on running jobs
39    
40 templon 2357 import time
41     now=int(time.mktime(time.localtime()))
42 templon 2364 end = (now / 240) * 240
43 templon 2372 start = end - (3600 * 30) + 240
44 templon 2357
45 templon 2362 import glob # number of running jobs
46    
47     ### block finding 'top N' group list ###
48    
49 templon 2357 day_groups = dict() # structure day_group[groupname] = total of hourly average
50 templon 2362
51 templon 2357 running_files = glob.glob(DATADIR+'*.running.rrd')
52     for db in running_files:
53     group = db[len(DATADIR):db.find('.running.rrd')]
54     if group == "total":
55     continue
56 templon 2364 tup = rrdtool.fetch(db,'AVERAGE','-r',"240",
57 templon 2357 '-s', repr(start), '-e', repr(end))
58     vallist = [0] # start with zero, in case no vals returned, get zero as answer
59     for tup2 in tup[2]:
60     val = tup2[0]
61     if val:
62     vallist.append(val)
63     day_groups[group] = sum(vallist[:-1]) # chop weird last value from rrdtool
64    
65     day_groups_sorted = sorted(day_groups, key=day_groups.get, reverse=False)
66     topgroups_day=day_groups_sorted[-NUMGROUPS:]
67    
68 templon 2362 ### end block 'top N' group list ###
69 templon 2358
70 templon 2362 ### block generating plots ###
71 templon 2358
72 templon 2375 commonargs = ['--imgformat', 'PNG' ]
73     sizeargs = { 'small' : [ '--width', '285', '--height', '125' ],
74     'large' : [ '--width', '950', '--height', '500' ]
75     }
76 templon 2362 def doplot(grouplist, dbtype, psize, timetag):
77    
78     defs = list()
79     plots = list()
80 templon 2358
81 templon 2362 data_defs = list()
82     plot_defs = list()
83    
84     for group in (grouplist + ['total']):
85     data_defs.append('DEF:'+group+'='+DATADIR+group+'.'+dbtype+'.rrd:'+dbtype+':AVERAGE')
86    
87     for idx in range(len(grouplist)):
88     group = grouplist[idx]
89     pdefstr = 'AREA' ':' + group + colors[idx] + ':' + group
90     if idx < 5 and psize == 'small':
91     pdefstr = pdefstr + '\g'
92     if idx > 0:
93     pdefstr = pdefstr + ':STACK'
94     plot_defs.append(pdefstr)
95    
96     plot_defs.append('LINE:total#000000:total')
97    
98     pargs = [ PLOTDIR + dbtype + '-' + timetag + '-' + psize + '.png'] + \
99 templon 2375 commonargs + sizeargs[psize] + timeargs[timetag] + \
100     data_defs + plot_defs
101 templon 2362 rrdtool.graph( *pargs )
102    
103 templon 2365 def doplot_wait(grouplist, dbtype, psize, timetag):
104    
105     defs = list()
106     plots = list()
107    
108     data_defs = list()
109     plot_defs = list()
110    
111     for group in (grouplist + ['rollover']):
112     data_defs.append('DEF:'+group+'='+DATADIR+group+'.'+dbtype+'.rrd:'+dbtype+':AVERAGE')
113    
114     for idx in range(len(grouplist)):
115     group = grouplist[idx]
116 templon 2366 pdefstr = 'LINE3' ':' + group + colors[idx] + ':' + group
117 templon 2365 if idx < 5 and psize == 'small':
118     pdefstr = pdefstr + '\g'
119     plot_defs.append(pdefstr)
120    
121     plot_defs.append('LINE2:rollover#000000:rollover')
122    
123     pargs = [ PLOTDIR + dbtype + '-' + timetag + '-' + psize + '.png'] + \
124     ['--slope-mode', '-o'] + \
125 templon 2375 commonargs + sizeargs[psize] + timeargs[timetag] + \
126     data_defs + plot_defs
127 templon 2365 rrdtool.graph( *pargs )
128    
129     for dbtype in ['queued', 'running', 'waittime']:
130 templon 2362 for psize in ['small', 'large']:
131     for timetag in ['hr', 'day']:
132 templon 2365 if dbtype == 'waittime':
133     doplot_wait(topgroups_day, dbtype, psize, timetag)
134     else:
135     doplot(topgroups_day, dbtype, psize, timetag)
136 templon 2362
137 templon 2366 import time
138     now=int(time.mktime(time.localtime()))
139 templon 2369 end = (now / 7200 ) * 7200
140 templon 2372 start = end - (86400*35) + 7200
141 templon 2365
142 templon 2366 ### block finding 'top N' group list ###
143    
144     month_groups = dict() # structure day_group[groupname] = total of hourly average
145    
146     for db in running_files:
147     group = db[len(DATADIR):db.find('.running.rrd')]
148     if group == "total":
149     continue
150 templon 2376 tup = rrdtool.fetch(db,'AVERAGE','-r',"7200",
151 templon 2366 '-s', repr(start), '-e', repr(end))
152     vallist = [0] # start with zero, in case no vals returned, get zero as answer
153     for tup2 in tup[2]:
154     val = tup2[0]
155     if val:
156     vallist.append(val)
157     month_groups[group] = sum(vallist[:-1]) # chop weird last value from rrdtool
158    
159     month_groups_sorted = sorted(month_groups, key=month_groups.get, reverse=False)
160     topgroups_month=month_groups_sorted[-NUMGROUPS:]
161    
162     ### end block 'top N' group list ###
163    
164     for dbtype in ['queued', 'running', 'waittime']:
165     for psize in ['small', 'large']:
166     for timetag in ['week', 'month']:
167     if dbtype == 'waittime':
168     doplot_wait(topgroups_month, dbtype, psize, timetag)
169     else:
170     doplot(topgroups_month, dbtype, psize, timetag)
171 templon 2357 import sys
172     sys.exit(0)
173    
174 templon 2355 pfnam = os.environ['HOME']+'/tmp/run.plt'
175    
176     infh = open(os.environ['HOME']+'/tmp/groups.rdata','r')
177     datablocks = infh.read().split('\n\n')
178    
179     countslist = list() # holds one rundict per timestamp
180     timestringlist = list() # same order as countslist
181    
182     for block in datablocks[:-1]:
183     lines = block.split('\n')
184     timestr = lines[0] ; timestringlist.append(timestr)
185     rundict = dict() # format rundict['vo'] = number_running
186     for l in lines[1:]:
187     vo, num = eval(l, {}, {})
188     rundict[vo] = num
189     countslist.append(rundict)
190    
191     def avgdata(vals, tstamps, step):
192     # step must be odd
193     # lengths of vals and tstamps must be identical, and
194     # also must be integer multiple of step
195    
196     newvals = list()
197     newstamps = list()
198     if len(vals) % step != 0:
199     print "length of input arrays must be multiple of step size"
200     sys.exit(0)
201     optr = 0
202     while optr < len(vals):
203     print optr
204     print step
205     print vals[optr:optr+step]
206    
207     newvals.append(sum(vals[optr:optr+step]))
208     newstamps.append(tstamps[optr + (fact/2)])
209     optr += fact
210     return newvals, newstamps
211    
212     sl1 = countslist[-75:]
213     sl2 = timestringlist[-75:]
214    
215     hrtuple_small = (
216     sl1, sl2,
217     [
218     # 'set terminal x11 size 1000,535\n',
219     'set terminal png size 365,201 font "arial,8"\n',
220     'set key autotitle columnheader\n',
221     'set xdata time\n',
222     'set timefmt "%b-%d-%Y-%H:%M:%S"\n',
223     'set output "/user/templon/public_html/run-hr-small.png"\n',
224     '# time range must be in same format as data file\n',
225     'set grid xtics mxtics ytics mytics front\n',
226     'set yrange [0 : *]\n',
227     'set auto fix\n'
228     'set key outside samplen 1\n',
229     'set format x "%H:%M"\n',
230     'set xtics 3600\n',
231     'set mxtics 6\n',
232     'set mytics 2\n'
233     ],
234     os.environ['HOME']+'/tmp/myd_hr_s'
235     )
236    
237     hrtuple_large = (
238     sl1, sl2,
239     [
240     'set terminal png size 1000,535 font "arial,10"\n',
241     'set output "/user/templon/public_html/run-hr-large.png"\n',
242     'set format x "%H:%M"\n',
243     'set xtics 3600\n',
244     'set mxtics 6\n',
245     'set mytics 4\n'
246     'set key autotitle columnheader\n',
247     'set xdata time\n',
248     'set timefmt "%b-%d-%Y-%H:%M:%S"\n',
249     '# time range must be in same format as data file\n',
250     'set grid xtics mxtics ytics mytics front\n',
251     'set yrange [0 : *]\n',
252     'set auto fix\n'
253     'set key outside samplen 1\n',
254     ],
255     os.environ['HOME']+'/tmp/myd_hr_l'
256     )
257    
258     sl3 = countslist[-720:]
259     sl4 = timestringlist[-720:]
260    
261     daytuple_small = (
262     sl3, sl4,
263     [
264     'set terminal png size 365,201 font "arial,8"\n',
265     'set output "/user/templon/public_html/run-day-small.png"\n',
266     'set format x "%H:%M"\n',
267     'set xtics 21600\n',
268     'set mxtics 3\n'
269     ],
270     os.environ['HOME']+'/tmp/myd_day_s'
271     )
272    
273     daytuple_large = (
274     sl3, sl4,
275     [
276     'set terminal png size 1000,535 font "arial,10"\n',
277     'set output "/user/templon/public_html/run-day-large.png"\n',
278     'set format x "%H:%M"\n',
279     'set xtics 7200\n',
280     'set mxtics 4\n'
281     ],
282     os.environ['HOME']+'/tmp/myd_day_l'
283     )
284    
285     sl5 = countslist[-5760:]
286     sl6 = timestringlist[-5760:]
287    
288     weektuple_small = (
289     sl5[::5], sl6[::5],
290     [
291     'set terminal png size 365,201 font "arial,8"\n',
292     'set output "/user/templon/public_html/run-week-small.png"\n',
293     'set format x "%a"\n',
294     'set xtics 86400\n',
295     'set nomxtics \n'
296     ],
297     os.environ['HOME']+'/tmp/myd_week_s'
298     )
299    
300     # sl7, sl8 = avgdata(sl5, sl6, 5)
301    
302     weektuple_large = (
303     sl5[::2], sl6[::2],
304     [
305     'set terminal png size 1000,535 font "arial,10"\n',
306     'set output "/user/templon/public_html/run-week-large.png"\n',
307     'set format x "%a"\n',
308     'set xtics 86400\n',
309     'set mxtics 3\n'
310     ],
311     os.environ['HOME']+'/tmp/myd_week_l'
312     )
313    
314     sl10 = countslist[-22300:]
315     sl11 = timestringlist[-22300:]
316    
317     # sl12, sl13 = avgdata(sl10, sl11, 100)
318    
319     monthtuple_small = (
320     sl10[::100], sl11[::100],
321     [
322     'set terminal png size 365,201 font "arial,8"\n',
323     'set output "/user/templon/public_html/run-month-small.png"\n',
324     'set format x "%m/%d"\n',
325     'set xtics 604800\n',
326     'set nomxtics \n'
327     ],
328     os.environ['HOME']+'/tmp/myd_month_s'
329     )
330    
331     # sl14, sl15 = avgdata(sl10, sl11, 20)
332    
333     monthtuple_large = (
334     sl10[::20], sl11[::20],
335     [
336     'set terminal png size 1000,535 font "arial,10"\n',
337     'set output "/user/templon/public_html/run-month-large.png"\n',
338     'set format x "%m/%d"\n',
339     'set xtics 604800\n',
340     'set mxtics 7\n'
341     ],
342     os.environ['HOME']+'/tmp/myd_month_l'
343     )
344    
345    
346     # do it again but for queued jobs
347    
348     infh = open(os.environ['HOME']+'/tmp/groups.qdata','r')
349     datablocks = infh.read().split('\n\n')
350    
351     countslist = list() # holds one rundict per timestamp
352     timestringlist = list() # same order as countslist
353    
354     for block in datablocks[:-1]:
355     lines = block.split('\n')
356     timestr = lines[0] ; timestringlist.append(timestr)
357     rundict = dict() # format rundict['vo'] = number_running
358     for l in lines[1:]:
359     vo, num = eval(l, {}, {})
360     rundict[vo] = num
361     countslist.append(rundict)
362    
363     qhrtuple_small = (
364     countslist[-75:],
365     timestringlist[-75:],
366     [
367     # 'set terminal x11 size 1000,535\n',
368     'set terminal png size 365,201 font "arial,8"\n',
369     'set key autotitle columnheader\n',
370     'set xdata time\n',
371     'set timefmt "%b-%d-%Y-%H:%M:%S"\n',
372     'set output "/user/templon/public_html/queued-hr-small.png"\n',
373     '# time range must be in same format as data file\n',
374     'set grid xtics mxtics ytics mytics front\n',
375     'set yrange [0 : *]\n',
376     'set auto fix\n'
377     'set key outside samplen 1\n',
378     'set format x "%H:%M"\n',
379     'set xtics 3600\n',
380     'set mxtics 6\n',
381     'set mytics 3\n'
382     ],
383     os.environ['HOME']+'/tmp/myqd_hr'
384     )
385    
386     qhrtuple_large = (
387     countslist[-75:],
388     timestringlist[-75:],
389     [
390     'set terminal png size 1000,535 font "arial,10"\n',
391     'set output "/user/templon/public_html/queued-hr-large.png"\n',
392     'set format x "%H:%M"\n',
393     'set xtics 3600\n',
394     'set mxtics 6\n',
395     'set mytics 3\n'
396     'set key autotitle columnheader\n',
397     'set xdata time\n',
398     'set timefmt "%b-%d-%Y-%H:%M:%S"\n',
399     '# time range must be in same format as data file\n',
400     'set grid xtics mxtics ytics mytics front\n',
401     # 'set yrange [0 : 2600]\n',
402     'set auto fix\n'
403     'set key outside samplen 1\n',
404     ],
405     os.environ['HOME']+'/tmp/myqd_hr'
406     )
407    
408     qdaytuple_small = (
409     countslist[-720:],
410     timestringlist[-720:],
411     [
412     'set terminal png size 365,201 font "arial,8"\n',
413     'set output "/user/templon/public_html/queued-day-small.png"\n',
414     'set format x "%H:%M"\n',
415     'set xtics 21600\n',
416     'set mxtics 3\n'
417     ],
418     os.environ['HOME']+'/tmp/myqd_day'
419     )
420    
421     qdaytuple_large = (
422     countslist[-720:],
423     timestringlist[-720:],
424     [
425     'set terminal png size 1000,535 font "arial,10"\n',
426     'set output "/user/templon/public_html/queued-day-large.png"\n',
427     'set format x "%H:%M"\n',
428     'set xtics 7200\n',
429     'set mxtics 4\n'
430     ],
431     os.environ['HOME']+'/tmp/myqd_day'
432     )
433    
434    
435     plottuple_list = [ hrtuple_small, hrtuple_large, daytuple_small,
436     daytuple_large, weektuple_small, weektuple_large,
437     monthtuple_small, monthtuple_large,
438     qhrtuple_small, qhrtuple_large,
439     qdaytuple_small, qdaytuple_large
440     ]
441    
442     plotfileh = open(pfnam,'w')
443    
444     def emit_plotfile(pfh, plotslist):
445     # pfh is handle to plot file object
446     # first make the right data structure: rundata['vo']['timestamp'] = number
447     rundata = dict() # format rundata['vo']['timestamp'] = number
448     tsl = plotslist[1]
449     ctl = plotslist[0]
450     for i in range(len(tsl)):
451     for vo in ctl[i].keys():
452     if vo not in rundata.keys():
453     rundata[vo] = dict()
454     rundata[vo][tsl[i]] = ctl[i][vo]
455    
456     sums = dict()
457     for k in rundata.keys():
458     sums[k] = sum(rundata[k].values())
459    
460     vos_by_jobs = sorted(sums, key=sums.get, reverse=False)
461    
462     topvos = vos_by_jobs[-NUMGROUPS:]
463    
464     dfh = open(plotslist[3], 'w')
465     dfh.write('timestamp ')
466     for k in topvos:
467     dfh.write("%8s " % k)
468     dfh.write('\n')
469     for t in tsl:
470     dfh.write(t+ " ")
471     for k in topvos:
472     if t in rundata[k].keys():
473     val = rundata[k][t]
474     else:
475     val = 0
476     dfh.write("%4d " % val)
477     dfh.write('\n')
478     dfh.close()
479    
480     import glob
481     import os
482    
483     gplines = plotslist[2]
484    
485     # gplines.append('set terminal png size 335,201\n')
486    
487     ctr = 1
488     for vo in reversed(topvos):
489     if ctr == 1:
490     pstr = 'plot '
491     datafnam = plotslist[3]
492     pstr += '"' + datafnam + '" using 1:($' + repr(topvos.index(vo)+2)
493     for k2 in range(topvos.index(vo)+1, 1, -1):
494     pstr += '+$' + repr(k2)
495     pstr += ') t "' + vo + '"'
496     if ctr < len(topvos):
497     pstr += ' w boxes fs solid,'
498     ctr += 1
499     else:
500     pstr += ' w boxes fs solid'
501     gplines.append(pstr + '\n')
502    
503     for l in gplines:
504     pfh.write(l)
505    
506     # gplines.append('set terminal png size 1000,535\n')
507     # gplines.append('set output "/user/templon/public_html/wait-hr-large.png"\n')
508     # gplines.append('set format x "%H:%M"\n')
509     # gplines.append('set format y "%.0te+%S"\n')
510     # gplines.append('set xtics 3600\n')
511    
512     # ctr = 1
513     # for vo in volist:
514     # if ctr == 1:
515     # pstr = 'plot '
516     # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
517     # pstr += '"< tail -75 ' + datafnam + '" using 1:3' + \
518     # ' title "' + vo + '"'
519     # if ctr < len(volist):
520     # pstr += ' with lines lw 3,'
521     # ctr += 1
522     # else:
523     # pstr += ' with lines lw 1'
524     # gplines.append(pstr + '\n')
525    
526     # gplines.append('set terminal png size 335,201\n')
527     # gplines.append('set output "/user/templon/public_html/wait-day-small.png"\n')
528     # gplines.append('set format x "%H:%M"\n')
529     # gplines.append('set format y "%.0te+%S"\n')
530     # gplines.append('set xtics 21600\n')
531     # gplines.append('set mxtics 3\n')
532    
533     # ctr = 1
534     # for vo in volist:
535     # if ctr == 1:
536     # pstr = 'plot '
537     # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
538     # pstr += '"< tail -720 ' + datafnam + '" using 1:3' + \
539     # ' title "' + vo + '"'
540     # if ctr < len(volist):
541     # pstr += ' with lines lw 3,'
542     # ctr += 1
543     # else:
544     # pstr += ' with lines lw 1'
545     # gplines.append(pstr + '\n')
546    
547     # gplines.append('set terminal png size 1000,535\n')
548     # gplines.append('set output "/user/templon/public_html/wait-day-large.png"\n')
549     # gplines.append('set format x "%H:%M"\n')
550     # gplines.append('set format y "%.0te+%S"\n')
551     # gplines.append('set xtics 7200\n')
552     # gplines.append('set mxtics 4\n')
553    
554     # ctr = 1
555     # for vo in volist:
556     # if ctr == 1:
557     # pstr = 'plot '
558     # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
559     # pstr += '"< tail -720 ' + datafnam + '" using 1:3' + \
560     # ' title "' + vo + '"'
561     # if ctr < len(volist):
562     # pstr += ' with lines lw 3,'
563     # ctr += 1
564     # else:
565     # pstr += ' with lines lw 1'
566     # gplines.append(pstr + '\n')
567    
568     # gplines.append('set terminal png size 335,201\n')
569     # gplines.append('set output "/user/templon/public_html/wait-week-small.png"\n')
570     # gplines.append('set format x "%a"\n')
571     # gplines.append('set format y "%.0te+%S"\n')
572     # gplines.append('set xtics 86400\n')
573     # gplines.append('set mxtics 3\n')
574    
575     # ctr = 1
576     # for vo in volist:
577     # if ctr == 1:
578     # pstr = 'plot '
579     # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
580     # pstr += '"< tail -5760 ' + datafnam + '" using 1:3' + \
581     # ' title "' + vo + '"'
582     # if ctr < len(volist):
583     # pstr += ' with lines lw 3,'
584     # ctr += 1
585     # else:
586     # pstr += ' with lines lw 1'
587     # gplines.append(pstr + '\n')
588    
589     # gplines.append('set terminal png size 1000,535\n')
590     # gplines.append('set output "/user/templon/public_html/wait-week-large.png"\n')
591     # gplines.append('set format x "%a %H:%M"\n')
592     # gplines.append('set format y "%.0te+%S"\n')
593     # gplines.append('set xtics 86400\n')
594     # gplines.append('set mxtics 3\n')
595    
596     # ctr = 1
597     # for vo in volist:
598     # if ctr == 1:
599     # pstr = 'plot '
600     # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
601     # pstr += '"< tail -5760 ' + datafnam + '" using 1:3' + \
602     # ' title "' + vo + '"'
603     # if ctr < len(volist):
604     # pstr += ' with lines lw 3,'
605     # ctr += 1
606     # else:
607     # pstr += ' with lines lw 1'
608    
609     for itm in plottuple_list:
610     emit_plotfile(plotfileh, itm)
611    
612     plotfileh.close()
613     os.system(os.environ['HOME']+'/bin/gnuplot ' + pfnam)

Properties

Name Value
svn:executable *
svn:keywords Id URL

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28