/[pdpsoft]/nl.nikhef.ndpf.groupviews/trunk/ndpf-gv-mkplots
ViewVC logotype

Contents of /nl.nikhef.ndpf.groupviews/trunk/ndpf-gv-mkplots

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2363 - (show annotations) (download)
Sat Jul 23 09:05:22 2011 UTC (11 years, 2 months ago) by templon
File size: 15739 byte(s)
improved large size argument

1 #! /usr/bin/env python
2
3 import os
4
5 NUMGROUPS=8
6 DATADIR=os.environ['HOME'] + '/ndpfdata/'
7 PLOTDIR=os.environ['HOME'] + '/public_html/'
8 # 8 class qualitative paired color scheme
9
10 colors = [ "#A6CEE3", "#1F77B4", "#B2DF8A", "#33A02C",
11 "#FB9A99", "#E31A1C", "#FDBF6F", "#FF7F00" ]
12
13 colors.reverse()
14
15 # for reference : resolutions of the RRAs
16 # 120s x 75 points = 2,5 hr
17 # 240s x 360 points = 24 hr
18 # 3600s x 192 points = 192 hr (8 days)
19 # 14400s (4 hrs) x 186 points = 31 days
20 # 86400s (1 day) x 1825 points = 5 years
21
22 import rrdtool
23
24 # first need to find four "top eight" lists
25 # one for the past day, which will be used for the hour / day page
26 # one for the past week
27 # one for the past month
28 # one for the past year
29 # base it on running jobs
30
31 import time
32 now=int(time.mktime(time.localtime()))
33 end = (now / 3600) * 3600
34 start = end - 86400
35
36 import glob # number of running jobs
37
38 ### block finding 'top N' group list ###
39
40 day_groups = dict() # structure day_group[groupname] = total of hourly average
41
42 running_files = glob.glob(DATADIR+'*.running.rrd')
43 for db in running_files:
44 group = db[len(DATADIR):db.find('.running.rrd')]
45 if group == "total":
46 continue
47 tup = rrdtool.fetch(db,'AVERAGE','-r',"3600",
48 '-s', repr(start), '-e', repr(end))
49 vallist = [0] # start with zero, in case no vals returned, get zero as answer
50 for tup2 in tup[2]:
51 val = tup2[0]
52 if val:
53 vallist.append(val)
54 day_groups[group] = sum(vallist[:-1]) # chop weird last value from rrdtool
55
56 day_groups_sorted = sorted(day_groups, key=day_groups.get, reverse=False)
57 topgroups_day=day_groups_sorted[-NUMGROUPS:]
58
59 ### end block 'top N' group list ###
60
61 ### block generating plots ###
62
63 def doplot(grouplist, dbtype, psize, timetag):
64
65 sizeargs = { 'small' : [ '--imgformat', 'PNG', '--width', '285', '--height', '125' ],
66 'large' : [ '--imgformat', 'PNG', '--width', '950', '--height', '500' ]
67 }
68
69 timeargs = { 'hr' : [ '-s', 'n-2h', '-e', 'n' ],
70 'day' : [ '-s', 'n-24h', '-e', 'n' ]
71 }
72
73 defs = list()
74 plots = list()
75
76 data_defs = list()
77 plot_defs = list()
78
79 for group in (grouplist + ['total']):
80 data_defs.append('DEF:'+group+'='+DATADIR+group+'.'+dbtype+'.rrd:'+dbtype+':AVERAGE')
81
82 for idx in range(len(grouplist)):
83 group = grouplist[idx]
84 pdefstr = 'AREA' ':' + group + colors[idx] + ':' + group
85 if idx < 5 and psize == 'small':
86 pdefstr = pdefstr + '\g'
87 if idx > 0:
88 pdefstr = pdefstr + ':STACK'
89 plot_defs.append(pdefstr)
90
91 plot_defs.append('LINE:total#000000:total')
92
93 pargs = [ PLOTDIR + dbtype + '-' + timetag + '-' + psize + '.png'] + \
94 sizeargs[psize] + timeargs[timetag] + data_defs + plot_defs
95 rrdtool.graph( *pargs )
96
97 for dbtype in ['queued', 'running']:
98 for psize in ['small', 'large']:
99 for timetag in ['hr', 'day']:
100 doplot(topgroups_day, dbtype, psize, timetag)
101
102 import sys
103 sys.exit(0)
104
105 pfnam = os.environ['HOME']+'/tmp/run.plt'
106
107 infh = open(os.environ['HOME']+'/tmp/groups.rdata','r')
108 datablocks = infh.read().split('\n\n')
109
110 countslist = list() # holds one rundict per timestamp
111 timestringlist = list() # same order as countslist
112
113 for block in datablocks[:-1]:
114 lines = block.split('\n')
115 timestr = lines[0] ; timestringlist.append(timestr)
116 rundict = dict() # format rundict['vo'] = number_running
117 for l in lines[1:]:
118 vo, num = eval(l, {}, {})
119 rundict[vo] = num
120 countslist.append(rundict)
121
122 def avgdata(vals, tstamps, step):
123 # step must be odd
124 # lengths of vals and tstamps must be identical, and
125 # also must be integer multiple of step
126
127 newvals = list()
128 newstamps = list()
129 if len(vals) % step != 0:
130 print "length of input arrays must be multiple of step size"
131 sys.exit(0)
132 optr = 0
133 while optr < len(vals):
134 print optr
135 print step
136 print vals[optr:optr+step]
137
138 newvals.append(sum(vals[optr:optr+step]))
139 newstamps.append(tstamps[optr + (fact/2)])
140 optr += fact
141 return newvals, newstamps
142
143 sl1 = countslist[-75:]
144 sl2 = timestringlist[-75:]
145
146 hrtuple_small = (
147 sl1, sl2,
148 [
149 # 'set terminal x11 size 1000,535\n',
150 'set terminal png size 365,201 font "arial,8"\n',
151 'set key autotitle columnheader\n',
152 'set xdata time\n',
153 'set timefmt "%b-%d-%Y-%H:%M:%S"\n',
154 'set output "/user/templon/public_html/run-hr-small.png"\n',
155 '# time range must be in same format as data file\n',
156 'set grid xtics mxtics ytics mytics front\n',
157 'set yrange [0 : *]\n',
158 'set auto fix\n'
159 'set key outside samplen 1\n',
160 'set format x "%H:%M"\n',
161 'set xtics 3600\n',
162 'set mxtics 6\n',
163 'set mytics 2\n'
164 ],
165 os.environ['HOME']+'/tmp/myd_hr_s'
166 )
167
168 hrtuple_large = (
169 sl1, sl2,
170 [
171 'set terminal png size 1000,535 font "arial,10"\n',
172 'set output "/user/templon/public_html/run-hr-large.png"\n',
173 'set format x "%H:%M"\n',
174 'set xtics 3600\n',
175 'set mxtics 6\n',
176 'set mytics 4\n'
177 'set key autotitle columnheader\n',
178 'set xdata time\n',
179 'set timefmt "%b-%d-%Y-%H:%M:%S"\n',
180 '# time range must be in same format as data file\n',
181 'set grid xtics mxtics ytics mytics front\n',
182 'set yrange [0 : *]\n',
183 'set auto fix\n'
184 'set key outside samplen 1\n',
185 ],
186 os.environ['HOME']+'/tmp/myd_hr_l'
187 )
188
189 sl3 = countslist[-720:]
190 sl4 = timestringlist[-720:]
191
192 daytuple_small = (
193 sl3, sl4,
194 [
195 'set terminal png size 365,201 font "arial,8"\n',
196 'set output "/user/templon/public_html/run-day-small.png"\n',
197 'set format x "%H:%M"\n',
198 'set xtics 21600\n',
199 'set mxtics 3\n'
200 ],
201 os.environ['HOME']+'/tmp/myd_day_s'
202 )
203
204 daytuple_large = (
205 sl3, sl4,
206 [
207 'set terminal png size 1000,535 font "arial,10"\n',
208 'set output "/user/templon/public_html/run-day-large.png"\n',
209 'set format x "%H:%M"\n',
210 'set xtics 7200\n',
211 'set mxtics 4\n'
212 ],
213 os.environ['HOME']+'/tmp/myd_day_l'
214 )
215
216 sl5 = countslist[-5760:]
217 sl6 = timestringlist[-5760:]
218
219 weektuple_small = (
220 sl5[::5], sl6[::5],
221 [
222 'set terminal png size 365,201 font "arial,8"\n',
223 'set output "/user/templon/public_html/run-week-small.png"\n',
224 'set format x "%a"\n',
225 'set xtics 86400\n',
226 'set nomxtics \n'
227 ],
228 os.environ['HOME']+'/tmp/myd_week_s'
229 )
230
231 # sl7, sl8 = avgdata(sl5, sl6, 5)
232
233 weektuple_large = (
234 sl5[::2], sl6[::2],
235 [
236 'set terminal png size 1000,535 font "arial,10"\n',
237 'set output "/user/templon/public_html/run-week-large.png"\n',
238 'set format x "%a"\n',
239 'set xtics 86400\n',
240 'set mxtics 3\n'
241 ],
242 os.environ['HOME']+'/tmp/myd_week_l'
243 )
244
245 sl10 = countslist[-22300:]
246 sl11 = timestringlist[-22300:]
247
248 # sl12, sl13 = avgdata(sl10, sl11, 100)
249
250 monthtuple_small = (
251 sl10[::100], sl11[::100],
252 [
253 'set terminal png size 365,201 font "arial,8"\n',
254 'set output "/user/templon/public_html/run-month-small.png"\n',
255 'set format x "%m/%d"\n',
256 'set xtics 604800\n',
257 'set nomxtics \n'
258 ],
259 os.environ['HOME']+'/tmp/myd_month_s'
260 )
261
262 # sl14, sl15 = avgdata(sl10, sl11, 20)
263
264 monthtuple_large = (
265 sl10[::20], sl11[::20],
266 [
267 'set terminal png size 1000,535 font "arial,10"\n',
268 'set output "/user/templon/public_html/run-month-large.png"\n',
269 'set format x "%m/%d"\n',
270 'set xtics 604800\n',
271 'set mxtics 7\n'
272 ],
273 os.environ['HOME']+'/tmp/myd_month_l'
274 )
275
276
277 # do it again but for queued jobs
278
279 infh = open(os.environ['HOME']+'/tmp/groups.qdata','r')
280 datablocks = infh.read().split('\n\n')
281
282 countslist = list() # holds one rundict per timestamp
283 timestringlist = list() # same order as countslist
284
285 for block in datablocks[:-1]:
286 lines = block.split('\n')
287 timestr = lines[0] ; timestringlist.append(timestr)
288 rundict = dict() # format rundict['vo'] = number_running
289 for l in lines[1:]:
290 vo, num = eval(l, {}, {})
291 rundict[vo] = num
292 countslist.append(rundict)
293
294 qhrtuple_small = (
295 countslist[-75:],
296 timestringlist[-75:],
297 [
298 # 'set terminal x11 size 1000,535\n',
299 'set terminal png size 365,201 font "arial,8"\n',
300 'set key autotitle columnheader\n',
301 'set xdata time\n',
302 'set timefmt "%b-%d-%Y-%H:%M:%S"\n',
303 'set output "/user/templon/public_html/queued-hr-small.png"\n',
304 '# time range must be in same format as data file\n',
305 'set grid xtics mxtics ytics mytics front\n',
306 'set yrange [0 : *]\n',
307 'set auto fix\n'
308 'set key outside samplen 1\n',
309 'set format x "%H:%M"\n',
310 'set xtics 3600\n',
311 'set mxtics 6\n',
312 'set mytics 3\n'
313 ],
314 os.environ['HOME']+'/tmp/myqd_hr'
315 )
316
317 qhrtuple_large = (
318 countslist[-75:],
319 timestringlist[-75:],
320 [
321 'set terminal png size 1000,535 font "arial,10"\n',
322 'set output "/user/templon/public_html/queued-hr-large.png"\n',
323 'set format x "%H:%M"\n',
324 'set xtics 3600\n',
325 'set mxtics 6\n',
326 'set mytics 3\n'
327 'set key autotitle columnheader\n',
328 'set xdata time\n',
329 'set timefmt "%b-%d-%Y-%H:%M:%S"\n',
330 '# time range must be in same format as data file\n',
331 'set grid xtics mxtics ytics mytics front\n',
332 # 'set yrange [0 : 2600]\n',
333 'set auto fix\n'
334 'set key outside samplen 1\n',
335 ],
336 os.environ['HOME']+'/tmp/myqd_hr'
337 )
338
339 qdaytuple_small = (
340 countslist[-720:],
341 timestringlist[-720:],
342 [
343 'set terminal png size 365,201 font "arial,8"\n',
344 'set output "/user/templon/public_html/queued-day-small.png"\n',
345 'set format x "%H:%M"\n',
346 'set xtics 21600\n',
347 'set mxtics 3\n'
348 ],
349 os.environ['HOME']+'/tmp/myqd_day'
350 )
351
352 qdaytuple_large = (
353 countslist[-720:],
354 timestringlist[-720:],
355 [
356 'set terminal png size 1000,535 font "arial,10"\n',
357 'set output "/user/templon/public_html/queued-day-large.png"\n',
358 'set format x "%H:%M"\n',
359 'set xtics 7200\n',
360 'set mxtics 4\n'
361 ],
362 os.environ['HOME']+'/tmp/myqd_day'
363 )
364
365
366 plottuple_list = [ hrtuple_small, hrtuple_large, daytuple_small,
367 daytuple_large, weektuple_small, weektuple_large,
368 monthtuple_small, monthtuple_large,
369 qhrtuple_small, qhrtuple_large,
370 qdaytuple_small, qdaytuple_large
371 ]
372
373 plotfileh = open(pfnam,'w')
374
375 def emit_plotfile(pfh, plotslist):
376 # pfh is handle to plot file object
377 # first make the right data structure: rundata['vo']['timestamp'] = number
378 rundata = dict() # format rundata['vo']['timestamp'] = number
379 tsl = plotslist[1]
380 ctl = plotslist[0]
381 for i in range(len(tsl)):
382 for vo in ctl[i].keys():
383 if vo not in rundata.keys():
384 rundata[vo] = dict()
385 rundata[vo][tsl[i]] = ctl[i][vo]
386
387 sums = dict()
388 for k in rundata.keys():
389 sums[k] = sum(rundata[k].values())
390
391 vos_by_jobs = sorted(sums, key=sums.get, reverse=False)
392
393 topvos = vos_by_jobs[-NUMGROUPS:]
394
395 dfh = open(plotslist[3], 'w')
396 dfh.write('timestamp ')
397 for k in topvos:
398 dfh.write("%8s " % k)
399 dfh.write('\n')
400 for t in tsl:
401 dfh.write(t+ " ")
402 for k in topvos:
403 if t in rundata[k].keys():
404 val = rundata[k][t]
405 else:
406 val = 0
407 dfh.write("%4d " % val)
408 dfh.write('\n')
409 dfh.close()
410
411 import glob
412 import os
413
414 gplines = plotslist[2]
415
416 # gplines.append('set terminal png size 335,201\n')
417
418 ctr = 1
419 for vo in reversed(topvos):
420 if ctr == 1:
421 pstr = 'plot '
422 datafnam = plotslist[3]
423 pstr += '"' + datafnam + '" using 1:($' + repr(topvos.index(vo)+2)
424 for k2 in range(topvos.index(vo)+1, 1, -1):
425 pstr += '+$' + repr(k2)
426 pstr += ') t "' + vo + '"'
427 if ctr < len(topvos):
428 pstr += ' w boxes fs solid,'
429 ctr += 1
430 else:
431 pstr += ' w boxes fs solid'
432 gplines.append(pstr + '\n')
433
434 for l in gplines:
435 pfh.write(l)
436
437 # gplines.append('set terminal png size 1000,535\n')
438 # gplines.append('set output "/user/templon/public_html/wait-hr-large.png"\n')
439 # gplines.append('set format x "%H:%M"\n')
440 # gplines.append('set format y "%.0te+%S"\n')
441 # gplines.append('set xtics 3600\n')
442
443 # ctr = 1
444 # for vo in volist:
445 # if ctr == 1:
446 # pstr = 'plot '
447 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
448 # pstr += '"< tail -75 ' + datafnam + '" using 1:3' + \
449 # ' title "' + vo + '"'
450 # if ctr < len(volist):
451 # pstr += ' with lines lw 3,'
452 # ctr += 1
453 # else:
454 # pstr += ' with lines lw 1'
455 # gplines.append(pstr + '\n')
456
457 # gplines.append('set terminal png size 335,201\n')
458 # gplines.append('set output "/user/templon/public_html/wait-day-small.png"\n')
459 # gplines.append('set format x "%H:%M"\n')
460 # gplines.append('set format y "%.0te+%S"\n')
461 # gplines.append('set xtics 21600\n')
462 # gplines.append('set mxtics 3\n')
463
464 # ctr = 1
465 # for vo in volist:
466 # if ctr == 1:
467 # pstr = 'plot '
468 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
469 # pstr += '"< tail -720 ' + datafnam + '" using 1:3' + \
470 # ' title "' + vo + '"'
471 # if ctr < len(volist):
472 # pstr += ' with lines lw 3,'
473 # ctr += 1
474 # else:
475 # pstr += ' with lines lw 1'
476 # gplines.append(pstr + '\n')
477
478 # gplines.append('set terminal png size 1000,535\n')
479 # gplines.append('set output "/user/templon/public_html/wait-day-large.png"\n')
480 # gplines.append('set format x "%H:%M"\n')
481 # gplines.append('set format y "%.0te+%S"\n')
482 # gplines.append('set xtics 7200\n')
483 # gplines.append('set mxtics 4\n')
484
485 # ctr = 1
486 # for vo in volist:
487 # if ctr == 1:
488 # pstr = 'plot '
489 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
490 # pstr += '"< tail -720 ' + datafnam + '" using 1:3' + \
491 # ' title "' + vo + '"'
492 # if ctr < len(volist):
493 # pstr += ' with lines lw 3,'
494 # ctr += 1
495 # else:
496 # pstr += ' with lines lw 1'
497 # gplines.append(pstr + '\n')
498
499 # gplines.append('set terminal png size 335,201\n')
500 # gplines.append('set output "/user/templon/public_html/wait-week-small.png"\n')
501 # gplines.append('set format x "%a"\n')
502 # gplines.append('set format y "%.0te+%S"\n')
503 # gplines.append('set xtics 86400\n')
504 # gplines.append('set mxtics 3\n')
505
506 # ctr = 1
507 # for vo in volist:
508 # if ctr == 1:
509 # pstr = 'plot '
510 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
511 # pstr += '"< tail -5760 ' + datafnam + '" using 1:3' + \
512 # ' title "' + vo + '"'
513 # if ctr < len(volist):
514 # pstr += ' with lines lw 3,'
515 # ctr += 1
516 # else:
517 # pstr += ' with lines lw 1'
518 # gplines.append(pstr + '\n')
519
520 # gplines.append('set terminal png size 1000,535\n')
521 # gplines.append('set output "/user/templon/public_html/wait-week-large.png"\n')
522 # gplines.append('set format x "%a %H:%M"\n')
523 # gplines.append('set format y "%.0te+%S"\n')
524 # gplines.append('set xtics 86400\n')
525 # gplines.append('set mxtics 3\n')
526
527 # ctr = 1
528 # for vo in volist:
529 # if ctr == 1:
530 # pstr = 'plot '
531 # datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'
532 # pstr += '"< tail -5760 ' + datafnam + '" using 1:3' + \
533 # ' title "' + vo + '"'
534 # if ctr < len(volist):
535 # pstr += ' with lines lw 3,'
536 # ctr += 1
537 # else:
538 # pstr += ' with lines lw 1'
539
540 for itm in plottuple_list:
541 emit_plotfile(plotfileh, itm)
542
543 plotfileh.close()
544 os.system(os.environ['HOME']+'/bin/gnuplot ' + pfnam)

Properties

Name Value
svn:executable *
svn:keywords Id URL

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28