/[pdpsoft]/nl.nikhef.ndpf.groupviews/trunk/ndpf-gv-mkplots
ViewVC logotype

Diff of /nl.nikhef.ndpf.groupviews/trunk/ndpf-gv-mkplots

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 2395 by templon, Thu Jul 28 12:49:35 2011 UTC revision 2396 by templon, Mon Aug 8 10:37:29 2011 UTC
# Line 15  colors = [ "#A6CEE3",  "#1F77B4",  "#B2D Line 15  colors = [ "#A6CEE3",  "#1F77B4",  "#B2D
15    
16  colors.reverse()  colors.reverse()
17    
18  timeargs = { 'hr'    : [ '-s', 'n-3h30min',  '-e', 'n' ],  timeargs = { 'hr'    : [ '-s', 'n-4h',  '-e', 'n' ],
19               'day'   : [ '-s', 'n-29h', '-e', 'n' ],               'day'   : [ '-s', 'n-29h', '-e', 'n' ],
20               'week'  : [ '-s', 'n-8d', '-e', 'n' ],               'week'  : [ '-s', 'n-8d', '-e', 'n' ],
21               'month' : [ '-s', 'n-35d', '-e', 'n' ]               'month' : [ '-s', 'n-35d', '-e', 'n' ]
# Line 43  import rrdtool Line 43  import rrdtool
43  import time  import time
44  now=int(time.mktime(time.localtime()))  now=int(time.mktime(time.localtime()))
45  end = (now / 60) * 60  end = (now / 60) * 60
46  start = end - (3600 * 30) + 60  start  = end - (3600 * 24) + 60
47    
48  import glob         # number of running jobs  import glob         # number of running jobs
49    
# Line 140  for dbtype in ['queued', 'running', 'wai Line 140  for dbtype in ['queued', 'running', 'wai
140  import time  import time
141  now=int(time.mktime(time.localtime()))  now=int(time.mktime(time.localtime()))
142  end = (now / 3600 ) * 3600  end = (now / 3600 ) * 3600
143  start = end - (86400*35) + 3600  start = end - (86400*30) + 3600
144    
145  ### block finding 'top N' group list ###  ### block finding 'top N' group list ###
146    
# Line 173  for dbtype in ['queued', 'running', 'wai Line 173  for dbtype in ['queued', 'running', 'wai
173                  doplot(topgroups_month, dbtype, psize, timetag)                  doplot(topgroups_month, dbtype, psize, timetag)
174  import sys  import sys
175  sys.exit(0)  sys.exit(0)
   
 pfnam = os.environ['HOME']+'/tmp/run.plt'  
   
 infh = open(os.environ['HOME']+'/tmp/groups.rdata','r')  
 datablocks = infh.read().split('\n\n')  
   
 countslist     = list() # holds one rundict per timestamp  
 timestringlist = list() # same order as countslist  
   
 for block in datablocks[:-1]:  
     lines = block.split('\n')  
     timestr = lines[0] ; timestringlist.append(timestr)  
     rundict = dict()  # format rundict['vo'] = number_running  
     for l in lines[1:]:  
         vo, num = eval(l, {}, {})  
         rundict[vo] = num  
     countslist.append(rundict)  
   
 def avgdata(vals, tstamps, step):  
     # step must be odd  
     # lengths of vals and tstamps must be identical, and  
     # also must be integer multiple of step  
   
     newvals = list()  
     newstamps = list()  
     if len(vals) % step != 0:  
         print "length of input arrays must be multiple of step size"  
         sys.exit(0)  
     optr = 0  
     while optr < len(vals):  
         print optr  
         print step  
         print vals[optr:optr+step]  
           
         newvals.append(sum(vals[optr:optr+step]))  
         newstamps.append(tstamps[optr + (fact/2)])  
         optr += fact  
     return newvals, newstamps  
   
 sl1 = countslist[-75:]  
 sl2 = timestringlist[-75:]  
   
 hrtuple_small = (  
     sl1, sl2,  
     [  
 #    'set terminal x11 size 1000,535\n',  
     'set terminal png size 365,201 font "arial,8"\n',  
     'set key autotitle columnheader\n',  
     'set xdata time\n',  
     'set timefmt "%b-%d-%Y-%H:%M:%S"\n',  
     'set output "/user/templon/public_html/run-hr-small.png"\n',  
     '# time range must be in same format as data file\n',  
     'set grid xtics mxtics ytics mytics front\n',  
     'set yrange [0 : *]\n',  
     'set auto fix\n'  
     'set key outside samplen 1\n',  
     'set format x "%H:%M"\n',  
     'set xtics 3600\n',  
     'set mxtics 6\n',  
     'set mytics 2\n'  
     ],  
     os.environ['HOME']+'/tmp/myd_hr_s'  
     )  
   
 hrtuple_large = (  
     sl1, sl2,  
     [  
     'set terminal png size 1000,535 font "arial,10"\n',  
     'set output "/user/templon/public_html/run-hr-large.png"\n',  
     'set format x "%H:%M"\n',  
     'set xtics 3600\n',  
     'set mxtics 6\n',  
     'set mytics 4\n'  
     'set key autotitle columnheader\n',  
     'set xdata time\n',  
     'set timefmt "%b-%d-%Y-%H:%M:%S"\n',  
     '# time range must be in same format as data file\n',  
     'set grid xtics mxtics ytics mytics front\n',  
     'set yrange [0 : *]\n',  
     'set auto fix\n'  
     'set key outside samplen 1\n',  
     ],  
     os.environ['HOME']+'/tmp/myd_hr_l'  
     )  
   
 sl3 = countslist[-720:]  
 sl4 = timestringlist[-720:]  
   
 daytuple_small = (  
     sl3, sl4,  
     [  
     'set terminal png size 365,201  font "arial,8"\n',  
     'set output "/user/templon/public_html/run-day-small.png"\n',  
      'set format x "%H:%M"\n',  
      'set xtics 21600\n',  
      'set mxtics 3\n'  
      ],  
     os.environ['HOME']+'/tmp/myd_day_s'  
     )  
   
 daytuple_large = (  
     sl3, sl4,  
     [  
     'set terminal png size 1000,535  font "arial,10"\n',  
     'set output "/user/templon/public_html/run-day-large.png"\n',  
     'set format x "%H:%M"\n',  
     'set xtics 7200\n',  
     'set mxtics 4\n'  
      ],  
     os.environ['HOME']+'/tmp/myd_day_l'  
     )  
   
 sl5 = countslist[-5760:]  
 sl6 = timestringlist[-5760:]  
   
 weektuple_small = (  
     sl5[::5], sl6[::5],  
     [  
     'set terminal png size 365,201 font "arial,8"\n',  
     'set output "/user/templon/public_html/run-week-small.png"\n',  
      'set format x "%a"\n',  
      'set xtics 86400\n',  
      'set nomxtics \n'  
      ],  
     os.environ['HOME']+'/tmp/myd_week_s'  
     )  
   
 # sl7, sl8 = avgdata(sl5, sl6, 5)  
   
 weektuple_large = (  
     sl5[::2], sl6[::2],  
     [  
     'set terminal png size 1000,535 font "arial,10"\n',  
     'set output "/user/templon/public_html/run-week-large.png"\n',  
     'set format x "%a"\n',  
     'set xtics 86400\n',  
     'set mxtics 3\n'  
     ],  
     os.environ['HOME']+'/tmp/myd_week_l'  
     )  
   
 sl10 = countslist[-22300:]  
 sl11 = timestringlist[-22300:]  
   
 # sl12, sl13 = avgdata(sl10, sl11, 100)  
   
 monthtuple_small = (  
     sl10[::100], sl11[::100],  
     [  
     'set terminal png size 365,201 font "arial,8"\n',  
     'set output "/user/templon/public_html/run-month-small.png"\n',  
     'set format x "%m/%d"\n',  
     'set xtics 604800\n',  
     'set nomxtics \n'  
      ],  
     os.environ['HOME']+'/tmp/myd_month_s'  
     )  
   
 # sl14, sl15 = avgdata(sl10, sl11, 20)  
   
 monthtuple_large = (  
     sl10[::20], sl11[::20],  
     [  
     'set terminal png size 1000,535 font "arial,10"\n',  
     'set output "/user/templon/public_html/run-month-large.png"\n',  
     'set format x "%m/%d"\n',  
     'set xtics 604800\n',  
     'set mxtics 7\n'  
     ],  
     os.environ['HOME']+'/tmp/myd_month_l'  
     )  
   
   
 # do it again but for queued jobs  
   
 infh = open(os.environ['HOME']+'/tmp/groups.qdata','r')  
 datablocks = infh.read().split('\n\n')  
   
 countslist     = list() # holds one rundict per timestamp  
 timestringlist = list() # same order as countslist  
   
 for block in datablocks[:-1]:  
     lines = block.split('\n')  
     timestr = lines[0] ; timestringlist.append(timestr)  
     rundict = dict()  # format rundict['vo'] = number_running  
     for l in lines[1:]:  
         vo, num = eval(l, {}, {})  
         rundict[vo] = num  
     countslist.append(rundict)  
   
 qhrtuple_small = (  
     countslist[-75:],  
     timestringlist[-75:],  
     [  
 #    'set terminal x11 size 1000,535\n',  
     'set terminal png size 365,201 font "arial,8"\n',  
     'set key autotitle columnheader\n',  
     'set xdata time\n',  
     'set timefmt "%b-%d-%Y-%H:%M:%S"\n',  
     'set output "/user/templon/public_html/queued-hr-small.png"\n',  
     '# time range must be in same format as data file\n',  
     'set grid xtics mxtics ytics mytics front\n',  
     'set yrange [0 : *]\n',  
     'set auto fix\n'  
     'set key outside samplen 1\n',  
     'set format x "%H:%M"\n',  
     'set xtics 3600\n',  
     'set mxtics 6\n',  
     'set mytics 3\n'  
     ],  
     os.environ['HOME']+'/tmp/myqd_hr'  
     )  
   
 qhrtuple_large = (  
     countslist[-75:],  
     timestringlist[-75:],  
     [  
     'set terminal png size 1000,535 font "arial,10"\n',  
     'set output "/user/templon/public_html/queued-hr-large.png"\n',  
     'set format x "%H:%M"\n',  
     'set xtics 3600\n',  
     'set mxtics 6\n',  
     'set mytics 3\n'  
     'set key autotitle columnheader\n',  
     'set xdata time\n',  
     'set timefmt "%b-%d-%Y-%H:%M:%S"\n',  
     '# time range must be in same format as data file\n',  
     'set grid xtics mxtics ytics mytics front\n',  
 #    'set yrange [0 : 2600]\n',  
     'set auto fix\n'  
     'set key outside samplen 1\n',  
     ],  
     os.environ['HOME']+'/tmp/myqd_hr'  
     )  
   
 qdaytuple_small = (  
     countslist[-720:],  
     timestringlist[-720:],  
     [  
     'set terminal png size 365,201 font "arial,8"\n',  
     'set output "/user/templon/public_html/queued-day-small.png"\n',  
      'set format x "%H:%M"\n',  
      'set xtics 21600\n',  
      'set mxtics 3\n'  
      ],  
     os.environ['HOME']+'/tmp/myqd_day'  
     )  
   
 qdaytuple_large = (  
     countslist[-720:],  
     timestringlist[-720:],  
     [  
     'set terminal png size 1000,535 font "arial,10"\n',  
     'set output "/user/templon/public_html/queued-day-large.png"\n',  
     'set format x "%H:%M"\n',  
     'set xtics 7200\n',  
     'set mxtics 4\n'  
      ],  
     os.environ['HOME']+'/tmp/myqd_day'  
     )  
   
   
 plottuple_list = [ hrtuple_small, hrtuple_large, daytuple_small,  
                    daytuple_large, weektuple_small, weektuple_large,  
                    monthtuple_small, monthtuple_large,  
                    qhrtuple_small, qhrtuple_large,  
                    qdaytuple_small, qdaytuple_large  
                    ]  
   
 plotfileh = open(pfnam,'w')  
   
 def emit_plotfile(pfh, plotslist):  
     # pfh is handle to plot file object  
     # first make the right data structure: rundata['vo']['timestamp'] =  number  
     rundata = dict()  # format rundata['vo']['timestamp'] = number  
     tsl = plotslist[1]  
     ctl = plotslist[0]  
     for i in range(len(tsl)):  
         for vo in ctl[i].keys():  
             if vo not in rundata.keys():  
                 rundata[vo] = dict()  
             rundata[vo][tsl[i]] = ctl[i][vo]  
   
     sums = dict()  
     for k in rundata.keys():  
         sums[k] = sum(rundata[k].values())  
   
     vos_by_jobs = sorted(sums, key=sums.get, reverse=False)  
       
     topvos = vos_by_jobs[-NUMGROUPS:]  
   
     dfh = open(plotslist[3], 'w')  
     dfh.write('timestamp          ')  
     for k in topvos:  
         dfh.write("%8s " % k)  
     dfh.write('\n')  
     for t in tsl:  
         dfh.write(t+ "   ")  
         for k in topvos:  
             if t in rundata[k].keys():  
                 val = rundata[k][t]  
             else:  
                 val = 0  
             dfh.write("%4d     " % val)  
         dfh.write('\n')  
     dfh.close()  
       
     import glob  
     import os  
   
     gplines = plotslist[2]  
   
     # gplines.append('set terminal png size 335,201\n')  
   
     ctr = 1  
     for vo in reversed(topvos):  
         if ctr == 1:  
             pstr = 'plot '  
         datafnam = plotslist[3]  
         pstr += '"' + datafnam + '" using 1:($' + repr(topvos.index(vo)+2)  
         for k2 in range(topvos.index(vo)+1, 1, -1):  
             pstr += '+$' + repr(k2)  
         pstr += ') t "' + vo + '"'  
         if ctr < len(topvos):  
             pstr += ' w boxes fs solid,'  
             ctr += 1  
         else:  
             pstr += ' w boxes fs solid'  
     gplines.append(pstr + '\n')  
   
     for l in gplines:  
         pfh.write(l)  
   
 # gplines.append('set terminal png size 1000,535\n')  
 # gplines.append('set output "/user/templon/public_html/wait-hr-large.png"\n')  
 # gplines.append('set format x "%H:%M"\n')  
 # gplines.append('set format y "%.0te+%S"\n')  
 # gplines.append('set xtics 3600\n')  
   
 # ctr = 1  
 # for vo in volist:  
 #     if ctr == 1:  
 #         pstr = 'plot '  
 #     datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'  
 #     pstr += '"< tail -75 ' + datafnam + '" using 1:3' + \  
 #            ' title "' + vo + '"'  
 #     if ctr < len(volist):  
 #         pstr += ' with lines lw 3,'  
 #         ctr += 1  
 #     else:  
 #         pstr += ' with lines lw 1'  
 # gplines.append(pstr + '\n')  
   
 # gplines.append('set terminal png size 335,201\n')  
 # gplines.append('set output "/user/templon/public_html/wait-day-small.png"\n')  
 # gplines.append('set format x "%H:%M"\n')  
 # gplines.append('set format y "%.0te+%S"\n')  
 # gplines.append('set xtics 21600\n')  
 # gplines.append('set mxtics 3\n')  
   
 # ctr = 1  
 # for vo in volist:  
 #     if ctr == 1:  
 #         pstr = 'plot '  
 #     datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'  
 #     pstr += '"< tail -720 ' + datafnam + '" using 1:3' + \  
 #            ' title "' + vo + '"'  
 #     if ctr < len(volist):  
 #         pstr += ' with lines lw 3,'  
 #         ctr += 1  
 #     else:  
 #         pstr += ' with lines lw 1'  
 # gplines.append(pstr + '\n')  
   
 # gplines.append('set terminal png size 1000,535\n')  
 # gplines.append('set output "/user/templon/public_html/wait-day-large.png"\n')  
 # gplines.append('set format x "%H:%M"\n')  
 # gplines.append('set format y "%.0te+%S"\n')  
 # gplines.append('set xtics 7200\n')  
 # gplines.append('set mxtics 4\n')  
   
 # ctr = 1  
 # for vo in volist:  
 #     if ctr == 1:  
 #         pstr = 'plot '  
 #     datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'  
 #     pstr += '"< tail -720 ' + datafnam + '" using 1:3' + \  
 #            ' title "' + vo + '"'  
 #     if ctr < len(volist):  
 #         pstr += ' with lines lw 3,'  
 #         ctr += 1  
 #     else:  
 #         pstr += ' with lines lw 1'  
 # gplines.append(pstr + '\n')  
   
 # gplines.append('set terminal png size 335,201\n')  
 # gplines.append('set output "/user/templon/public_html/wait-week-small.png"\n')  
 # gplines.append('set format x "%a"\n')  
 # gplines.append('set format y "%.0te+%S"\n')  
 # gplines.append('set xtics 86400\n')  
 # gplines.append('set mxtics 3\n')  
   
 # ctr = 1  
 # for vo in volist:  
 #     if ctr == 1:  
 #         pstr = 'plot '  
 #     datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'  
 #     pstr += '"< tail -5760 ' + datafnam + '" using 1:3' + \  
 #            ' title "' + vo + '"'  
 #     if ctr < len(volist):  
 #         pstr += ' with lines lw 3,'  
 #         ctr += 1  
 #     else:  
 #         pstr += ' with lines lw 1'  
 # gplines.append(pstr + '\n')  
   
 # gplines.append('set terminal png size 1000,535\n')  
 # gplines.append('set output "/user/templon/public_html/wait-week-large.png"\n')  
 # gplines.append('set format x "%a %H:%M"\n')  
 # gplines.append('set format y "%.0te+%S"\n')  
 # gplines.append('set xtics 86400\n')  
 # gplines.append('set mxtics 3\n')  
   
 # ctr = 1  
 # for vo in volist:  
 #     if ctr == 1:  
 #         pstr = 'plot '  
 #     datafnam = os.environ['HOME']+'/tmp/'+vo+'.wdata'  
 #     pstr += '"< tail -5760 ' + datafnam + '" using 1:3' + \  
 #            ' title "' + vo + '"'  
 #     if ctr < len(volist):  
 #         pstr += ' with lines lw 3,'  
 #         ctr += 1  
 #     else:  
 #         pstr += ' with lines lw 1'  
   
 for itm in plottuple_list:  
     emit_plotfile(plotfileh, itm)  
   
 plotfileh.close()  
 os.system(os.environ['HOME']+'/bin/gnuplot ' + pfnam)  

Legend:
Removed from v.2395  
changed lines
  Added in v.2396

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28