/[pdpsoft]/nl.nikhef.ndpf.groupviews/trunk/ndpf-mc-mkplots
ViewVC logotype

Contents of /nl.nikhef.ndpf.groupviews/trunk/ndpf-mc-mkplots

Parent Directory Parent Directory | Revision Log Revision Log


Revision 3005 - (show annotations) (download)
Tue Apr 19 07:59:39 2016 UTC (6 years, 1 month ago) by templon
File size: 6312 byte(s)
fix bug handling absence of 'nonmc' cycles in a plot range

1 #! /usr/bin/env python
2 # $Id: ndpf-gv-mkplots 2688 2014-01-21 16:20:14Z templon $
3 # Source: $URL: svn+ssh://svn@ndpfsvn.nikhef.nl/repos/pdpsoft/nl.nikhef.ndpf.groupviews/trunk/ndpf-gv-mkplots $
4 # J. A. Templon, NIKHEF/PDP 2011
5
6 import optparse
7
8 p = optparse.OptionParser(description="Program to make rrdtool plots " + \
9 "of performance of multicore pool")
10
11 # p.add_option("-r",action="store",dest="minsize",default='0',help="minimum size of dirs considered; can use suffixes k,M,G for multiples of 1000**{1,2,3} bytes")
12 # p.add_option("--qdel",action="store_true",dest="deljobs",help="delete jobs for which TMPDIR is larger than MINSIZE",default=False)
13
14 p.add_option("-r",action="store",dest="ranges",
15 help="comma-separated list of ranges to plot. Supported: hour, day, week, month, year, alltime",
16 default="day")
17 p.add_option("--rank-only",action="store_true",dest="rankonly",
18 help="don't plot, just print ranking of groups",default=False)
19
20 debug = 0
21
22 opts, args = p.parse_args()
23
24 timetags = opts.ranges.split(",")
25 import os
26
27 DATADIR=os.environ['HOME'] + '/ndpfdata/'
28 PLOTDIR='/www/grid/stats/groupviews/mcpool/'
29
30
31 import ndpf_gv_plots_common as plots
32
33 datafile_dict_run = {
34 'unused' : ('running', DATADIR + 'unused.mcpool.rrd'),
35 }
36
37 import glob
38 running_files = glob.glob(DATADIR+'*.running.mcpool.rrd')
39 for db in running_files:
40 group = db[len(DATADIR):db.find('.running.mcpool.rrd')]
41 datafile_dict_run[group] = ( 'running', db)
42
43 datafile_dict_cap = { 'capacity' : ('running', DATADIR + 'capacity.mcpool.rrd') }
44
45 tmpk = datafile_dict_run.keys()
46 tmpk.remove('unused')
47 tmpk.remove('nonmc')
48
49 datafile_dict_que = dict()
50 for k in tmpk:
51 datafile_dict_que[k] = ('queued', DATADIR + k + '.queued.mcpool.rrd')
52
53 tp = plots.Plot()
54
55 # 8 class qualitative paired color scheme
56
57 colors = [ "#A6CEE3", "#1F77B4", "#B2DF8A", "#33A02C",
58 "#FB9A99", "#E31A1C", "#FDBF6F", "#FF7F00" ]
59
60 tp.colors = { 'unused' : plots.colors['stdgray'],
61 'nonmc' : plots.colors['lightgreen'],
62 'capacity' : plots.colors['black']
63 }
64
65 tp.labels = dict()
66 for g in tp.colors:
67 tp.labels[g] = g
68
69 import sys
70 if opts.rankonly:
71 for t in timetags:
72 tgroup, pgroup = plots.getfracs(datafile_dict_run, datafile_dict_cap['capacity'],
73 plots.pdefs[t]['range'],
74 plots.pdefs[t]['res'])
75 groups_sorted = sorted(tgroup, key=tgroup.get, reverse=True) # decreasing useage towards end of list
76 idx = 0
77 print " group av.run %pool"
78 print " ----- ------ -----"
79 for k in groups_sorted:
80 idx += 1
81 print "%2d. %8s %5.1f %5.2f" % (idx, k, tgroup[k], pgroup[k])
82 sys.exit(0)
83
84 tp.labels = dict()
85 tp.colors = dict()
86
87 for g in tp.colors:
88 tp.labels[g] = g
89
90 active_groups = list()
91
92 for t in timetags:
93 # first pass: just get list of all relevant groups for this time range
94 grouplist_timetag = list()
95 tgroup, pgroup = plots.getfracs(datafile_dict_run, datafile_dict_cap['capacity'],
96 plots.pdefs[t]['range'],
97 plots.pdefs[t]['res'])
98 for k in pgroup.keys():
99 if pgroup[k] > 0.01 or k == 'unused':
100 grouplist_timetag.append(k)
101
102 qgroup = plots.getqueued(datafile_dict_que,
103 plots.pdefs[t]['range'],
104 plots.pdefs[t]['res'])
105
106 for k in qgroup.keys():
107 if qgroup[k] > 0.001:
108 if k not in grouplist_timetag:
109 grouplist_timetag.append(k)
110
111
112 for qr in ['running', 'queued']:
113 if qr == 'running':
114 tgroup, pgroup = plots.getfracs(datafile_dict_run, datafile_dict_cap['capacity'],
115 plots.pdefs[t]['range'],
116 plots.pdefs[t]['res'])
117 tgroup_seen = dict((k, tgroup[k]) for k in grouplist_timetag)
118 groups_sorted = sorted(tgroup_seen, key=tgroup_seen.get, reverse=False) # increasing useage towards end of list
119
120 # first assign labels, including all groups
121
122 for g in groups_sorted:
123 tp.labels[g] = "%-8s (%4.1f)" % (g, pgroup[g])
124
125 # now remove standard color groups nonmc, unused and assign colors to rest.
126
127 if 'nonmc' in groups_sorted:
128 groups_sorted.remove('nonmc')
129 if 'unused' in groups_sorted:
130 groups_sorted.remove('unused')
131
132 idx = len(groups_sorted)
133 for g in groups_sorted:
134 idx -= 1
135 tp.colors[g] = plots.colors[plots.colormap[idx]]
136
137 # now add them back so that they appear on the plots, in the right location
138
139 if 'nonmc' in grouplist_timetag:
140 groups_sorted.append('nonmc')
141 if 'unused' in grouplist_timetag:
142 groups_sorted.append('unused')
143
144 if qr == 'running':
145 tp.datafile = datafile_dict_run.copy()
146 tp.datafile.update(datafile_dict_cap)
147 tp.lines = ['capacity' ]
148 tp.bands = groups_sorted
149 tp.colors['unused'] = plots.colors['stdgray']
150 tp.colors['nonmc'] = plots.colors['lightgreen']
151 tp.colors['capacity'] = plots.colors['black']
152 tp.labels['capacity'] = 'capacity'
153 else:
154 tp.datafile = datafile_dict_que.copy()
155 tp.datafile.update(datafile_dict_que)
156 tp.lines = [ ]
157 tp.bands = groups_sorted
158 if 'nonmc' in tp.bands:
159 tp.bands.remove('nonmc')
160 tp.bands.remove('unused')
161
162 for s in ['small', 'large']:
163
164 tp.width = plots.pdefs[t][s]['width']
165 tp.height = plots.pdefs[t][s]['height']
166 tp.opts = plots.pdefs[t][s]['opts']
167 tp.res = plots.pdefs[t]['res']
168 tp.range = plots.pdefs[t]['range']
169 tp.timeargs = plots.pdefs[t]['timeargs']
170 tp.plotfile = '%s%s-%s-%s-%s.png' % (PLOTDIR, 'mcpool', qr, t, s)
171
172 tp.generate()
173

Properties

Name Value
svn:executable *

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28