nl.nikhef.ndpf.groupviews/trunk/ndpf-mc-mkplots

#! /usr/bin/env python
# $Id: ndpf-gv-mkplots 2688 2014-01-21 16:20:14Z templon $
# Source: $URL: svn+ssh://svn@ndpfsvn.nikhef.nl/repos/pdpsoft/nl.nikhef.ndpf.groupviews/trunk/ndpf-gv-mkplots $
# J. A. Templon, NIKHEF/PDP 2011

import optparse

p = optparse.OptionParser(description="Program to make rrdtool plots " + \
                          "of performance of multicore pool")

# p.add_option("-r",action="store",dest="minsize",default='0',help="minimum size of dirs considered; can use suffixes k,M,G for multiples of 1000**{1,2,3} bytes")
# p.add_option("--qdel",action="store_true",dest="deljobs",help="delete jobs for which TMPDIR is larger than MINSIZE",default=False)

p.add_option("-r",action="store",dest="ranges",
             help="comma-separated list of ranges to plot. Supported: hour, day, week, month, year, alltime",
             default="day")
p.add_option("--rank-only",action="store_true",dest="rankonly",
             help="don't plot, just print ranking of groups",default=False)

debug = 0

opts, args = p.parse_args()

timetags = opts.ranges.split(",")
import os

DATADIR=os.environ['HOME'] + '/ndpfdata/'
PLOTDIR='/www/grid/stats/groupviews/mcpool/'


import ndpf_gv_plots_common as plots

datafile_dict_run = {
    'unused'   : ('running', DATADIR + 'unused.mcpool.rrd'),
    }

import glob
running_files = glob.glob(DATADIR+'*.running.mcpool.rrd')
for db in running_files:
    group = db[len(DATADIR):db.find('.running.mcpool.rrd')]
    datafile_dict_run[group] = ( 'running', db)

datafile_dict_cap = { 'capacity' : ('running', DATADIR + 'capacity.mcpool.rrd') }

tmpk = datafile_dict_run.keys()
tmpk.remove('unused')
tmpk.remove('nonmc')

datafile_dict_que = dict()
for k in tmpk:
    datafile_dict_que[k] = ('queued', DATADIR + k + '.queued.mcpool.rrd')

tp = plots.Plot()

# 8 class qualitative paired color scheme

colors = [ "#A6CEE3",  "#1F77B4",  "#B2DF8A",  "#33A02C",
           "#FB9A99",  "#E31A1C",  "#FDBF6F",  "#FF7F00"    ]

tp.colors = { 'unused' : plots.colors['stdgray'],
            'nonmc'    : plots.colors['lightgreen'],
            'capacity' : plots.colors['black']
            }

tp.labels = dict()
for g in tp.colors:
    tp.labels[g] = g

import sys
if opts.rankonly:
    for t in timetags: 
        tgroup, pgroup = plots.getfracs(datafile_dict_run, datafile_dict_cap['capacity'],
                                        plots.pdefs[t]['range'],
                                        plots.pdefs[t]['res'])
        groups_sorted = sorted(tgroup, key=tgroup.get, reverse=True) # decreasing useage towards end of list
        idx = 0
        print "      group   av.run   %pool"
        print "      -----   ------   -----"
        for k in groups_sorted:
            idx += 1
            print "%2d. %8s   %5.1f   %5.2f" % (idx, k, tgroup[k], pgroup[k])
    sys.exit(0)
    
tp.labels = dict()
tp.colors = dict()

for g in tp.colors:
    tp.labels[g] = g

active_groups = list()

for t in timetags:
# first pass: just get list of all relevant groups for this time range
    grouplist_timetag = list()
    tgroup, pgroup = plots.getfracs(datafile_dict_run, datafile_dict_cap['capacity'],
                                        plots.pdefs[t]['range'],
                                        plots.pdefs[t]['res'])
    for k in pgroup.keys():
        if pgroup[k] > 0.01 or k == 'unused':
            grouplist_timetag.append(k)

    qgroup = plots.getqueued(datafile_dict_que,
                             plots.pdefs[t]['range'],
                             plots.pdefs[t]['res'])

    for k in qgroup.keys():
        if qgroup[k] > 0.001:
            if k not in grouplist_timetag:
                grouplist_timetag.append(k)
    
    
    for qr in ['running', 'queued']:
        if qr == 'running':
            tgroup, pgroup = plots.getfracs(datafile_dict_run, datafile_dict_cap['capacity'],
                                        plots.pdefs[t]['range'],
                                        plots.pdefs[t]['res'])
            tgroup_seen = dict((k, tgroup[k]) for k in grouplist_timetag)
            groups_sorted = sorted(tgroup_seen, key=tgroup_seen.get, reverse=False) # increasing useage towards end of list

            # first assign labels, including all groups
            
            for g in groups_sorted:
                tp.labels[g] = "%-8s (%4.1f)" % (g, pgroup[g])
            
            # now remove standard color groups nonmc, unused and assign colors to rest.
            
            if 'nonmc' in groups_sorted:
                groups_sorted.remove('nonmc')
            if 'unused' in groups_sorted:
                groups_sorted.remove('unused')
            
            idx = len(groups_sorted)
            for g in groups_sorted:
                idx -= 1
                tp.colors[g] = plots.colors[plots.colormap[idx]]

            # now add them back so that they appear on the plots, in the right location
            
            if 'nonmc' in grouplist_timetag:
                groups_sorted.append('nonmc')
            if 'unused' in grouplist_timetag:
                groups_sorted.append('unused')

        if qr == 'running':
            tp.datafile = datafile_dict_run.copy()
            tp.datafile.update(datafile_dict_cap)
            tp.lines = ['capacity' ]
            tp.bands = groups_sorted
            tp.colors['unused']   = plots.colors['stdgray']
            tp.colors['nonmc']    = plots.colors['lightgreen']
            tp.colors['capacity'] = plots.colors['black']
            tp.labels['capacity'] = 'capacity'
        else:
            tp.datafile = datafile_dict_que.copy()
            tp.datafile.update(datafile_dict_que)
            tp.lines = [ ]
            tp.bands = groups_sorted
            if 'nonmc' in tp.bands:
                tp.bands.remove('nonmc')
            tp.bands.remove('unused')

        for s in ['small', 'large']:

            tp.width    = plots.pdefs[t][s]['width']
            tp.height   = plots.pdefs[t][s]['height']
            tp.opts     = plots.pdefs[t][s]['opts']
            tp.res      = plots.pdefs[t]['res']
            tp.range    = plots.pdefs[t]['range']
            tp.timeargs = plots.pdefs[t]['timeargs']
            tp.plotfile = '%s%s-%s-%s-%s.png' % (PLOTDIR, 'mcpool', qr, t, s)
            
            tp.generate()
            
1	#! /usr/bin/env python
2	# $Id: ndpf-gv-mkplots 2688 2014-01-21 16:20:14Z templon $
3	# Source: $URL: svn+ssh://svn@ndpfsvn.nikhef.nl/repos/pdpsoft/nl.nikhef.ndpf.groupviews/trunk/ndpf-gv-mkplots $
4	# J. A. Templon, NIKHEF/PDP 2011
5
6	import optparse
7
8	p = optparse.OptionParser(description="Program to make rrdtool plots " + \
9	"of performance of multicore pool")
10
11	# p.add_option("-r",action="store",dest="minsize",default='0',help="minimum size of dirs considered; can use suffixes k,M,G for multiples of 1000**{1,2,3} bytes")
12	# p.add_option("--qdel",action="store_true",dest="deljobs",help="delete jobs for which TMPDIR is larger than MINSIZE",default=False)
13
14	p.add_option("-r",action="store",dest="ranges",
15	help="comma-separated list of ranges to plot. Supported: hour, day, week, month, year, alltime",
16	default="day")
17	p.add_option("--rank-only",action="store_true",dest="rankonly",
18	help="don't plot, just print ranking of groups",default=False)
19
20	debug = 0
21
22	opts, args = p.parse_args()
23
24	timetags = opts.ranges.split(",")
25	import os
26
27	DATADIR=os.environ['HOME'] + '/ndpfdata/'
28	PLOTDIR='/www/grid/stats/groupviews/mcpool/'
29
30
31	import ndpf_gv_plots_common as plots
32
33	datafile_dict_run = {
34	'unused' : ('running', DATADIR + 'unused.mcpool.rrd'),
35	}
36
37	import glob
38	running_files = glob.glob(DATADIR+'*.running.mcpool.rrd')
39	for db in running_files:
40	group = db[len(DATADIR):db.find('.running.mcpool.rrd')]
41	datafile_dict_run[group] = ( 'running', db)
42
43	datafile_dict_cap = { 'capacity' : ('running', DATADIR + 'capacity.mcpool.rrd') }
44
45	tmpk = datafile_dict_run.keys()
46	tmpk.remove('unused')
47	tmpk.remove('nonmc')
48
49	datafile_dict_que = dict()
50	for k in tmpk:
51	datafile_dict_que[k] = ('queued', DATADIR + k + '.queued.mcpool.rrd')
52
53	tp = plots.Plot()
54
55	# 8 class qualitative paired color scheme
56
57	colors = [ "#A6CEE3", "#1F77B4", "#B2DF8A", "#33A02C",
58	"#FB9A99", "#E31A1C", "#FDBF6F", "#FF7F00" ]
59
60	tp.colors = { 'unused' : plots.colors['stdgray'],
61	'nonmc' : plots.colors['lightgreen'],
62	'capacity' : plots.colors['black']
63	}
64
65	tp.labels = dict()
66	for g in tp.colors:
67	tp.labels[g] = g
68
69	import sys
70	if opts.rankonly:
71	for t in timetags:
72	tgroup, pgroup = plots.getfracs(datafile_dict_run, datafile_dict_cap['capacity'],
73	plots.pdefs[t]['range'],
74	plots.pdefs[t]['res'])
75	groups_sorted = sorted(tgroup, key=tgroup.get, reverse=True) # decreasing useage towards end of list
76	idx = 0
77	print " group av.run %pool"
78	print " ----- ------ -----"
79	for k in groups_sorted:
80	idx += 1
81	print "%2d. %8s %5.1f %5.2f" % (idx, k, tgroup[k], pgroup[k])
82	sys.exit(0)
83
84	tp.labels = dict()
85	tp.colors = dict()
86
87	for g in tp.colors:
88	tp.labels[g] = g
89
90	active_groups = list()
91
92	for t in timetags:
93	# first pass: just get list of all relevant groups for this time range
94	grouplist_timetag = list()
95	tgroup, pgroup = plots.getfracs(datafile_dict_run, datafile_dict_cap['capacity'],
96	plots.pdefs[t]['range'],
97	plots.pdefs[t]['res'])
98	for k in pgroup.keys():
99	if pgroup[k] > 0.01 or k == 'unused':
100	grouplist_timetag.append(k)
101
102	qgroup = plots.getqueued(datafile_dict_que,
103	plots.pdefs[t]['range'],
104	plots.pdefs[t]['res'])
105
106	for k in qgroup.keys():
107	if qgroup[k] > 0.001:
108	if k not in grouplist_timetag:
109	grouplist_timetag.append(k)
110
111
112	for qr in ['running', 'queued']:
113	if qr == 'running':
114	tgroup, pgroup = plots.getfracs(datafile_dict_run, datafile_dict_cap['capacity'],
115	plots.pdefs[t]['range'],
116	plots.pdefs[t]['res'])
117	tgroup_seen = dict((k, tgroup[k]) for k in grouplist_timetag)
118	groups_sorted = sorted(tgroup_seen, key=tgroup_seen.get, reverse=False) # increasing useage towards end of list
119
120	# first assign labels, including all groups
121
122	for g in groups_sorted:
123	tp.labels[g] = "%-8s (%4.1f)" % (g, pgroup[g])
124
125	# now remove standard color groups nonmc, unused and assign colors to rest.
126
127	if 'nonmc' in groups_sorted:
128	groups_sorted.remove('nonmc')
129	if 'unused' in groups_sorted:
130	groups_sorted.remove('unused')
131
132	idx = len(groups_sorted)
133	for g in groups_sorted:
134	idx -= 1
135	tp.colors[g] = plots.colors[plots.colormap[idx]]
136
137	# now add them back so that they appear on the plots, in the right location
138
139	if 'nonmc' in grouplist_timetag:
140	groups_sorted.append('nonmc')
141	if 'unused' in grouplist_timetag:
142	groups_sorted.append('unused')
143
144	if qr == 'running':
145	tp.datafile = datafile_dict_run.copy()
146	tp.datafile.update(datafile_dict_cap)
147	tp.lines = ['capacity' ]
148	tp.bands = groups_sorted
149	tp.colors['unused'] = plots.colors['stdgray']
150	tp.colors['nonmc'] = plots.colors['lightgreen']
151	tp.colors['capacity'] = plots.colors['black']
152	tp.labels['capacity'] = 'capacity'
153	else:
154	tp.datafile = datafile_dict_que.copy()
155	tp.datafile.update(datafile_dict_que)
156	tp.lines = [ ]
157	tp.bands = groups_sorted
158	if 'nonmc' in tp.bands:
159	tp.bands.remove('nonmc')
160	tp.bands.remove('unused')
161
162	for s in ['small', 'large']:
163
164	tp.width = plots.pdefs[t][s]['width']
165	tp.height = plots.pdefs[t][s]['height']
166	tp.opts = plots.pdefs[t][s]['opts']
167	tp.res = plots.pdefs[t]['res']
168	tp.range = plots.pdefs[t]['range']
169	tp.timeargs = plots.pdefs[t]['timeargs']
170	tp.plotfile = '%s%s-%s-%s-%s.png' % (PLOTDIR, 'mcpool', qr, t, s)
171
172	tp.generate()
173
grid.support@nikhef.nl	ViewVC Help
Powered by ViewVC 1.1.28