/[pdpsoft]/nl.nikhef.ndpf.3maand/trunk/3maand.py
ViewVC logotype

Annotation of /nl.nikhef.ndpf.3maand/trunk/3maand.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2685 - (hide annotations) (download) (as text)
Sun Dec 15 16:14:36 2013 UTC (8 years, 8 months ago) by templon
File MIME type: application/x-python
File size: 12528 byte(s)
move computer to correct place (gridstaff)

1 templon 2447 #!/usr/bin/env python
2 templon 2448 # $Id$
3     # Source: $URL$
4 templon 2447 # J. A. Templon, NIKHEF/PDP 2011
5    
6     # script to do yearly WLCG accounting and format it like I want.
7     # note : sorting logic is based on a by-month grouping; if you change that,
8     # you'll need to change some of the python code.
9    
10 templon 2448 # constants to be changed each year
11    
12 templon 2628 pledges = { # pledges in SI06 units
13 templon 2679 'alice' : 3210,
14     'atlas' : 30598,
15     'lhcb' : 15393
16 templon 2448 }
17 templon 2679 total_cap = 81619
18 templon 2448
19 templon 2447 import optparse
20    
21 templon 2455 usage = "usage: %prog -p DBpwd start-date [end-date]\n" + \
22     "date format yyyy-mm-dd; default end date is 3 months after start date"
23    
24 templon 2447 p = optparse.OptionParser(description="Program to generate a CSV dump of computing usage " + \
25 templon 2455 " by WLCG over a specified period",usage=usage)
26 templon 2447
27     # dbpassword is mandatory
28     # start date is mandatory
29     # end date is optional; proving nothing means end date is 3 months after start
30     # end date can be provided in format yyyy-mm-dd or as +3m (for 3 months later than start)
31    
32     p.add_option("-p",action="store",dest="dbpassw",default=None,
33     help="password for NDPF accounting DB")
34     debug = 0
35    
36     opts, args = p.parse_args()
37    
38     import sys
39     if not opts.dbpassw :
40     sys.stderr.write("Error: password to NDPF accounting DB must be provided with -p\n")
41     sys.stderr.write("Bailing out.\n")
42     sys.exit(1)
43    
44     if len(args) < 1:
45     sys.stderr.write("Error: no date argument detected. A start date must be provided.\n")
46 templon 2455 sys.stderr.write("Format: yyyy-mm-dd\n")
47 templon 2447 sys.stderr.write("Bailing out.\n")
48     sys.exit(1)
49    
50     start_ascii = args[0]
51     import datetime
52     def parsedate(s):
53     try:
54     farr=s.split("-")
55     iarr = list()
56     for f in farr:
57     iarr.append(int(f))
58     return datetime.date(iarr[0],iarr[1],iarr[2])
59     except:
60     sys.stderr.write("Error parsing date string " + s + "\n")
61     raise
62    
63     if len(args) > 1:
64     end_ascii = args[1]
65     else:
66     end_ascii = None
67    
68     SDATE = parsedate(start_ascii) # starting date; script logic assumes start of month
69     if not end_ascii:
70     nmon = 3
71     elif end_ascii[:1] == '+':
72     if end_ascii[-1:] != 'm':
73     sys.stderr.write("error in end date string. accepted formats are"+
74     " 2011-05-27 or +3m\n")
75     sys.exit(1)
76     nmon = int(end_ascii[1:end_ascii.index('m')])
77     else:
78     nmon = None
79    
80     if nmon:
81     eyear = SDATE.year
82     emonth = SDATE.month + nmon
83     if emonth > 12:
84     emonth = emonth - 12
85     eyear = eyear + 1
86     EDATE = datetime.date(eyear,emonth,SDATE.day)
87     else:
88     EDATE = parsedate(end_ascii) # starting date; script logic assumes start of month
89    
90     print "generating data for jobs between", SDATE, "and", EDATE
91     vos = ['alice', 'atlas', 'lhcb']
92     groups = {
93 templon 2654 'alice': ['palice', 'alicesgm', 'alicepil'],
94 templon 2447 'atlas': ['atlb', 'atlaspil', 'patlas', 'atlsgm'],
95     'lhcb' : ['lhcb', 'lhcbpil', 'lhcbprd', 'lhcbsgm']
96     }
97    
98     ACCBASECMD = ['/usr/local/bin/accuse']
99     ACCBASEARG = ('--byendtime -dbpasswd ' + opts.dbpassw + ' -f lcg -m').split()
100    
101     import subprocess
102    
103     perlout=dict()
104    
105     # gather raw accuse output
106    
107     for vo in vos:
108     groupargs = []
109     for g in groups[vo]:
110     groupargs += ["-g",g]
111     args = ACCBASECMD + ACCBASEARG + ["-s", SDATE.isoformat(), "-e", EDATE.isoformat()] + groupargs
112     perlout[vo] = subprocess.Popen(args, stdout=subprocess.PIPE).communicate()[0]
113    
114     def hms2dec(str):
115     h,m,s = str.split(':')
116     return float(h) + (float(m) + float(s)/60.)/60.
117    
118     # parse output
119     # results in data structure like this
120    
121 templon 2628 # data = parsed['alice']['2010-02']['si06days']
122 templon 2447
123     import re
124     mpatt = re.compile(r'20[012][0-9]-[01][0-9]')
125     parsed = dict()
126     for vo in vos:
127     parsed[vo] = dict()
128     lines = perlout[vo].split('\n')
129     for line in lines:
130     if mpatt.match(line) or line.find('Summed') == 0:
131     fields = line.split()
132     cpu = hms2dec(fields[1])
133     wall = hms2dec(fields[2])
134 templon 2628 si06days = float(fields[4]) / 0.185
135 templon 2447 njobs = int(fields[5])
136     parsed[vo][fields[0]] = { 'cpu' : cpu, 'wall': wall,
137 templon 2628 'si06days': si06days, 'njobs' : njobs }
138 templon 2447
139 templon 2628 hepvos = [ 'alice', 'atlas', 'lhcb', 'auger', 'cms', 'geant', 'dzero', 'xenon', 'gear' ]
140     othervos = [ 'biomed','dans','enmr.eu','esr', 'lofar', 'lsgrid', 'ncf', 'bbmri.nl', 'vlemed',
141     'tutor', 'phicos', 'chem.biggrid.nl' ]
142 templon 2685 local = [ 'nikatlas', 'niklhcb', 'astrop', 'virgo', 'antares' ]
143 templon 2628 admin = [ 'gridstaff' ]
144     probes = [ 'ops', 'ops.nl', 'p4', 'dteam']
145    
146     groups = {
147 templon 2654 'alice': ['palice', 'alicesgm', 'alicepil'],
148 templon 2628 'atlas': ['atlb', 'atlaspil', 'patlas', 'atlsgm'],
149     'lhcb' : ['lhcb', 'lhcbpil', 'lhcbprd', 'lhcbsgm'],
150     'gear' : ['poola' ],
151     'auger' : ['auger', 'augsgm'],
152     'cms' : ['cms'],
153     'geant' : ['geant'],
154     'dzero' : ['pdzero'],
155     'xenon' : ['pxenon', 'pxenonsm'],
156     'biomed' : ['biome','biomesgm'],
157     'dans' : ['dans'],
158     'enmr.eu' : ['enmr', 'enmrsgm'],
159     'esr' : ['esr'],
160     'lofar' : ['lofar','plofarsm'],
161     'lsgrid' : ['lsgrid'],
162     'ncf' : [ 'ncf'],
163     'tutor' : [ 'tutor' ],
164     'ops' : [ 'ops', 'opspil'],
165     'phicos' : [ 'phicos','phicosgm' ],
166     'bbmri.nl' : [ 'pbbmri','poolb' ],
167     'chem.biggrid.nl' : [ 'pbchem' ],
168 templon 2630 'drihm' : [ 'pdrihm' ],
169 templon 2628 'ops.nl' : ['popsnl'],
170     'p4' : [ 'pvier', 'pviersm' ],
171     'vlemed' : [ 'vlemed', 'vlemedsm' ],
172     'dteam' : ['dteam' ],
173     'nikatlas' : ['atlas'],
174     'niklhcb' : [ 'bfys'],
175     'astrop' : [ 'astrop' ],
176     'antares' : [ 'antares' ],
177     'virgo' : [ 'virgo' ],
178 templon 2685 'gridstaff' : ['datagrid','emin', 'tbadmin', 'computer']
179 templon 2628 }
180    
181     # gather all unix groups, for check later that we haven't missed anything
182    
183     allunixgroups = [ ]
184     for g in groups.keys():
185     allunixgroups.extend(groups[g])
186    
187     basecmd = 'mysql -u anon -p' + opts.dbpassw + ' -h bedstee.nikhef.nl accounting'
188    
189     ACCSELECT = "select sum(job.WallGHzEquivalent),groupid.id,count(job.JobID) from job,groupid where job.groupid=groupid.UniqueID and job.EndTime >= " \
190     + "'" + SDATE.isoformat() + "' and job.EndTime <= '" + EDATE.isoformat() + "' group by groupid.id;"
191    
192     import subprocess
193    
194     p = subprocess.Popen(basecmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
195     out, err = p.communicate(ACCSELECT)
196    
197     lines = out.split('\n')
198     resdict = dict()
199    
200     for l in lines[1:-1]:
201     f = l.split()
202     resdict[f[1]] = float(f[0])
203    
204     for k in resdict.keys():
205     if k not in allunixgroups:
206     print
207     print "unknown unix group " + k + " found in mysql output, please check"
208     print
209    
210     sums = dict()
211     for g in groups:
212     ts = 0
213     for ug in groups[g]:
214     if ug in resdict.keys():
215     ts = ts + resdict[ug]
216     sums[g] = ts
217    
218     sumothervos = 0
219     for g in othervos:
220     sumothervos = sumothervos + sums[g]
221    
222     sumlocal = 0
223     for g in local:
224     sumlocal = sumlocal + sums[g]
225    
226     sumprobes = 0
227     for g in probes:
228     sumprobes = sumprobes + sums[g]
229    
230     sumadmin = 0
231     for g in admin:
232     sumadmin = sumadmin + sums[g]
233    
234     class Table(object):
235     def __init__(self, header, columnheaders):
236     self.header = header
237     self.columnheaders = columnheaders
238     self.data = dict()
239     def add(self, name, val):
240     self.data[name] = val
241    
242 templon 2679 heptab = Table("Usage @ Nikhef from " + SDATE.isoformat() + " to " + EDATE.isoformat(),
243 templon 2628 ["group", "WallGHzEquiv"])
244     for g in hepvos:
245     heptab.add(g, sums[g])
246     heptab.add("local", sumlocal)
247    
248 templon 2679 othertab = Table("Usage @ Nikhef from " + SDATE.isoformat() + " to " + EDATE.isoformat(),
249 templon 2628 ["group", "WallGHzEquiv"])
250    
251     for g in othervos:
252     othertab.add(g, sums[g])
253     othertab.add("probes", sumprobes)
254     othertab.add("admin", sumadmin)
255    
256 templon 2447 # output generation ... big damn csv file
257    
258     import csv
259     writer = csv.writer(open('tmp.csv', 'wb'), delimiter=',',
260     quotechar='|', quoting=csv.QUOTE_MINIMAL)
261    
262     ONEDAY=datetime.timedelta(days=1)
263    
264     # per-VO segment
265    
266     for vo in vos:
267     writer.writerow(["Data for",vo])
268 templon 2628 writer.writerow(["Month", "si06days.used", "cpu/wall", "pledged", "jobs", "days"])
269 templon 2447
270     monthstart = SDATE
271    
272     while monthstart < EDATE:
273    
274     if monthstart.month < 12 :
275     startnextmonth = monthstart.replace(month=monthstart.month+1)
276     else:
277     d1 = monthstart.replace(month=1)
278     startnextmonth = d1.replace(year=monthstart.year+1)
279     monthend = startnextmonth - ONEDAY
280     if monthend > EDATE:
281     monthend = EDATE
282     ndays = (monthend - monthstart + ONEDAY).days
283     monthkey = monthstart.isoformat()[:7]
284     if monthkey not in parsed[vo].keys():
285     parsed[vo][monthkey] = {
286 templon 2628 'si06days' : 0, 'cpu' : 0.1, 'wall' : 0.1, 'njobs' : 0
287 templon 2447 }
288     td = parsed[vo][monthkey]
289     writer.writerow([monthkey,
290 templon 2628 td['si06days'],
291 templon 2447 td['cpu']/td['wall'],
292     ndays * pledges[vo],
293     td['njobs'],
294     ndays
295     ])
296     monthstart = monthend + ONEDAY
297    
298     writer.writerow([' ',' '])
299    
300     # usage plots
301    
302 templon 2628 writer.writerow(["si06 days used per VO"])
303 templon 2447 writer.writerow(["Month", "lhcb.used", "atlas.used", "alice.used"])
304     monthstart = SDATE
305     while monthstart < EDATE:
306     if monthstart.month < 12 :
307     startnextmonth = monthstart.replace(month=monthstart.month+1)
308     else:
309     d1 = monthstart.replace(month=1)
310     startnextmonth = d1.replace(year=monthstart.year+1)
311     monthend = startnextmonth - ONEDAY
312     if monthend > EDATE:
313     monthend = EDATE
314     ndays = (monthend - monthstart + ONEDAY).days
315     monthkey = monthstart.isoformat()[:7]
316     td = parsed[vo][monthkey]
317     writer.writerow([monthkey,
318 templon 2628 parsed['lhcb'][monthkey]['si06days'],
319     parsed['atlas'][monthkey]['si06days'],
320     parsed['alice'][monthkey]['si06days']
321 templon 2447 ])
322     monthstart = monthend + ONEDAY
323    
324     writer.writerow([' ',' '])
325    
326     # pledge fraction plots
327    
328     writer.writerow(["pledge fraction used per VO"])
329     writer.writerow(["Month", "lhcb.frac", "atlas.frac", "alice.frac"])
330     monthstart = SDATE
331     while monthstart < EDATE:
332     if monthstart.month < 12 :
333     startnextmonth = monthstart.replace(month=monthstart.month+1)
334     else:
335     d1 = monthstart.replace(month=1)
336     startnextmonth = d1.replace(year=monthstart.year+1)
337     monthend = startnextmonth - ONEDAY
338     if monthend > EDATE:
339     monthend = EDATE
340     ndays = (monthend - monthstart + ONEDAY).days
341     monthkey = monthstart.isoformat()[:7]
342     td = parsed[vo][monthkey]
343     writer.writerow([monthkey,
344 templon 2628 parsed['lhcb'][monthkey]['si06days']/(pledges['lhcb']*ndays),
345     parsed['atlas'][monthkey]['si06days']/(pledges['atlas']*ndays),
346     parsed['alice'][monthkey]['si06days']/(pledges['alice']*ndays)
347 templon 2447 ])
348     monthstart = monthend + ONEDAY
349    
350     writer.writerow([' ',' '])
351    
352     # cpu eff plots
353    
354     writer.writerow(["ratio cpu to wall time used (eff) per VO"])
355     writer.writerow(["Month", "lhcb.eff", "atlas.eff", "alice.eff"])
356     monthstart = SDATE
357     while monthstart < EDATE:
358     if monthstart.month < 12 :
359     startnextmonth = monthstart.replace(month=monthstart.month+1)
360     else:
361     d1 = monthstart.replace(month=1)
362     startnextmonth = d1.replace(year=monthstart.year+1)
363     monthend = startnextmonth - ONEDAY
364     if monthend > EDATE:
365     monthend = EDATE
366     ndays = (monthend - monthstart + ONEDAY).days
367     monthkey = monthstart.isoformat()[:7]
368     td = parsed[vo][monthkey]
369     writer.writerow([monthkey,
370     parsed['lhcb'][monthkey]['cpu']/parsed['lhcb'][monthkey]['wall'],
371     parsed['atlas'][monthkey]['cpu']/parsed['atlas'][monthkey]['wall'],
372     parsed['alice'][monthkey]['cpu']/parsed['alice'][monthkey]['wall'],
373     ])
374     monthstart = monthend + ONEDAY
375    
376     writer.writerow([' ',' '])
377 templon 2448
378 templon 2628 writer.writerow(["Pledges and capacties (SI06)"])
379 templon 2448 writer.writerow(["Group", "power"])
380     writer.writerow(["ATLAS", pledges["atlas"]])
381     writer.writerow(["LHCb", pledges["lhcb"] ])
382     writer.writerow(["ALICE", pledges["alice"]])
383     writer.writerow(["farmcap", total_cap])
384 templon 2628 writer.writerow([' ',' '])
385 templon 2448
386 templon 2679 first_tabl = True
387 templon 2628 for tabl in [heptab, othertab]:
388 templon 2679 if first_tabl:
389     writer.writerow([tabl.header])
390     writer.writerow(tabl.columnheaders)
391     first_tabl = False
392 templon 2628
393     groups_sorted = sorted(tabl.data, key=tabl.data.get, reverse=True)
394     for g in groups_sorted:
395     writer.writerow([g, tabl.data[g]])
396    
397    

Properties

Name Value
svn:executable *
svn:keywords Id URL

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28