/[pdpsoft]/nl.nikhef.ndpf.3maand/trunk/3maand.py
ViewVC logotype

Contents of /nl.nikhef.ndpf.3maand/trunk/3maand.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2629 - (show annotations) (download) (as text)
Wed Jun 5 12:48:39 2013 UTC (9 years ago) by templon
File MIME type: application/x-python
File size: 12531 byte(s)
update with 2013 pledges and total capacity


1 #!/usr/bin/env python
2 # $Id$
3 # Source: $URL$
4 # J. A. Templon, NIKHEF/PDP 2011
5
6 # script to do yearly WLCG accounting and format it like I want.
7 # note : sorting logic is based on a by-month grouping; if you change that,
8 # you'll need to change some of the python code.
9
10 # constants to be changed each year
11
12 pledges = { # pledges in SI06 units
13 'alice' : 3209,
14 'atlas' : 30586,
15 'lhcb' : 15388
16 }
17 total_cap = 81718
18
19 import optparse
20
21 usage = "usage: %prog -p DBpwd start-date [end-date]\n" + \
22 "date format yyyy-mm-dd; default end date is 3 months after start date"
23
24 p = optparse.OptionParser(description="Program to generate a CSV dump of computing usage " + \
25 " by WLCG over a specified period",usage=usage)
26
27 # dbpassword is mandatory
28 # start date is mandatory
29 # end date is optional; proving nothing means end date is 3 months after start
30 # end date can be provided in format yyyy-mm-dd or as +3m (for 3 months later than start)
31
32 p.add_option("-p",action="store",dest="dbpassw",default=None,
33 help="password for NDPF accounting DB")
34 debug = 0
35
36 opts, args = p.parse_args()
37
38 import sys
39 if not opts.dbpassw :
40 sys.stderr.write("Error: password to NDPF accounting DB must be provided with -p\n")
41 sys.stderr.write("Bailing out.\n")
42 sys.exit(1)
43
44 if len(args) < 1:
45 sys.stderr.write("Error: no date argument detected. A start date must be provided.\n")
46 sys.stderr.write("Format: yyyy-mm-dd\n")
47 sys.stderr.write("Bailing out.\n")
48 sys.exit(1)
49
50 start_ascii = args[0]
51 import datetime
52 def parsedate(s):
53 try:
54 farr=s.split("-")
55 iarr = list()
56 for f in farr:
57 iarr.append(int(f))
58 return datetime.date(iarr[0],iarr[1],iarr[2])
59 except:
60 sys.stderr.write("Error parsing date string " + s + "\n")
61 raise
62
63 if len(args) > 1:
64 end_ascii = args[1]
65 else:
66 end_ascii = None
67
68 SDATE = parsedate(start_ascii) # starting date; script logic assumes start of month
69 if not end_ascii:
70 nmon = 3
71 elif end_ascii[:1] == '+':
72 if end_ascii[-1:] != 'm':
73 sys.stderr.write("error in end date string. accepted formats are"+
74 " 2011-05-27 or +3m\n")
75 sys.exit(1)
76 nmon = int(end_ascii[1:end_ascii.index('m')])
77 else:
78 nmon = None
79
80 if nmon:
81 eyear = SDATE.year
82 emonth = SDATE.month + nmon
83 if emonth > 12:
84 emonth = emonth - 12
85 eyear = eyear + 1
86 EDATE = datetime.date(eyear,emonth,SDATE.day)
87 else:
88 EDATE = parsedate(end_ascii) # starting date; script logic assumes start of month
89
90 print "generating data for jobs between", SDATE, "and", EDATE
91 vos = ['alice', 'atlas', 'lhcb']
92 groups = {
93 'alice': ['palice', 'alicesgm'],
94 'atlas': ['atlb', 'atlaspil', 'patlas', 'atlsgm'],
95 'lhcb' : ['lhcb', 'lhcbpil', 'lhcbprd', 'lhcbsgm']
96 }
97
98 ACCBASECMD = ['/usr/local/bin/accuse']
99 ACCBASEARG = ('--byendtime -dbpasswd ' + opts.dbpassw + ' -f lcg -m').split()
100
101 import subprocess
102
103 perlout=dict()
104
105 # gather raw accuse output
106
107 for vo in vos:
108 groupargs = []
109 for g in groups[vo]:
110 groupargs += ["-g",g]
111 args = ACCBASECMD + ACCBASEARG + ["-s", SDATE.isoformat(), "-e", EDATE.isoformat()] + groupargs
112 perlout[vo] = subprocess.Popen(args, stdout=subprocess.PIPE).communicate()[0]
113
114 def hms2dec(str):
115 h,m,s = str.split(':')
116 return float(h) + (float(m) + float(s)/60.)/60.
117
118 # parse output
119 # results in data structure like this
120
121 # data = parsed['alice']['2010-02']['si06days']
122
123 import re
124 mpatt = re.compile(r'20[012][0-9]-[01][0-9]')
125 parsed = dict()
126 for vo in vos:
127 parsed[vo] = dict()
128 lines = perlout[vo].split('\n')
129 for line in lines:
130 if mpatt.match(line) or line.find('Summed') == 0:
131 fields = line.split()
132 cpu = hms2dec(fields[1])
133 wall = hms2dec(fields[2])
134 si06days = float(fields[4]) / 0.185
135 njobs = int(fields[5])
136 parsed[vo][fields[0]] = { 'cpu' : cpu, 'wall': wall,
137 'si06days': si06days, 'njobs' : njobs }
138
139 hepvos = [ 'alice', 'atlas', 'lhcb', 'auger', 'cms', 'geant', 'dzero', 'xenon', 'gear' ]
140 othervos = [ 'biomed','dans','enmr.eu','esr', 'lofar', 'lsgrid', 'ncf', 'bbmri.nl', 'vlemed',
141 'tutor', 'phicos', 'chem.biggrid.nl' ]
142 local = [ 'nikatlas', 'niklhcb', 'astrop', 'virgo', 'antares' ]
143 admin = [ 'gridstaff' ]
144 probes = [ 'ops', 'ops.nl', 'p4', 'dteam']
145
146 groups = {
147 'alice': ['palice', 'alicesgm'],
148 'atlas': ['atlb', 'atlaspil', 'patlas', 'atlsgm'],
149 'lhcb' : ['lhcb', 'lhcbpil', 'lhcbprd', 'lhcbsgm'],
150 'gear' : ['poola' ],
151 'auger' : ['auger', 'augsgm'],
152 'cms' : ['cms'],
153 'geant' : ['geant'],
154 'dzero' : ['pdzero'],
155 'xenon' : ['pxenon', 'pxenonsm'],
156 'biomed' : ['biome','biomesgm'],
157 'dans' : ['dans'],
158 'enmr.eu' : ['enmr', 'enmrsgm'],
159 'esr' : ['esr'],
160 'lofar' : ['lofar','plofarsm'],
161 'lsgrid' : ['lsgrid'],
162 'ncf' : [ 'ncf'],
163 'tutor' : [ 'tutor' ],
164 'ops' : [ 'ops', 'opspil'],
165 'phicos' : [ 'phicos','phicosgm' ],
166 'bbmri.nl' : [ 'pbbmri','poolb' ],
167 'chem.biggrid.nl' : [ 'pbchem' ],
168 'ops.nl' : ['popsnl'],
169 'p4' : [ 'pvier', 'pviersm' ],
170 'vlemed' : [ 'vlemed', 'vlemedsm' ],
171 'dteam' : ['dteam' ],
172 'nikatlas' : ['atlas'],
173 'niklhcb' : [ 'bfys'],
174 'astrop' : [ 'astrop' ],
175 'antares' : [ 'antares' ],
176 'virgo' : [ 'virgo' ],
177 'gridstaff' : ['datagrid','emin', 'tbadmin']
178 }
179
180 # gather all unix groups, for check later that we haven't missed anything
181
182 allunixgroups = [ ]
183 for g in groups.keys():
184 allunixgroups.extend(groups[g])
185
186 basecmd = 'mysql -u anon -p' + opts.dbpassw + ' -h bedstee.nikhef.nl accounting'
187
188 ACCSELECT = "select sum(job.WallGHzEquivalent),groupid.id,count(job.JobID) from job,groupid where job.groupid=groupid.UniqueID and job.EndTime >= " \
189 + "'" + SDATE.isoformat() + "' and job.EndTime <= '" + EDATE.isoformat() + "' group by groupid.id;"
190
191 import subprocess
192
193 p = subprocess.Popen(basecmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
194 out, err = p.communicate(ACCSELECT)
195
196 lines = out.split('\n')
197 resdict = dict()
198
199 for l in lines[1:-1]:
200 f = l.split()
201 resdict[f[1]] = float(f[0])
202
203 for k in resdict.keys():
204 if k not in allunixgroups:
205 print
206 print "unknown unix group " + k + " found in mysql output, please check"
207 print
208
209 sums = dict()
210 for g in groups:
211 ts = 0
212 for ug in groups[g]:
213 if ug in resdict.keys():
214 ts = ts + resdict[ug]
215 sums[g] = ts
216
217 sumothervos = 0
218 for g in othervos:
219 sumothervos = sumothervos + sums[g]
220
221 sumlocal = 0
222 for g in local:
223 sumlocal = sumlocal + sums[g]
224
225 sumprobes = 0
226 for g in probes:
227 sumprobes = sumprobes + sums[g]
228
229 sumadmin = 0
230 for g in admin:
231 sumadmin = sumadmin + sums[g]
232
233 class Table(object):
234 def __init__(self, header, columnheaders):
235 self.header = header
236 self.columnheaders = columnheaders
237 self.data = dict()
238 def add(self, name, val):
239 self.data[name] = val
240
241 heptab = Table("HEP Usage @ Nikhef from " + SDATE.isoformat() + " to " + EDATE.isoformat(),
242 ["group", "WallGHzEquiv"])
243 for g in hepvos:
244 heptab.add(g, sums[g])
245 heptab.add("non-HEP",sumothervos)
246 heptab.add("local", sumlocal)
247 heptab.add("probes", sumprobes)
248 heptab.add("admin", sumadmin)
249
250 othertab = Table("non-HEP Usage @ Nikhef from " + SDATE.isoformat() + " to " + EDATE.isoformat(),
251 ["group", "WallGHzEquiv"])
252
253 for g in othervos:
254 othertab.add(g, sums[g])
255 othertab.add("probes", sumprobes)
256 othertab.add("admin", sumadmin)
257
258 # output generation ... big damn csv file
259
260 import csv
261 writer = csv.writer(open('tmp.csv', 'wb'), delimiter=',',
262 quotechar='|', quoting=csv.QUOTE_MINIMAL)
263
264 ONEDAY=datetime.timedelta(days=1)
265
266 # per-VO segment
267
268 for vo in vos:
269 writer.writerow(["Data for",vo])
270 writer.writerow(["Month", "si06days.used", "cpu/wall", "pledged", "jobs", "days"])
271
272 monthstart = SDATE
273
274 while monthstart < EDATE:
275
276 if monthstart.month < 12 :
277 startnextmonth = monthstart.replace(month=monthstart.month+1)
278 else:
279 d1 = monthstart.replace(month=1)
280 startnextmonth = d1.replace(year=monthstart.year+1)
281 monthend = startnextmonth - ONEDAY
282 if monthend > EDATE:
283 monthend = EDATE
284 ndays = (monthend - monthstart + ONEDAY).days
285 monthkey = monthstart.isoformat()[:7]
286 if monthkey not in parsed[vo].keys():
287 parsed[vo][monthkey] = {
288 'si06days' : 0, 'cpu' : 0.1, 'wall' : 0.1, 'njobs' : 0
289 }
290 td = parsed[vo][monthkey]
291 writer.writerow([monthkey,
292 td['si06days'],
293 td['cpu']/td['wall'],
294 ndays * pledges[vo],
295 td['njobs'],
296 ndays
297 ])
298 monthstart = monthend + ONEDAY
299
300 writer.writerow([' ',' '])
301
302 # usage plots
303
304 writer.writerow(["si06 days used per VO"])
305 writer.writerow(["Month", "lhcb.used", "atlas.used", "alice.used"])
306 monthstart = SDATE
307 while monthstart < EDATE:
308 if monthstart.month < 12 :
309 startnextmonth = monthstart.replace(month=monthstart.month+1)
310 else:
311 d1 = monthstart.replace(month=1)
312 startnextmonth = d1.replace(year=monthstart.year+1)
313 monthend = startnextmonth - ONEDAY
314 if monthend > EDATE:
315 monthend = EDATE
316 ndays = (monthend - monthstart + ONEDAY).days
317 monthkey = monthstart.isoformat()[:7]
318 td = parsed[vo][monthkey]
319 writer.writerow([monthkey,
320 parsed['lhcb'][monthkey]['si06days'],
321 parsed['atlas'][monthkey]['si06days'],
322 parsed['alice'][monthkey]['si06days']
323 ])
324 monthstart = monthend + ONEDAY
325
326 writer.writerow([' ',' '])
327
328 # pledge fraction plots
329
330 writer.writerow(["pledge fraction used per VO"])
331 writer.writerow(["Month", "lhcb.frac", "atlas.frac", "alice.frac"])
332 monthstart = SDATE
333 while monthstart < EDATE:
334 if monthstart.month < 12 :
335 startnextmonth = monthstart.replace(month=monthstart.month+1)
336 else:
337 d1 = monthstart.replace(month=1)
338 startnextmonth = d1.replace(year=monthstart.year+1)
339 monthend = startnextmonth - ONEDAY
340 if monthend > EDATE:
341 monthend = EDATE
342 ndays = (monthend - monthstart + ONEDAY).days
343 monthkey = monthstart.isoformat()[:7]
344 td = parsed[vo][monthkey]
345 writer.writerow([monthkey,
346 parsed['lhcb'][monthkey]['si06days']/(pledges['lhcb']*ndays),
347 parsed['atlas'][monthkey]['si06days']/(pledges['atlas']*ndays),
348 parsed['alice'][monthkey]['si06days']/(pledges['alice']*ndays)
349 ])
350 monthstart = monthend + ONEDAY
351
352 writer.writerow([' ',' '])
353
354 # cpu eff plots
355
356 writer.writerow(["ratio cpu to wall time used (eff) per VO"])
357 writer.writerow(["Month", "lhcb.eff", "atlas.eff", "alice.eff"])
358 monthstart = SDATE
359 while monthstart < EDATE:
360 if monthstart.month < 12 :
361 startnextmonth = monthstart.replace(month=monthstart.month+1)
362 else:
363 d1 = monthstart.replace(month=1)
364 startnextmonth = d1.replace(year=monthstart.year+1)
365 monthend = startnextmonth - ONEDAY
366 if monthend > EDATE:
367 monthend = EDATE
368 ndays = (monthend - monthstart + ONEDAY).days
369 monthkey = monthstart.isoformat()[:7]
370 td = parsed[vo][monthkey]
371 writer.writerow([monthkey,
372 parsed['lhcb'][monthkey]['cpu']/parsed['lhcb'][monthkey]['wall'],
373 parsed['atlas'][monthkey]['cpu']/parsed['atlas'][monthkey]['wall'],
374 parsed['alice'][monthkey]['cpu']/parsed['alice'][monthkey]['wall'],
375 ])
376 monthstart = monthend + ONEDAY
377
378 writer.writerow([' ',' '])
379
380 writer.writerow(["Pledges and capacties (SI06)"])
381 writer.writerow(["Group", "power"])
382 writer.writerow(["ATLAS", pledges["atlas"]])
383 writer.writerow(["LHCb", pledges["lhcb"] ])
384 writer.writerow(["ALICE", pledges["alice"]])
385 writer.writerow(["farmcap", total_cap])
386 writer.writerow([' ',' '])
387
388 for tabl in [heptab, othertab]:
389 writer.writerow([tabl.header])
390 writer.writerow(tabl.columnheaders)
391
392 groups_sorted = sorted(tabl.data, key=tabl.data.get, reverse=True)
393 for g in groups_sorted:
394 writer.writerow([g, tabl.data[g]])
395
396 writer.writerow([' ',' '])
397

Properties

Name Value
svn:executable *
svn:keywords Id URL

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28