/[pdpsoft]/nl.nikhef.ndpf.3maand/trunk/3maand.py
ViewVC logotype

Contents of /nl.nikhef.ndpf.3maand/trunk/3maand.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2455 - (show annotations) (download) (as text)
Mon Dec 5 15:01:50 2011 UTC (10 years, 9 months ago) by templon
File MIME type: application/x-python
File size: 8625 byte(s)
improve usage message.

1 #!/usr/bin/env python
2 # $Id$
3 # Source: $URL$
4 # J. A. Templon, NIKHEF/PDP 2011
5
6 # script to do yearly WLCG accounting and format it like I want.
7 # note : sorting logic is based on a by-month grouping; if you change that,
8 # you'll need to change some of the python code.
9
10 # constants to be changed each year
11
12 pledges = { # pledges in HS06 units
13 'alice' : 1844,
14 'atlas' : 17580,
15 'lhcb' : 8827
16 }
17 total_cap = 39671
18
19 import optparse
20
21 usage = "usage: %prog -p DBpwd start-date [end-date]\n" + \
22 "date format yyyy-mm-dd; default end date is 3 months after start date"
23
24 p = optparse.OptionParser(description="Program to generate a CSV dump of computing usage " + \
25 " by WLCG over a specified period",usage=usage)
26
27 # dbpassword is mandatory
28 # start date is mandatory
29 # end date is optional; proving nothing means end date is 3 months after start
30 # end date can be provided in format yyyy-mm-dd or as +3m (for 3 months later than start)
31
32 p.add_option("-p",action="store",dest="dbpassw",default=None,
33 help="password for NDPF accounting DB")
34 debug = 0
35
36 opts, args = p.parse_args()
37
38 import sys
39 if not opts.dbpassw :
40 sys.stderr.write("Error: password to NDPF accounting DB must be provided with -p\n")
41 sys.stderr.write("Bailing out.\n")
42 sys.exit(1)
43
44 if len(args) < 1:
45 sys.stderr.write("Error: no date argument detected. A start date must be provided.\n")
46 sys.stderr.write("Format: yyyy-mm-dd\n")
47 sys.stderr.write("Bailing out.\n")
48 sys.exit(1)
49
50 start_ascii = args[0]
51 import datetime
52 def parsedate(s):
53 try:
54 farr=s.split("-")
55 iarr = list()
56 for f in farr:
57 iarr.append(int(f))
58 return datetime.date(iarr[0],iarr[1],iarr[2])
59 except:
60 sys.stderr.write("Error parsing date string " + s + "\n")
61 raise
62
63 if len(args) > 1:
64 end_ascii = args[1]
65 else:
66 end_ascii = None
67
68 SDATE = parsedate(start_ascii) # starting date; script logic assumes start of month
69 if not end_ascii:
70 nmon = 3
71 elif end_ascii[:1] == '+':
72 if end_ascii[-1:] != 'm':
73 sys.stderr.write("error in end date string. accepted formats are"+
74 " 2011-05-27 or +3m\n")
75 sys.exit(1)
76 nmon = int(end_ascii[1:end_ascii.index('m')])
77 else:
78 nmon = None
79
80 if nmon:
81 eyear = SDATE.year
82 emonth = SDATE.month + nmon
83 if emonth > 12:
84 emonth = emonth - 12
85 eyear = eyear + 1
86 EDATE = datetime.date(eyear,emonth,SDATE.day)
87 else:
88 EDATE = parsedate(end_ascii) # starting date; script logic assumes start of month
89
90 print "generating data for jobs between", SDATE, "and", EDATE
91 vos = ['alice', 'atlas', 'lhcb']
92 groups = {
93 'alice': ['palice', 'alicesgm'],
94 'atlas': ['atlb', 'atlaspil', 'patlas', 'atlsgm'],
95 'lhcb' : ['lhcb', 'lhcbpil', 'lhcbprd', 'lhcbsgm']
96 }
97
98 ACCBASECMD = ['/usr/local/bin/accuse']
99 ACCBASEARG = ('--byendtime -dbpasswd ' + opts.dbpassw + ' -f lcg -m').split()
100
101 import subprocess
102
103 perlout=dict()
104
105 # gather raw accuse output
106
107 for vo in vos:
108 groupargs = []
109 for g in groups[vo]:
110 groupargs += ["-g",g]
111 args = ACCBASECMD + ACCBASEARG + ["-s", SDATE.isoformat(), "-e", EDATE.isoformat()] + groupargs
112 perlout[vo] = subprocess.Popen(args, stdout=subprocess.PIPE).communicate()[0]
113
114 def hms2dec(str):
115 h,m,s = str.split(':')
116 return float(h) + (float(m) + float(s)/60.)/60.
117
118 # parse output
119 # results in data structure like this
120
121 # data = parsed['alice']['2010-02']['hs06days']
122
123 import re
124 mpatt = re.compile(r'20[012][0-9]-[01][0-9]')
125 parsed = dict()
126 for vo in vos:
127 parsed[vo] = dict()
128 lines = perlout[vo].split('\n')
129 for line in lines:
130 if mpatt.match(line) or line.find('Summed') == 0:
131 fields = line.split()
132 cpu = hms2dec(fields[1])
133 wall = hms2dec(fields[2])
134 hs06days = float(fields[4]) * 4
135 njobs = int(fields[5])
136 parsed[vo][fields[0]] = { 'cpu' : cpu, 'wall': wall,
137 'hs06days': hs06days, 'njobs' : njobs }
138
139 # output generation ... big damn csv file
140
141 import csv
142 writer = csv.writer(open('tmp.csv', 'wb'), delimiter=',',
143 quotechar='|', quoting=csv.QUOTE_MINIMAL)
144
145 ONEDAY=datetime.timedelta(days=1)
146
147 # per-VO segment
148
149 for vo in vos:
150 writer.writerow(["Data for",vo])
151 writer.writerow(["Month", "hs06days.used", "cpu/wall", "pledged", "jobs", "days"])
152
153 monthstart = SDATE
154
155 while monthstart < EDATE:
156
157 if monthstart.month < 12 :
158 startnextmonth = monthstart.replace(month=monthstart.month+1)
159 else:
160 d1 = monthstart.replace(month=1)
161 startnextmonth = d1.replace(year=monthstart.year+1)
162 monthend = startnextmonth - ONEDAY
163 if monthend > EDATE:
164 monthend = EDATE
165 ndays = (monthend - monthstart + ONEDAY).days
166 monthkey = monthstart.isoformat()[:7]
167 if monthkey not in parsed[vo].keys():
168 parsed[vo][monthkey] = {
169 'hs06days' : 0, 'cpu' : 0.1, 'wall' : 0.1, 'njobs' : 0
170 }
171 td = parsed[vo][monthkey]
172 writer.writerow([monthkey,
173 td['hs06days'],
174 td['cpu']/td['wall'],
175 ndays * pledges[vo],
176 td['njobs'],
177 ndays
178 ])
179 monthstart = monthend + ONEDAY
180
181 writer.writerow([' ',' '])
182
183 # usage plots
184
185 writer.writerow(["hs06 days used per VO"])
186 writer.writerow(["Month", "lhcb.used", "atlas.used", "alice.used"])
187 monthstart = SDATE
188 while monthstart < EDATE:
189 if monthstart.month < 12 :
190 startnextmonth = monthstart.replace(month=monthstart.month+1)
191 else:
192 d1 = monthstart.replace(month=1)
193 startnextmonth = d1.replace(year=monthstart.year+1)
194 monthend = startnextmonth - ONEDAY
195 if monthend > EDATE:
196 monthend = EDATE
197 ndays = (monthend - monthstart + ONEDAY).days
198 monthkey = monthstart.isoformat()[:7]
199 td = parsed[vo][monthkey]
200 writer.writerow([monthkey,
201 parsed['lhcb'][monthkey]['hs06days'],
202 parsed['atlas'][monthkey]['hs06days'],
203 parsed['alice'][monthkey]['hs06days']
204 ])
205 monthstart = monthend + ONEDAY
206
207 writer.writerow([' ',' '])
208
209 # pledge fraction plots
210
211 writer.writerow(["pledge fraction used per VO"])
212 writer.writerow(["Month", "lhcb.frac", "atlas.frac", "alice.frac"])
213 monthstart = SDATE
214 while monthstart < EDATE:
215 if monthstart.month < 12 :
216 startnextmonth = monthstart.replace(month=monthstart.month+1)
217 else:
218 d1 = monthstart.replace(month=1)
219 startnextmonth = d1.replace(year=monthstart.year+1)
220 monthend = startnextmonth - ONEDAY
221 if monthend > EDATE:
222 monthend = EDATE
223 ndays = (monthend - monthstart + ONEDAY).days
224 monthkey = monthstart.isoformat()[:7]
225 td = parsed[vo][monthkey]
226 writer.writerow([monthkey,
227 parsed['lhcb'][monthkey]['hs06days']/(pledges['lhcb']*ndays),
228 parsed['atlas'][monthkey]['hs06days']/(pledges['atlas']*ndays),
229 parsed['alice'][monthkey]['hs06days']/(pledges['alice']*ndays)
230 ])
231 monthstart = monthend + ONEDAY
232
233 writer.writerow([' ',' '])
234
235 # cpu eff plots
236
237 writer.writerow(["ratio cpu to wall time used (eff) per VO"])
238 writer.writerow(["Month", "lhcb.eff", "atlas.eff", "alice.eff"])
239 monthstart = SDATE
240 while monthstart < EDATE:
241 if monthstart.month < 12 :
242 startnextmonth = monthstart.replace(month=monthstart.month+1)
243 else:
244 d1 = monthstart.replace(month=1)
245 startnextmonth = d1.replace(year=monthstart.year+1)
246 monthend = startnextmonth - ONEDAY
247 if monthend > EDATE:
248 monthend = EDATE
249 ndays = (monthend - monthstart + ONEDAY).days
250 monthkey = monthstart.isoformat()[:7]
251 td = parsed[vo][monthkey]
252 writer.writerow([monthkey,
253 parsed['lhcb'][monthkey]['cpu']/parsed['lhcb'][monthkey]['wall'],
254 parsed['atlas'][monthkey]['cpu']/parsed['atlas'][monthkey]['wall'],
255 parsed['alice'][monthkey]['cpu']/parsed['alice'][monthkey]['wall'],
256 ])
257 monthstart = monthend + ONEDAY
258
259 writer.writerow([' ',' '])
260
261 writer.writerow(["Pledges and capacties (HS06)"])
262 writer.writerow(["Group", "power"])
263 writer.writerow(["ATLAS", pledges["atlas"]])
264 writer.writerow(["LHCb", pledges["lhcb"] ])
265 writer.writerow(["ALICE", pledges["alice"]])
266 writer.writerow(["farmcap", total_cap])
267

Properties

Name Value
svn:executable *
svn:keywords Id URL

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28