/[pdpsoft]/nl.nikhef.ndpf.3maand/trunk/3maand.py
ViewVC logotype

Annotation of /nl.nikhef.ndpf.3maand/trunk/3maand.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2455 - (hide annotations) (download) (as text)
Mon Dec 5 15:01:50 2011 UTC (10 years, 10 months ago) by templon
File MIME type: application/x-python
File size: 8625 byte(s)
improve usage message.

1 templon 2447 #!/usr/bin/env python
2 templon 2448 # $Id$
3     # Source: $URL$
4 templon 2447 # J. A. Templon, NIKHEF/PDP 2011
5    
6     # script to do yearly WLCG accounting and format it like I want.
7     # note : sorting logic is based on a by-month grouping; if you change that,
8     # you'll need to change some of the python code.
9    
10 templon 2448 # constants to be changed each year
11    
12     pledges = { # pledges in HS06 units
13     'alice' : 1844,
14     'atlas' : 17580,
15     'lhcb' : 8827
16     }
17     total_cap = 39671
18    
19 templon 2447 import optparse
20    
21 templon 2455 usage = "usage: %prog -p DBpwd start-date [end-date]\n" + \
22     "date format yyyy-mm-dd; default end date is 3 months after start date"
23    
24 templon 2447 p = optparse.OptionParser(description="Program to generate a CSV dump of computing usage " + \
25 templon 2455 " by WLCG over a specified period",usage=usage)
26 templon 2447
27     # dbpassword is mandatory
28     # start date is mandatory
29     # end date is optional; proving nothing means end date is 3 months after start
30     # end date can be provided in format yyyy-mm-dd or as +3m (for 3 months later than start)
31    
32     p.add_option("-p",action="store",dest="dbpassw",default=None,
33     help="password for NDPF accounting DB")
34     debug = 0
35    
36     opts, args = p.parse_args()
37    
38     import sys
39     if not opts.dbpassw :
40     sys.stderr.write("Error: password to NDPF accounting DB must be provided with -p\n")
41     sys.stderr.write("Bailing out.\n")
42     sys.exit(1)
43    
44     if len(args) < 1:
45     sys.stderr.write("Error: no date argument detected. A start date must be provided.\n")
46 templon 2455 sys.stderr.write("Format: yyyy-mm-dd\n")
47 templon 2447 sys.stderr.write("Bailing out.\n")
48     sys.exit(1)
49    
50     start_ascii = args[0]
51     import datetime
52     def parsedate(s):
53     try:
54     farr=s.split("-")
55     iarr = list()
56     for f in farr:
57     iarr.append(int(f))
58     return datetime.date(iarr[0],iarr[1],iarr[2])
59     except:
60     sys.stderr.write("Error parsing date string " + s + "\n")
61     raise
62    
63     if len(args) > 1:
64     end_ascii = args[1]
65     else:
66     end_ascii = None
67    
68     SDATE = parsedate(start_ascii) # starting date; script logic assumes start of month
69     if not end_ascii:
70     nmon = 3
71     elif end_ascii[:1] == '+':
72     if end_ascii[-1:] != 'm':
73     sys.stderr.write("error in end date string. accepted formats are"+
74     " 2011-05-27 or +3m\n")
75     sys.exit(1)
76     nmon = int(end_ascii[1:end_ascii.index('m')])
77     else:
78     nmon = None
79    
80     if nmon:
81     eyear = SDATE.year
82     emonth = SDATE.month + nmon
83     if emonth > 12:
84     emonth = emonth - 12
85     eyear = eyear + 1
86     EDATE = datetime.date(eyear,emonth,SDATE.day)
87     else:
88     EDATE = parsedate(end_ascii) # starting date; script logic assumes start of month
89    
90     print "generating data for jobs between", SDATE, "and", EDATE
91     vos = ['alice', 'atlas', 'lhcb']
92     groups = {
93     'alice': ['palice', 'alicesgm'],
94     'atlas': ['atlb', 'atlaspil', 'patlas', 'atlsgm'],
95     'lhcb' : ['lhcb', 'lhcbpil', 'lhcbprd', 'lhcbsgm']
96     }
97    
98     ACCBASECMD = ['/usr/local/bin/accuse']
99     ACCBASEARG = ('--byendtime -dbpasswd ' + opts.dbpassw + ' -f lcg -m').split()
100    
101     import subprocess
102    
103     perlout=dict()
104    
105     # gather raw accuse output
106    
107     for vo in vos:
108     groupargs = []
109     for g in groups[vo]:
110     groupargs += ["-g",g]
111     args = ACCBASECMD + ACCBASEARG + ["-s", SDATE.isoformat(), "-e", EDATE.isoformat()] + groupargs
112     perlout[vo] = subprocess.Popen(args, stdout=subprocess.PIPE).communicate()[0]
113    
114     def hms2dec(str):
115     h,m,s = str.split(':')
116     return float(h) + (float(m) + float(s)/60.)/60.
117    
118     # parse output
119     # results in data structure like this
120    
121     # data = parsed['alice']['2010-02']['hs06days']
122    
123     import re
124     mpatt = re.compile(r'20[012][0-9]-[01][0-9]')
125     parsed = dict()
126     for vo in vos:
127     parsed[vo] = dict()
128     lines = perlout[vo].split('\n')
129     for line in lines:
130     if mpatt.match(line) or line.find('Summed') == 0:
131     fields = line.split()
132     cpu = hms2dec(fields[1])
133     wall = hms2dec(fields[2])
134     hs06days = float(fields[4]) * 4
135     njobs = int(fields[5])
136     parsed[vo][fields[0]] = { 'cpu' : cpu, 'wall': wall,
137     'hs06days': hs06days, 'njobs' : njobs }
138    
139     # output generation ... big damn csv file
140    
141     import csv
142     writer = csv.writer(open('tmp.csv', 'wb'), delimiter=',',
143     quotechar='|', quoting=csv.QUOTE_MINIMAL)
144    
145     ONEDAY=datetime.timedelta(days=1)
146    
147     # per-VO segment
148    
149     for vo in vos:
150     writer.writerow(["Data for",vo])
151     writer.writerow(["Month", "hs06days.used", "cpu/wall", "pledged", "jobs", "days"])
152    
153     monthstart = SDATE
154    
155     while monthstart < EDATE:
156    
157     if monthstart.month < 12 :
158     startnextmonth = monthstart.replace(month=monthstart.month+1)
159     else:
160     d1 = monthstart.replace(month=1)
161     startnextmonth = d1.replace(year=monthstart.year+1)
162     monthend = startnextmonth - ONEDAY
163     if monthend > EDATE:
164     monthend = EDATE
165     ndays = (monthend - monthstart + ONEDAY).days
166     monthkey = monthstart.isoformat()[:7]
167     if monthkey not in parsed[vo].keys():
168     parsed[vo][monthkey] = {
169     'hs06days' : 0, 'cpu' : 0.1, 'wall' : 0.1, 'njobs' : 0
170     }
171     td = parsed[vo][monthkey]
172     writer.writerow([monthkey,
173     td['hs06days'],
174     td['cpu']/td['wall'],
175     ndays * pledges[vo],
176     td['njobs'],
177     ndays
178     ])
179     monthstart = monthend + ONEDAY
180    
181     writer.writerow([' ',' '])
182    
183     # usage plots
184    
185     writer.writerow(["hs06 days used per VO"])
186     writer.writerow(["Month", "lhcb.used", "atlas.used", "alice.used"])
187     monthstart = SDATE
188     while monthstart < EDATE:
189     if monthstart.month < 12 :
190     startnextmonth = monthstart.replace(month=monthstart.month+1)
191     else:
192     d1 = monthstart.replace(month=1)
193     startnextmonth = d1.replace(year=monthstart.year+1)
194     monthend = startnextmonth - ONEDAY
195     if monthend > EDATE:
196     monthend = EDATE
197     ndays = (monthend - monthstart + ONEDAY).days
198     monthkey = monthstart.isoformat()[:7]
199     td = parsed[vo][monthkey]
200     writer.writerow([monthkey,
201     parsed['lhcb'][monthkey]['hs06days'],
202     parsed['atlas'][monthkey]['hs06days'],
203     parsed['alice'][monthkey]['hs06days']
204     ])
205     monthstart = monthend + ONEDAY
206    
207     writer.writerow([' ',' '])
208    
209     # pledge fraction plots
210    
211     writer.writerow(["pledge fraction used per VO"])
212     writer.writerow(["Month", "lhcb.frac", "atlas.frac", "alice.frac"])
213     monthstart = SDATE
214     while monthstart < EDATE:
215     if monthstart.month < 12 :
216     startnextmonth = monthstart.replace(month=monthstart.month+1)
217     else:
218     d1 = monthstart.replace(month=1)
219     startnextmonth = d1.replace(year=monthstart.year+1)
220     monthend = startnextmonth - ONEDAY
221     if monthend > EDATE:
222     monthend = EDATE
223     ndays = (monthend - monthstart + ONEDAY).days
224     monthkey = monthstart.isoformat()[:7]
225     td = parsed[vo][monthkey]
226     writer.writerow([monthkey,
227     parsed['lhcb'][monthkey]['hs06days']/(pledges['lhcb']*ndays),
228     parsed['atlas'][monthkey]['hs06days']/(pledges['atlas']*ndays),
229     parsed['alice'][monthkey]['hs06days']/(pledges['alice']*ndays)
230     ])
231     monthstart = monthend + ONEDAY
232    
233     writer.writerow([' ',' '])
234    
235     # cpu eff plots
236    
237     writer.writerow(["ratio cpu to wall time used (eff) per VO"])
238     writer.writerow(["Month", "lhcb.eff", "atlas.eff", "alice.eff"])
239     monthstart = SDATE
240     while monthstart < EDATE:
241     if monthstart.month < 12 :
242     startnextmonth = monthstart.replace(month=monthstart.month+1)
243     else:
244     d1 = monthstart.replace(month=1)
245     startnextmonth = d1.replace(year=monthstart.year+1)
246     monthend = startnextmonth - ONEDAY
247     if monthend > EDATE:
248     monthend = EDATE
249     ndays = (monthend - monthstart + ONEDAY).days
250     monthkey = monthstart.isoformat()[:7]
251     td = parsed[vo][monthkey]
252     writer.writerow([monthkey,
253     parsed['lhcb'][monthkey]['cpu']/parsed['lhcb'][monthkey]['wall'],
254     parsed['atlas'][monthkey]['cpu']/parsed['atlas'][monthkey]['wall'],
255     parsed['alice'][monthkey]['cpu']/parsed['alice'][monthkey]['wall'],
256     ])
257     monthstart = monthend + ONEDAY
258    
259     writer.writerow([' ',' '])
260 templon 2448
261     writer.writerow(["Pledges and capacties (HS06)"])
262     writer.writerow(["Group", "power"])
263     writer.writerow(["ATLAS", pledges["atlas"]])
264     writer.writerow(["LHCb", pledges["lhcb"] ])
265     writer.writerow(["ALICE", pledges["alice"]])
266     writer.writerow(["farmcap", total_cap])
267    

Properties

Name Value
svn:executable *
svn:keywords Id URL

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28