/[pdpsoft]/nl.nikhef.ndpf.3maand/trunk/3maand.py
ViewVC logotype

Annotation of /nl.nikhef.ndpf.3maand/trunk/3maand.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2447 - (hide annotations) (download) (as text)
Fri Nov 4 13:06:45 2011 UTC (10 years, 11 months ago) by templon
File MIME type: application/x-python
File size: 8261 byte(s)
initial import

1 templon 2447 #!/usr/bin/env python
2     # $Id: ndpf-gv-mkplots 2429 2011-09-09 19:27:11Z templon $
3     # Source: $URL: svn+ssh://svn@ndpfsvn.nikhef.nl/repos/pdpsoft/nl.nikhef.ndpf.groupviews/branches/RB-2.1.1/ndpf-gv-mkplots $
4     # J. A. Templon, NIKHEF/PDP 2011
5    
6     # script to do yearly WLCG accounting and format it like I want.
7     # note : sorting logic is based on a by-month grouping; if you change that,
8     # you'll need to change some of the python code.
9    
10     import optparse
11    
12     p = optparse.OptionParser(description="Program to generate a CSV dump of computing usage " + \
13     " by WLCG over a specified period")
14    
15     # dbpassword is mandatory
16     # start date is mandatory
17     # end date is optional; proving nothing means end date is 3 months after start
18     # end date can be provided in format yyyy-mm-dd or as +3m (for 3 months later than start)
19    
20     p.add_option("-p",action="store",dest="dbpassw",default=None,
21     help="password for NDPF accounting DB")
22     debug = 0
23    
24     opts, args = p.parse_args()
25    
26     import sys
27     if not opts.dbpassw :
28     sys.stderr.write("Error: password to NDPF accounting DB must be provided with -p\n")
29     sys.stderr.write("Bailing out.\n")
30     sys.exit(1)
31    
32     if len(args) < 1:
33     sys.stderr.write("Error: no date argument detected. A start date must be provided.\n")
34     sys.stderr.write("Bailing out.\n")
35     sys.exit(1)
36    
37     start_ascii = args[0]
38     import datetime
39     def parsedate(s):
40     try:
41     farr=s.split("-")
42     iarr = list()
43     for f in farr:
44     iarr.append(int(f))
45     return datetime.date(iarr[0],iarr[1],iarr[2])
46     except:
47     sys.stderr.write("Error parsing date string " + s + "\n")
48     raise
49    
50     if len(args) > 1:
51     end_ascii = args[1]
52     else:
53     end_ascii = None
54    
55     SDATE = parsedate(start_ascii) # starting date; script logic assumes start of month
56     if not end_ascii:
57     nmon = 3
58     elif end_ascii[:1] == '+':
59     if end_ascii[-1:] != 'm':
60     sys.stderr.write("error in end date string. accepted formats are"+
61     " 2011-05-27 or +3m\n")
62     sys.exit(1)
63     nmon = int(end_ascii[1:end_ascii.index('m')])
64     else:
65     nmon = None
66    
67     if nmon:
68     eyear = SDATE.year
69     emonth = SDATE.month + nmon
70     if emonth > 12:
71     emonth = emonth - 12
72     eyear = eyear + 1
73     EDATE = datetime.date(eyear,emonth,SDATE.day)
74     else:
75     EDATE = parsedate(end_ascii) # starting date; script logic assumes start of month
76    
77     print "generating data for jobs between", SDATE, "and", EDATE
78     vos = ['alice', 'atlas', 'lhcb']
79     groups = {
80     'alice': ['palice', 'alicesgm'],
81     'atlas': ['atlb', 'atlaspil', 'patlas', 'atlsgm'],
82     'lhcb' : ['lhcb', 'lhcbpil', 'lhcbprd', 'lhcbsgm']
83     }
84    
85     pledges = { # pledges in HS06 units
86     'alice' : 1844,
87     'atlas' : 17580,
88     'lhcb' : 8827
89     }
90    
91     ACCBASECMD = ['/usr/local/bin/accuse']
92     ACCBASEARG = ('--byendtime -dbpasswd ' + opts.dbpassw + ' -f lcg -m').split()
93    
94     import subprocess
95    
96     perlout=dict()
97    
98     # gather raw accuse output
99    
100     for vo in vos:
101     groupargs = []
102     for g in groups[vo]:
103     groupargs += ["-g",g]
104     args = ACCBASECMD + ACCBASEARG + ["-s", SDATE.isoformat(), "-e", EDATE.isoformat()] + groupargs
105     perlout[vo] = subprocess.Popen(args, stdout=subprocess.PIPE).communicate()[0]
106    
107     def hms2dec(str):
108     h,m,s = str.split(':')
109     return float(h) + (float(m) + float(s)/60.)/60.
110    
111     # parse output
112     # results in data structure like this
113    
114     # data = parsed['alice']['2010-02']['hs06days']
115    
116     import re
117     mpatt = re.compile(r'20[012][0-9]-[01][0-9]')
118     parsed = dict()
119     for vo in vos:
120     parsed[vo] = dict()
121     lines = perlout[vo].split('\n')
122     for line in lines:
123     if mpatt.match(line) or line.find('Summed') == 0:
124     fields = line.split()
125     cpu = hms2dec(fields[1])
126     wall = hms2dec(fields[2])
127     hs06days = float(fields[4]) * 4
128     njobs = int(fields[5])
129     parsed[vo][fields[0]] = { 'cpu' : cpu, 'wall': wall,
130     'hs06days': hs06days, 'njobs' : njobs }
131    
132     # output generation ... big damn csv file
133    
134     import csv
135     writer = csv.writer(open('tmp.csv', 'wb'), delimiter=',',
136     quotechar='|', quoting=csv.QUOTE_MINIMAL)
137    
138     ONEDAY=datetime.timedelta(days=1)
139    
140     # per-VO segment
141    
142     for vo in vos:
143     writer.writerow(["Data for",vo])
144     writer.writerow(["Month", "hs06days.used", "cpu/wall", "pledged", "jobs", "days"])
145    
146     monthstart = SDATE
147    
148     while monthstart < EDATE:
149    
150     if monthstart.month < 12 :
151     startnextmonth = monthstart.replace(month=monthstart.month+1)
152     else:
153     d1 = monthstart.replace(month=1)
154     startnextmonth = d1.replace(year=monthstart.year+1)
155     monthend = startnextmonth - ONEDAY
156     if monthend > EDATE:
157     monthend = EDATE
158     ndays = (monthend - monthstart + ONEDAY).days
159     monthkey = monthstart.isoformat()[:7]
160     if monthkey not in parsed[vo].keys():
161     parsed[vo][monthkey] = {
162     'hs06days' : 0, 'cpu' : 0.1, 'wall' : 0.1, 'njobs' : 0
163     }
164     td = parsed[vo][monthkey]
165     writer.writerow([monthkey,
166     td['hs06days'],
167     td['cpu']/td['wall'],
168     ndays * pledges[vo],
169     td['njobs'],
170     ndays
171     ])
172     monthstart = monthend + ONEDAY
173    
174     writer.writerow([' ',' '])
175    
176     # usage plots
177    
178     writer.writerow(["hs06 days used per VO"])
179     writer.writerow(["Month", "lhcb.used", "atlas.used", "alice.used"])
180     monthstart = SDATE
181     while monthstart < EDATE:
182     if monthstart.month < 12 :
183     startnextmonth = monthstart.replace(month=monthstart.month+1)
184     else:
185     d1 = monthstart.replace(month=1)
186     startnextmonth = d1.replace(year=monthstart.year+1)
187     monthend = startnextmonth - ONEDAY
188     if monthend > EDATE:
189     monthend = EDATE
190     ndays = (monthend - monthstart + ONEDAY).days
191     monthkey = monthstart.isoformat()[:7]
192     td = parsed[vo][monthkey]
193     writer.writerow([monthkey,
194     parsed['lhcb'][monthkey]['hs06days'],
195     parsed['atlas'][monthkey]['hs06days'],
196     parsed['alice'][monthkey]['hs06days']
197     ])
198     monthstart = monthend + ONEDAY
199    
200     writer.writerow([' ',' '])
201    
202     # pledge fraction plots
203    
204     writer.writerow(["pledge fraction used per VO"])
205     writer.writerow(["Month", "lhcb.frac", "atlas.frac", "alice.frac"])
206     monthstart = SDATE
207     while monthstart < EDATE:
208     if monthstart.month < 12 :
209     startnextmonth = monthstart.replace(month=monthstart.month+1)
210     else:
211     d1 = monthstart.replace(month=1)
212     startnextmonth = d1.replace(year=monthstart.year+1)
213     monthend = startnextmonth - ONEDAY
214     if monthend > EDATE:
215     monthend = EDATE
216     ndays = (monthend - monthstart + ONEDAY).days
217     monthkey = monthstart.isoformat()[:7]
218     td = parsed[vo][monthkey]
219     writer.writerow([monthkey,
220     parsed['lhcb'][monthkey]['hs06days']/(pledges['lhcb']*ndays),
221     parsed['atlas'][monthkey]['hs06days']/(pledges['atlas']*ndays),
222     parsed['alice'][monthkey]['hs06days']/(pledges['alice']*ndays)
223     ])
224     monthstart = monthend + ONEDAY
225    
226     writer.writerow([' ',' '])
227    
228     # cpu eff plots
229    
230     writer.writerow(["ratio cpu to wall time used (eff) per VO"])
231     writer.writerow(["Month", "lhcb.eff", "atlas.eff", "alice.eff"])
232     monthstart = SDATE
233     while monthstart < EDATE:
234     if monthstart.month < 12 :
235     startnextmonth = monthstart.replace(month=monthstart.month+1)
236     else:
237     d1 = monthstart.replace(month=1)
238     startnextmonth = d1.replace(year=monthstart.year+1)
239     monthend = startnextmonth - ONEDAY
240     if monthend > EDATE:
241     monthend = EDATE
242     ndays = (monthend - monthstart + ONEDAY).days
243     monthkey = monthstart.isoformat()[:7]
244     td = parsed[vo][monthkey]
245     writer.writerow([monthkey,
246     parsed['lhcb'][monthkey]['cpu']/parsed['lhcb'][monthkey]['wall'],
247     parsed['atlas'][monthkey]['cpu']/parsed['atlas'][monthkey]['wall'],
248     parsed['alice'][monthkey]['cpu']/parsed['alice'][monthkey]['wall'],
249     ])
250     monthstart = monthend + ONEDAY
251    
252     writer.writerow([' ',' '])

Properties

Name Value
svn:executable *

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28