/[pdpsoft]/nl.nikhef.ndpf.3maand/trunk/3maand.py
ViewVC logotype

Contents of /nl.nikhef.ndpf.3maand/trunk/3maand.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2447 - (show annotations) (download) (as text)
Fri Nov 4 13:06:45 2011 UTC (10 years, 10 months ago) by templon
File MIME type: application/x-python
File size: 8261 byte(s)
initial import

1 #!/usr/bin/env python
2 # $Id: ndpf-gv-mkplots 2429 2011-09-09 19:27:11Z templon $
3 # Source: $URL: svn+ssh://svn@ndpfsvn.nikhef.nl/repos/pdpsoft/nl.nikhef.ndpf.groupviews/branches/RB-2.1.1/ndpf-gv-mkplots $
4 # J. A. Templon, NIKHEF/PDP 2011
5
6 # script to do yearly WLCG accounting and format it like I want.
7 # note : sorting logic is based on a by-month grouping; if you change that,
8 # you'll need to change some of the python code.
9
10 import optparse
11
12 p = optparse.OptionParser(description="Program to generate a CSV dump of computing usage " + \
13 " by WLCG over a specified period")
14
15 # dbpassword is mandatory
16 # start date is mandatory
17 # end date is optional; proving nothing means end date is 3 months after start
18 # end date can be provided in format yyyy-mm-dd or as +3m (for 3 months later than start)
19
20 p.add_option("-p",action="store",dest="dbpassw",default=None,
21 help="password for NDPF accounting DB")
22 debug = 0
23
24 opts, args = p.parse_args()
25
26 import sys
27 if not opts.dbpassw :
28 sys.stderr.write("Error: password to NDPF accounting DB must be provided with -p\n")
29 sys.stderr.write("Bailing out.\n")
30 sys.exit(1)
31
32 if len(args) < 1:
33 sys.stderr.write("Error: no date argument detected. A start date must be provided.\n")
34 sys.stderr.write("Bailing out.\n")
35 sys.exit(1)
36
37 start_ascii = args[0]
38 import datetime
39 def parsedate(s):
40 try:
41 farr=s.split("-")
42 iarr = list()
43 for f in farr:
44 iarr.append(int(f))
45 return datetime.date(iarr[0],iarr[1],iarr[2])
46 except:
47 sys.stderr.write("Error parsing date string " + s + "\n")
48 raise
49
50 if len(args) > 1:
51 end_ascii = args[1]
52 else:
53 end_ascii = None
54
55 SDATE = parsedate(start_ascii) # starting date; script logic assumes start of month
56 if not end_ascii:
57 nmon = 3
58 elif end_ascii[:1] == '+':
59 if end_ascii[-1:] != 'm':
60 sys.stderr.write("error in end date string. accepted formats are"+
61 " 2011-05-27 or +3m\n")
62 sys.exit(1)
63 nmon = int(end_ascii[1:end_ascii.index('m')])
64 else:
65 nmon = None
66
67 if nmon:
68 eyear = SDATE.year
69 emonth = SDATE.month + nmon
70 if emonth > 12:
71 emonth = emonth - 12
72 eyear = eyear + 1
73 EDATE = datetime.date(eyear,emonth,SDATE.day)
74 else:
75 EDATE = parsedate(end_ascii) # starting date; script logic assumes start of month
76
77 print "generating data for jobs between", SDATE, "and", EDATE
78 vos = ['alice', 'atlas', 'lhcb']
79 groups = {
80 'alice': ['palice', 'alicesgm'],
81 'atlas': ['atlb', 'atlaspil', 'patlas', 'atlsgm'],
82 'lhcb' : ['lhcb', 'lhcbpil', 'lhcbprd', 'lhcbsgm']
83 }
84
85 pledges = { # pledges in HS06 units
86 'alice' : 1844,
87 'atlas' : 17580,
88 'lhcb' : 8827
89 }
90
91 ACCBASECMD = ['/usr/local/bin/accuse']
92 ACCBASEARG = ('--byendtime -dbpasswd ' + opts.dbpassw + ' -f lcg -m').split()
93
94 import subprocess
95
96 perlout=dict()
97
98 # gather raw accuse output
99
100 for vo in vos:
101 groupargs = []
102 for g in groups[vo]:
103 groupargs += ["-g",g]
104 args = ACCBASECMD + ACCBASEARG + ["-s", SDATE.isoformat(), "-e", EDATE.isoformat()] + groupargs
105 perlout[vo] = subprocess.Popen(args, stdout=subprocess.PIPE).communicate()[0]
106
107 def hms2dec(str):
108 h,m,s = str.split(':')
109 return float(h) + (float(m) + float(s)/60.)/60.
110
111 # parse output
112 # results in data structure like this
113
114 # data = parsed['alice']['2010-02']['hs06days']
115
116 import re
117 mpatt = re.compile(r'20[012][0-9]-[01][0-9]')
118 parsed = dict()
119 for vo in vos:
120 parsed[vo] = dict()
121 lines = perlout[vo].split('\n')
122 for line in lines:
123 if mpatt.match(line) or line.find('Summed') == 0:
124 fields = line.split()
125 cpu = hms2dec(fields[1])
126 wall = hms2dec(fields[2])
127 hs06days = float(fields[4]) * 4
128 njobs = int(fields[5])
129 parsed[vo][fields[0]] = { 'cpu' : cpu, 'wall': wall,
130 'hs06days': hs06days, 'njobs' : njobs }
131
132 # output generation ... big damn csv file
133
134 import csv
135 writer = csv.writer(open('tmp.csv', 'wb'), delimiter=',',
136 quotechar='|', quoting=csv.QUOTE_MINIMAL)
137
138 ONEDAY=datetime.timedelta(days=1)
139
140 # per-VO segment
141
142 for vo in vos:
143 writer.writerow(["Data for",vo])
144 writer.writerow(["Month", "hs06days.used", "cpu/wall", "pledged", "jobs", "days"])
145
146 monthstart = SDATE
147
148 while monthstart < EDATE:
149
150 if monthstart.month < 12 :
151 startnextmonth = monthstart.replace(month=monthstart.month+1)
152 else:
153 d1 = monthstart.replace(month=1)
154 startnextmonth = d1.replace(year=monthstart.year+1)
155 monthend = startnextmonth - ONEDAY
156 if monthend > EDATE:
157 monthend = EDATE
158 ndays = (monthend - monthstart + ONEDAY).days
159 monthkey = monthstart.isoformat()[:7]
160 if monthkey not in parsed[vo].keys():
161 parsed[vo][monthkey] = {
162 'hs06days' : 0, 'cpu' : 0.1, 'wall' : 0.1, 'njobs' : 0
163 }
164 td = parsed[vo][monthkey]
165 writer.writerow([monthkey,
166 td['hs06days'],
167 td['cpu']/td['wall'],
168 ndays * pledges[vo],
169 td['njobs'],
170 ndays
171 ])
172 monthstart = monthend + ONEDAY
173
174 writer.writerow([' ',' '])
175
176 # usage plots
177
178 writer.writerow(["hs06 days used per VO"])
179 writer.writerow(["Month", "lhcb.used", "atlas.used", "alice.used"])
180 monthstart = SDATE
181 while monthstart < EDATE:
182 if monthstart.month < 12 :
183 startnextmonth = monthstart.replace(month=monthstart.month+1)
184 else:
185 d1 = monthstart.replace(month=1)
186 startnextmonth = d1.replace(year=monthstart.year+1)
187 monthend = startnextmonth - ONEDAY
188 if monthend > EDATE:
189 monthend = EDATE
190 ndays = (monthend - monthstart + ONEDAY).days
191 monthkey = monthstart.isoformat()[:7]
192 td = parsed[vo][monthkey]
193 writer.writerow([monthkey,
194 parsed['lhcb'][monthkey]['hs06days'],
195 parsed['atlas'][monthkey]['hs06days'],
196 parsed['alice'][monthkey]['hs06days']
197 ])
198 monthstart = monthend + ONEDAY
199
200 writer.writerow([' ',' '])
201
202 # pledge fraction plots
203
204 writer.writerow(["pledge fraction used per VO"])
205 writer.writerow(["Month", "lhcb.frac", "atlas.frac", "alice.frac"])
206 monthstart = SDATE
207 while monthstart < EDATE:
208 if monthstart.month < 12 :
209 startnextmonth = monthstart.replace(month=monthstart.month+1)
210 else:
211 d1 = monthstart.replace(month=1)
212 startnextmonth = d1.replace(year=monthstart.year+1)
213 monthend = startnextmonth - ONEDAY
214 if monthend > EDATE:
215 monthend = EDATE
216 ndays = (monthend - monthstart + ONEDAY).days
217 monthkey = monthstart.isoformat()[:7]
218 td = parsed[vo][monthkey]
219 writer.writerow([monthkey,
220 parsed['lhcb'][monthkey]['hs06days']/(pledges['lhcb']*ndays),
221 parsed['atlas'][monthkey]['hs06days']/(pledges['atlas']*ndays),
222 parsed['alice'][monthkey]['hs06days']/(pledges['alice']*ndays)
223 ])
224 monthstart = monthend + ONEDAY
225
226 writer.writerow([' ',' '])
227
228 # cpu eff plots
229
230 writer.writerow(["ratio cpu to wall time used (eff) per VO"])
231 writer.writerow(["Month", "lhcb.eff", "atlas.eff", "alice.eff"])
232 monthstart = SDATE
233 while monthstart < EDATE:
234 if monthstart.month < 12 :
235 startnextmonth = monthstart.replace(month=monthstart.month+1)
236 else:
237 d1 = monthstart.replace(month=1)
238 startnextmonth = d1.replace(year=monthstart.year+1)
239 monthend = startnextmonth - ONEDAY
240 if monthend > EDATE:
241 monthend = EDATE
242 ndays = (monthend - monthstart + ONEDAY).days
243 monthkey = monthstart.isoformat()[:7]
244 td = parsed[vo][monthkey]
245 writer.writerow([monthkey,
246 parsed['lhcb'][monthkey]['cpu']/parsed['lhcb'][monthkey]['wall'],
247 parsed['atlas'][monthkey]['cpu']/parsed['atlas'][monthkey]['wall'],
248 parsed['alice'][monthkey]['cpu']/parsed['alice'][monthkey]['wall'],
249 ])
250 monthstart = monthend + ONEDAY
251
252 writer.writerow([' ',' '])

Properties

Name Value
svn:executable *

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28