/[pdpsoft]/nl.nikhef.ndpf.3maand/trunk/3maand.py
ViewVC logotype

Diff of /nl.nikhef.ndpf.3maand/trunk/3maand.py

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 2627 by templon, Mon Dec 5 15:01:50 2011 UTC revision 2628 by templon, Wed Jun 5 12:21:01 2013 UTC
# Line 9  Line 9 
9    
10  # constants to be changed each year  # constants to be changed each year
11    
12  pledges = {          # pledges in HS06 units  pledges = {          # pledges in SI06 units
13      'alice' : 1844,      'alice' : 3183,
14      'atlas' : 17580,      'atlas' : 30342,
15      'lhcb'  : 8827      'lhcb'  : 15264
16      }      }
17  total_cap = 39671  total_cap = 68062
18    
19  import optparse  import optparse
20    
# Line 118  def hms2dec(str): Line 118  def hms2dec(str):
118  # parse output  # parse output
119  # results in data structure like this  # results in data structure like this
120    
121  # data = parsed['alice']['2010-02']['hs06days']  # data = parsed['alice']['2010-02']['si06days']
122    
123  import re  import re
124  mpatt = re.compile(r'20[012][0-9]-[01][0-9]')  mpatt = re.compile(r'20[012][0-9]-[01][0-9]')
# Line 131  for vo in vos: Line 131  for vo in vos:
131              fields = line.split()              fields = line.split()
132              cpu = hms2dec(fields[1])              cpu = hms2dec(fields[1])
133              wall = hms2dec(fields[2])              wall = hms2dec(fields[2])
134              hs06days = float(fields[4]) * 4              si06days = float(fields[4]) / 0.185
135              njobs = int(fields[5])              njobs = int(fields[5])
136              parsed[vo][fields[0]] = { 'cpu' : cpu, 'wall': wall,              parsed[vo][fields[0]] = { 'cpu' : cpu, 'wall': wall,
137                                        'hs06days': hs06days, 'njobs' : njobs }                                        'si06days': si06days, 'njobs' : njobs }
138    
139    hepvos   = [ 'alice', 'atlas', 'lhcb', 'auger', 'cms', 'geant', 'dzero', 'xenon', 'gear' ]
140    othervos = [ 'biomed','dans','enmr.eu','esr', 'lofar', 'lsgrid', 'ncf', 'bbmri.nl', 'vlemed',
141                'tutor', 'phicos', 'chem.biggrid.nl' ]
142    local    = [ 'nikatlas', 'niklhcb', 'astrop', 'virgo', 'antares' ]
143    admin    = [ 'gridstaff' ]
144    probes   = [ 'ops', 'ops.nl', 'p4', 'dteam']
145    
146    groups = {
147        'alice': ['palice', 'alicesgm'],
148        'atlas': ['atlb', 'atlaspil', 'patlas', 'atlsgm'],
149        'lhcb' : ['lhcb', 'lhcbpil', 'lhcbprd', 'lhcbsgm'],
150        'gear' : ['poola' ],
151        'auger' : ['auger', 'augsgm'],
152        'cms' : ['cms'],
153        'geant' : ['geant'],
154        'dzero' : ['pdzero'],
155        'xenon' : ['pxenon', 'pxenonsm'],
156        'biomed' : ['biome','biomesgm'],
157        'dans'  : ['dans'],
158        'enmr.eu' : ['enmr', 'enmrsgm'],
159        'esr' : ['esr'],
160        'lofar' : ['lofar','plofarsm'],
161        'lsgrid' : ['lsgrid'],
162        'ncf' : [ 'ncf'],
163        'tutor' : [ 'tutor' ],
164        'ops' : [ 'ops', 'opspil'],
165        'phicos' : [ 'phicos','phicosgm' ],
166        'bbmri.nl' : [ 'pbbmri','poolb' ],
167        'chem.biggrid.nl' : [ 'pbchem' ],
168        'ops.nl' : ['popsnl'],
169        'p4' : [ 'pvier', 'pviersm' ],
170        'vlemed' : [ 'vlemed', 'vlemedsm' ],
171        'dteam' : ['dteam' ],
172        'nikatlas' : ['atlas'],
173        'niklhcb' : [ 'bfys'],
174        'astrop' : [ 'astrop' ],
175        'antares' : [ 'antares' ],
176        'virgo' : [ 'virgo' ],
177        'gridstaff' : ['datagrid','emin', 'tbadmin']
178        }
179    
180    # gather all unix groups, for check later that we haven't missed anything
181    
182    allunixgroups = [ ]
183    for g in groups.keys():
184        allunixgroups.extend(groups[g])
185    
186    basecmd = 'mysql -u anon -p' + opts.dbpassw + ' -h bedstee.nikhef.nl accounting'
187    
188    ACCSELECT  = "select sum(job.WallGHzEquivalent),groupid.id,count(job.JobID) from job,groupid where job.groupid=groupid.UniqueID and job.EndTime >= " \
189                  + "'" + SDATE.isoformat() + "' and job.EndTime <= '" + EDATE.isoformat() + "' group by groupid.id;"
190    
191    import subprocess
192    
193    p = subprocess.Popen(basecmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
194    out, err = p.communicate(ACCSELECT)
195    
196    lines = out.split('\n')
197    resdict = dict()
198    
199    for l in lines[1:-1]:
200        f = l.split()
201        resdict[f[1]] = float(f[0])
202    
203    for k in resdict.keys():
204        if k not in allunixgroups:
205            print
206            print "unknown unix group " + k + " found in mysql output, please check"
207            print
208            
209    sums = dict()
210    for g in groups:
211        ts = 0
212        for ug in groups[g]:
213            if ug in resdict.keys():
214                ts = ts + resdict[ug]
215        sums[g] = ts
216    
217    sumothervos = 0
218    for g in othervos:
219        sumothervos = sumothervos + sums[g]
220    
221    sumlocal = 0
222    for g in local:
223        sumlocal = sumlocal + sums[g]
224    
225    sumprobes = 0
226    for g in probes:
227        sumprobes = sumprobes + sums[g]
228    
229    sumadmin = 0
230    for g in admin:
231        sumadmin = sumadmin + sums[g]
232    
233    class Table(object):
234        def __init__(self, header, columnheaders):
235            self.header = header
236            self.columnheaders = columnheaders
237            self.data = dict()
238        def add(self, name, val):
239            self.data[name] = val
240    
241    heptab = Table("HEP Usage @ Nikhef from " + SDATE.isoformat() + " to " + EDATE.isoformat(),
242                   ["group", "WallGHzEquiv"])
243    for g in hepvos:
244        heptab.add(g, sums[g])
245    heptab.add("non-HEP",sumothervos)
246    heptab.add("local", sumlocal)
247    heptab.add("probes", sumprobes)
248    heptab.add("admin", sumadmin)
249    
250    othertab = Table("non-HEP Usage @ Nikhef from " + SDATE.isoformat() + " to " + EDATE.isoformat(),
251                   ["group", "WallGHzEquiv"])
252    
253    for g in othervos:
254        othertab.add(g, sums[g])
255    othertab.add("probes", sumprobes)
256    othertab.add("admin", sumadmin)
257    
258  # output generation ... big damn csv file  # output generation ... big damn csv file
259    
# Line 148  ONEDAY=datetime.timedelta(days=1) Line 267  ONEDAY=datetime.timedelta(days=1)
267    
268  for vo in vos:  for vo in vos:
269      writer.writerow(["Data for",vo])      writer.writerow(["Data for",vo])
270      writer.writerow(["Month", "hs06days.used", "cpu/wall", "pledged", "jobs", "days"])      writer.writerow(["Month", "si06days.used", "cpu/wall", "pledged", "jobs", "days"])
271            
272      monthstart = SDATE      monthstart = SDATE
273    
# Line 166  for vo in vos: Line 285  for vo in vos:
285          monthkey = monthstart.isoformat()[:7]          monthkey = monthstart.isoformat()[:7]
286          if monthkey not in parsed[vo].keys():          if monthkey not in parsed[vo].keys():
287              parsed[vo][monthkey] = {              parsed[vo][monthkey] = {
288                  'hs06days' : 0, 'cpu' : 0.1, 'wall' : 0.1, 'njobs' : 0                  'si06days' : 0, 'cpu' : 0.1, 'wall' : 0.1, 'njobs' : 0
289                  }                  }
290          td = parsed[vo][monthkey]          td = parsed[vo][monthkey]
291          writer.writerow([monthkey,          writer.writerow([monthkey,
292                           td['hs06days'],                           td['si06days'],
293                           td['cpu']/td['wall'],                           td['cpu']/td['wall'],
294                           ndays * pledges[vo],                           ndays * pledges[vo],
295                           td['njobs'],                           td['njobs'],
# Line 182  for vo in vos: Line 301  for vo in vos:
301    
302  # usage plots  # usage plots
303    
304  writer.writerow(["hs06 days used per VO"])  writer.writerow(["si06 days used per VO"])
305  writer.writerow(["Month", "lhcb.used", "atlas.used", "alice.used"])  writer.writerow(["Month", "lhcb.used", "atlas.used", "alice.used"])
306  monthstart = SDATE  monthstart = SDATE
307  while monthstart < EDATE:  while monthstart < EDATE:
# Line 198  while monthstart < EDATE: Line 317  while monthstart < EDATE:
317      monthkey = monthstart.isoformat()[:7]      monthkey = monthstart.isoformat()[:7]
318      td = parsed[vo][monthkey]      td = parsed[vo][monthkey]
319      writer.writerow([monthkey,      writer.writerow([monthkey,
320                       parsed['lhcb'][monthkey]['hs06days'],                       parsed['lhcb'][monthkey]['si06days'],
321                       parsed['atlas'][monthkey]['hs06days'],                       parsed['atlas'][monthkey]['si06days'],
322                       parsed['alice'][monthkey]['hs06days']                       parsed['alice'][monthkey]['si06days']
323                       ])                       ])
324      monthstart = monthend + ONEDAY      monthstart = monthend + ONEDAY
325    
# Line 224  while monthstart < EDATE: Line 343  while monthstart < EDATE:
343      monthkey = monthstart.isoformat()[:7]      monthkey = monthstart.isoformat()[:7]
344      td = parsed[vo][monthkey]      td = parsed[vo][monthkey]
345      writer.writerow([monthkey,      writer.writerow([monthkey,
346                       parsed['lhcb'][monthkey]['hs06days']/(pledges['lhcb']*ndays),                       parsed['lhcb'][monthkey]['si06days']/(pledges['lhcb']*ndays),
347                       parsed['atlas'][monthkey]['hs06days']/(pledges['atlas']*ndays),                       parsed['atlas'][monthkey]['si06days']/(pledges['atlas']*ndays),
348                       parsed['alice'][monthkey]['hs06days']/(pledges['alice']*ndays)                       parsed['alice'][monthkey]['si06days']/(pledges['alice']*ndays)
349                       ])                       ])
350      monthstart = monthend + ONEDAY      monthstart = monthend + ONEDAY
351    
# Line 258  while monthstart < EDATE: Line 377  while monthstart < EDATE:
377    
378  writer.writerow([' ',' '])  writer.writerow([' ',' '])
379    
380  writer.writerow(["Pledges and capacties (HS06)"])  writer.writerow(["Pledges and capacties (SI06)"])
381  writer.writerow(["Group",   "power"])  writer.writerow(["Group",   "power"])
382  writer.writerow(["ATLAS",   pledges["atlas"]])  writer.writerow(["ATLAS",   pledges["atlas"]])
383  writer.writerow(["LHCb",    pledges["lhcb"] ])  writer.writerow(["LHCb",    pledges["lhcb"] ])
384  writer.writerow(["ALICE",   pledges["alice"]])  writer.writerow(["ALICE",   pledges["alice"]])
385  writer.writerow(["farmcap", total_cap])  writer.writerow(["farmcap", total_cap])
386    writer.writerow([' ',' '])
387    
388    for tabl in [heptab, othertab]:
389        writer.writerow([tabl.header])
390        writer.writerow(tabl.columnheaders)
391        
392        groups_sorted = sorted(tabl.data, key=tabl.data.get, reverse=True)
393        for g in groups_sorted:
394            writer.writerow([g, tabl.data[g]])
395    
396        writer.writerow([' ',' '])
397    

Legend:
Removed from v.2627  
changed lines
  Added in v.2628

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28