| 1 |
#!/usr/bin/env python |
| 2 |
from torque_utils import pbsnodes |
| 3 |
|
| 4 |
import os |
| 5 |
import sys |
| 6 |
import subprocess |
| 7 |
|
| 8 |
GROUPS = ['atlb', 'datagrid', 'pbbmri'] |
| 9 |
MCQUEUE = 'atlasmc' |
| 10 |
TORQUE = "stro.nikhef.nl" |
| 11 |
|
| 12 |
wnodes = pbsnodes() |
| 13 |
mcnodes = list() |
| 14 |
|
| 15 |
for node in wnodes: |
| 16 |
if \ |
| 17 |
'mc' in node.properties \ |
| 18 |
and 'el6' not in node.properties \ |
| 19 |
and node.state.count('offline') == 0: |
| 20 |
mcnodes.append(node) |
| 21 |
|
| 22 |
mcslots = 0 |
| 23 |
mcfree = 0 |
| 24 |
mcrun = 0 |
| 25 |
|
| 26 |
usedkeys = ['egroup', 'jobid', 'queue', 'job_state', 'euser', 'exec_host' ] |
| 27 |
|
| 28 |
import torqueJobs |
| 29 |
import torqueAttMappers as tam |
| 30 |
|
| 31 |
import tempfile |
| 32 |
qsfname = tempfile.mktemp(".txt","qsmf",os.environ['HOME']+"/tmp") |
| 33 |
|
| 34 |
import time |
| 35 |
os.system('/usr/bin/qstat -f %s@%s' %(MCQUEUE, TORQUE) + ' > ' + qsfname) |
| 36 |
now = time.mktime(time.localtime()) |
| 37 |
|
| 38 |
jlist = torqueJobs.qs_parsefile(qsfname) |
| 39 |
|
| 40 |
os.system('mv ' + qsfname + ' ' + os.environ['HOME'] + '/tmp/qstat.last.mcpool.txt') |
| 41 |
|
| 42 |
def mapatts(indict,inkeys): |
| 43 |
sdict = tam.sub_dict(indict,inkeys) |
| 44 |
odict = dict() |
| 45 |
for k in sdict.keys(): |
| 46 |
if k in tam.tfl and sdict[k]: |
| 47 |
secs = tam.hms(sdict[k]) |
| 48 |
sdict[k] = secs/3600. |
| 49 |
if k in tam.mfl and sdict[k]: |
| 50 |
mebi = tam.memconvert(sdict[k]) |
| 51 |
sdict[k] = mebi |
| 52 |
if k in tam.tfl2 and sdict[k]: |
| 53 |
secs = tam.tconv(sdict[k]) |
| 54 |
sdict[k] = secs |
| 55 |
elif k == 'job_state': |
| 56 |
statelett = sdict[k] |
| 57 |
if statelett in ['Q','W']: |
| 58 |
sdict[k] = 'queued' |
| 59 |
elif statelett in ['R','E']: |
| 60 |
sdict[k] = 'running' |
| 61 |
elif k == 'exec_host' and sdict[k]: |
| 62 |
termpos = sdict[k].find('/') |
| 63 |
wnstring = sdict[k][:termpos] |
| 64 |
odict['wn'] = wnstring |
| 65 |
if sdict[k]: |
| 66 |
odict[k] = sdict[k] |
| 67 |
return odict |
| 68 |
|
| 69 |
mcjobdict = dict() |
| 70 |
for j in jlist: |
| 71 |
mcjobdict[j['jobid']] = mapatts(j,usedkeys) |
| 72 |
|
| 73 |
# {'queue': 'atlasmc', 'euser': 'templon', 'job_state': 'queued', \ |
| 74 |
# 'egroup': 'datagrid', 'jobid': '43609804.stro.nikhef.nl'} |
| 75 |
|
| 76 |
runjobdict = dict() # runjobdict[wn][group] = num running procs |
| 77 |
|
| 78 |
nq = dict() |
| 79 |
nr = dict() |
| 80 |
|
| 81 |
for g in GROUPS: |
| 82 |
nq[g] = 0 |
| 83 |
nr[g] = 0 |
| 84 |
|
| 85 |
for jid in mcjobdict : |
| 86 |
thisj = mcjobdict[jid] |
| 87 |
if thisj['job_state'] == 'queued': |
| 88 |
nq[thisj['egroup']] += 1 |
| 89 |
elif thisj['job_state'] == 'running': |
| 90 |
if thisj['wn'] not in runjobdict.keys(): |
| 91 |
runjobdict[thisj['wn']] = dict() |
| 92 |
if thisj['egroup'] not in runjobdict[thisj['wn']].keys(): |
| 93 |
runjobdict[thisj['wn']][thisj['egroup']] = 0 |
| 94 |
runjobdict[thisj['wn']][thisj['egroup']] += thisj['exec_host'].count('+') + 1 |
| 95 |
|
| 96 |
for n in mcnodes: |
| 97 |
mcslots += n.numCpu |
| 98 |
mcfree += n.freeCpu |
| 99 |
if n.name in runjobdict: |
| 100 |
for gd in runjobdict[n.name]: |
| 101 |
nr[gd] += runjobdict[n.name][gd] |
| 102 |
|
| 103 |
rtot = sum(nr.values()) |
| 104 |
|
| 105 |
import rrdtool |
| 106 |
DATADIR=os.environ['HOME'] + '/ndpfdata/' |
| 107 |
rrdtool.update(DATADIR+'capacity.mcpool.rrd', 'N:%d' % (mcslots)) |
| 108 |
rrdtool.update(DATADIR+'unused.mcpool.rrd', 'N:%d' % (mcfree)) |
| 109 |
rrdtool.update(DATADIR+'nonmc.running.mcpool.rrd', 'N:%d' % (mcslots - mcfree - rtot)) |
| 110 |
|
| 111 |
for g in nr: |
| 112 |
rrdtool.update(DATADIR+'%s.running.mcpool.rrd' % g, 'N:%d' % (nr[g])) |
| 113 |
rrdtool.update(DATADIR+'%s.queued.mcpool.rrd' % g, 'N:%d' % (nq[g])) |
Parent Directory
| 
Revision Log