/[pdpsoft]/nl.nikhef.pdp.dynsched-pbs-plugin/trunk/torqueJobs.py
ViewVC logotype

Contents of /nl.nikhef.pdp.dynsched-pbs-plugin/trunk/torqueJobs.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2330 - (show annotations) (download) (as text)
Thu Jul 7 08:32:04 2011 UTC (10 years, 6 months ago) by templon
File MIME type: application/x-python
File size: 5998 byte(s)
move mapper functions out of torqueJobs ... leave this pure parsing.

1 #--
2 # torqueJobs.py -- lower level functions for parsing various forms of information about
3 # jobs running under a Torque LRMS
4 # $URL$
5 # $Id$
6
7 #--
8
9 # logFileRecordParser : parse a line from a torque accounting file
10
11 # function logFileRecordParser
12 # input : a line from a PBS accounting file
13 # output : a tuple (timestamp, recordtype, jobid, attributes)
14 # timestamp : unix timestamp of the record in the log file
15 # recordtype : what kind of record was it, e.g. Q = job queued, S = job started, E = end, etc.
16 # attributes : dictionary containing all other information contained in the record.
17
18 # search pattern for parsing string using "re" module
19 # for successful search, fields are:
20 # 1) timestamp
21 # 2) event type (Q,S,E,D, etc)
22 # 3) local PBS jobID
23 # 4) rest of line (key=value) to be parsed otherwise
24 # this structure is matched by evpatt
25
26 import re
27 evpatt = "^(.+);([A-Z]);(.+);(.*)"
28
29 # tpatt matches strings of form key=val
30 # lookahead assertion is necessary to work around presence of ' ' and '=' in some
31 # 'val' strings (like account, or neednodes with multiple processors)
32
33 tpatt = r'[a-z._A-Z]+=[a-z0-9A-Z=/: -@_]+?(?=$| [a-z._A-Z]+=)'
34 tprog = re.compile(tpatt)
35
36 def logFileRecordParser(record,debug=None):
37 r_timestamp = None
38 r_type = None
39 r_jobid = None
40 r_attrs = None
41
42 m = re.search(evpatt,record)
43 if not m:
44 print "parse patt failed, offending line is"
45 print evstring
46 return
47 if debug:
48 print "timestamp", m.group(1)
49 print "code", m.group(2)
50 print "jobid", m.group(3)
51 print "attrs", m.group(4)
52
53 tmatch = tprog.findall(m.group(4))
54 if debug:
55 print "result of key=val match pattern:", tmatch
56
57 # parse timestamp
58
59 ttup=time.strptime(m.group(1),"%m/%d/%Y %H:%M:%S")
60
61 # last element of time tuple is DST, but PBS log files
62 # don't specify time zone. Setting the last element of
63 # the tuple to -1 asks libC to figure it out based on
64 # local time zone of machine
65
66 atup = ttup[:8] + (-1,)
67 r_timestamp = int(time.mktime(atup))
68 r_type = m.group(2)
69 r_jobid = m.group(3)
70 r_attrs = keyvallist2dict(tmatch)
71
72 return (r_timestamp, r_type, r_jobid, r_attrs)
73
74 ## following takes as arg a list of key=val pairs, returns a dict with the same
75 ## structure. example input string:
76 ## ['user=tdykstra', 'group=niktheorie', 'jobname=Q11_241828.gjob']
77
78 def keyvallist2dict(kvlist):
79 d = {}
80 for f in kvlist:
81 kv=f.split("=",1)
82 if len(kv) == 2:
83 d[kv[0]] = kv[1]
84 else:
85 print "tried to split:", f, ", result was:", kv
86 raise CantHappenException
87 return d
88
89 # attribute mapping dict
90 amap = {
91 'Resource_List.pvmem' : 'req.pvmem',
92 'resources_used.cput' : 'used.cpu',
93 'Resource_List.nodes' : 'req.nodes',
94 'Resource_List.cput' : 'req.cpu',
95 'resources_used.mem' : 'used.mem',
96 'Resource_List.walltime' : 'req.walltime',
97 'jobname' : 'name',
98 'Resource_List.pmem' : 'req.pmem',
99 'resources_used.vmem' : 'used.vmem',
100 'Resource_List.neednodes' : 'req.neednodes',
101 'resources_used.walltime' : 'walltime',
102 'Resource_List.nodect' : 'req.nodect'
103 }
104
105 # list of fields for which to convert times
106 tfl = ['resources_used.cput', 'Resource_List.cput', 'Resource_List.walltime', 'resources_used.walltime' ]
107
108 def hms(instring):
109 t = instring.split(':')
110 secs = int(t[2]) + 60.0*(int(t[1]) + 60*int(t[0]))
111 return secs
112
113 # list of fields for which to convert memory
114 mfl = ['Resource_List.pvmem', 'resources_used.mem', 'Resource_List.pmem', 'resources_used.vmem']
115
116 def memconvert(instr):
117 if instr[-2:] == 'mb':
118 return int(instr[:-2])
119 elif instr[-2:] == 'kb':
120 return float(instr[:-2])/1024.
121 else:
122 return 'BADVAL'
123
124 def mapatts(indict):
125 odict = dict()
126 changekeys = amap.keys()
127 for k in indict.keys():
128 if k in tfl:
129 secs = hms(indict[k])
130 indict[k] = secs
131 elif k in mfl:
132 mem_num = memconvert(indict[k])
133 indict[k] = mem_num
134 if k in changekeys:
135 odict[amap[k]] = indict[k]
136 else:
137 odict[k] = indict[k]
138 return odict
139
140 # functions for parsing output of 'qstat -f'.
141
142 # list of fields for which to convert timestamps
143
144 tfl2 = ['ctime', 'mtime', 'qtime', 'etime', 'start_time']
145
146 import time
147 def tconv(instring):
148 timetuple = time.strptime(instring,"%c")
149 ts = time.mktime(timetuple)
150 return ts
151
152 def mapatts2(indict):
153 odict = dict()
154 changekeys = amap.keys()
155 for k in indict.keys():
156 if k in tfl:
157 secs = hms(indict[k])
158 indict[k] = secs
159 elif k in mfl:
160 mem_num = memconvert(indict[k])
161 indict[k] = mem_num
162 elif k in tfl2:
163 ts = tconv(indict[k])
164 indict[k] = ts
165 if k in changekeys:
166 odict[amap[k]] = indict[k]
167 else:
168 odict[k] = indict[k]
169 return odict
170
171 def qs_parsejob(instring, mapper=None):
172 # function for parsing a block of text corresponding to a single job.
173 # optional mapper function can be used to change what is returned, for example rename attributes
174 # or convert units of the values.
175
176 # figure out indentation (avoid hard wiring ;-)) by comparing lengths of stripped and unstripped versions
177
178 l1 = instring.split('\n',2)
179 indent = len(l1[1]) - len(l1[1].lstrip())
180
181 # deal with continuation lines
182
183 decont_string = instring.replace('\n\t','')
184
185 # resplit based on indentation
186
187 l2 = decont_string.split('\n'+indent*' ')
188
189 indict = dict()
190
191 indict['jobid'] = l2[0].split(': ')[1]
192 for line in l2[1:]:
193 print "k ", line
194 flds=line.split(' = ')
195 indict[flds[0]] = flds[1]
196
197 for k in indict.keys():
198 print k, ' ... ', indict[k]
199
200 if mapper:
201 return mapper(indict)
202 else:
203 return indict

Properties

Name Value
svn:eol-style native
svn:keywords Id URL

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28