/[pdpsoft]/nl.nikhef.pdp.dynsched-pbs-plugin/trunk/torque_utils.py
ViewVC logotype

Contents of /nl.nikhef.pdp.dynsched-pbs-plugin/trunk/torque_utils.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2012 - (show annotations) (download) (as text)
Fri Oct 8 13:11:04 2010 UTC (11 years, 11 months ago) by templon
File MIME type: application/x-python
File size: 8735 byte(s)
first checkin, copied from old tree

1 #!/usr/bin/env python2
2 """
3 python utilities and wrappers for Torque.
4 """
5
6 __version__ = "0.3"
7 __author__ = "Davide Salomoni"
8
9 from __future__ import generators # only needed in Python 2.2
10 import os,re
11
12 # The programs needed by these utilities. If they are not in a location
13 # accessible by PATH, specify their location here.
14 PBSNODES = "pbsnodes"
15 MOMCTL = "/root/momctl"
16 QSTAT = "qstat"
17
18 def pbsnodes(nodes="", ignoreError=False):
19 """
20 Interface to the pbsnodes command. It returns a list of nodes found by PBS.
21 Specify the list of nodes as a comma-separated string.
22
23 Example usage:
24
25 for node in pbsnodes():
26 if node.isUp():
27 print node.name, "average load = ", node.status['loadave']
28 else:
29 print node.name, "is down"
30
31 See the __doc__ string of the nodes returned by pbsnodes() for a list
32 of the attributes and methods supported by the node object returned
33 by pbsnodes.
34
35 The following keyword arguments are supported:
36 ignoreError (True/False): ignore errors in processing individual pbsnodes queries.
37
38 If ignoreError is False and an error is encountered issuing pbsnodes commands, a
39 TorqueError exception will be raised.
40 """
41 if not _pbsnodesOK:
42 raise IOError, "'%s' not found" % PBSNODES
43
44 if not nodes:
45 nodes = [""] # meaning all nodes
46 else:
47 nodes = nodes.split(',')
48
49 result = []
50 for node in nodes:
51 fh = os.popen("%s -a %s" % (PBSNODES,node))
52 s = fh.readlines()
53 exitcode = fh.close()
54 if exitcode:
55 if ignoreError:
56 continue
57 else:
58 raise TorqueError, ''.join(s)
59
60 # A generator-based version would be:
61 # for entry in _paragraphs(os.popen("pbsnodes -a")): yield PBSNode(entry)
62 for entry in _paragraphs(s):
63 result.append(PBSNode(entry))
64
65 return result
66
67 def qstat(jobs=""):
68 if not jobs:
69 jobs = [""] # meaning all jobs
70 else:
71 jobs = jobs.split(',')
72
73 result = []
74 for job in jobs:
75 fh = os.popen("%s -f %s" % (QSTAT,job))
76 s = fh.readlines()
77 exitcode = fh.close()
78 if exitcode:
79 raise TorqueError, ''.join(s)
80
81 for entry in _paragraphs(s):
82 result.append(PBSJob(entry))
83
84 return result
85
86 def mom(node):
87 """
88 Interface to the MOM process on a given node.
89 """
90 if not _momctlOk:
91 raise IOError, "'%s' not found" % MOMCTL
92
93 return MOMNode(node)
94
95 class PBSJob(object):
96 def __init__(self, para):
97 # step1, split on newlines
98 step1 = para.split('\n')
99 self.header = step1[0]
100 step1 = step1[1:] # remove header line
101
102 # step2, join lines broken on \t characters, removing the \t
103 step2 = []
104 for brokenline in step1:
105 line = re.sub('\t+','',brokenline).strip()
106 if line.find(' = ') != -1:
107 step2.append(line) # this is the start of a real entry
108 else:
109 step2[-1] += line # append this line to the previous one
110
111 # now set instance variables, splitting on 'KEY = VALUE'
112 for entry in step2:
113 key,val = entry.split(' = ')
114 self.__dict__[key] = val
115
116 def __str__(self):
117 return self.header
118
119 class PBSNode(object):
120 """
121 The representation of a node as reported by Torque.
122
123 Public attributes:
124 name (node name:string)
125 state (node state:string)
126 ntype (node type:string)
127 np (number of processors:string)
128 properties (node properties:list)
129 jobs (jobs running on node:dictionary)
130 status (node status:dictionary)
131 numCpu (number of CPUs:integer)
132 freeCpu (number of free CPUs:integer)
133
134 Public methods:
135 isUp() -- return 1 if node is up
136 isDown() -- return 1 if node is down
137
138 This class is not meant for direct instantiation.
139 """
140 def __init__(self,para):
141 lines = para.splitlines()
142 params = ['state','np','properties','ntype','jobs','status']
143 self.name = lines[0]
144 for line in lines[1:]:
145 for param in params:
146 m = re.match('^\s+%s = (.*)' % param, line)
147 if m:
148 # params are named "_NAME", where NAME is the pbsnodes attribute
149 self.__dict__['_%s' % param] = m.group(1)
150 continue
151 # make sure all the required params are in __dict__
152 # and set e.g. state = _state (this can be overridden later with properties)
153 for param in params:
154 self.__dict__.setdefault('_%s' % param,"")
155 self.__dict__[param] = self.__dict__['_%s' % param]
156 def __str__(self):
157 return self.name
158 def isUp(self):
159 return (self._state.find('down')==-1 and self._state.find('offline')==-1)
160 def isDown(self):
161 return not self.isUp()
162
163 def _getNumCpu(self):
164 try:
165 return int(self.np)
166 except ValueError:
167 return 0
168 def _getFreeCpu(self):
169 if self.isDown(): return 0
170
171 free = int(self.np)
172 myJobs = self._getJobs()
173 for cpu in range(free):
174 if len(myJobs[cpu]): free = free-1
175
176 return free
177 def _getProperties(self):
178 return [s.strip() for s in self._properties.split(',')]
179 def _getJobs(self):
180 #return [s.strip()[2:] for s in self._jobs.split(',')]
181 d = {}
182 for cpu in range(int(self.np)):
183 d[cpu] = []
184
185 for entry in self._jobs.split(','):
186 try:
187 key,val = entry.split('/')
188 except ValueError:
189 break
190 key,val = int(key.strip()),val.strip()
191 d[key].append(val)
192
193 return d
194 def _getStatus(self):
195 d = {}
196 for entry in self._status.split(','):
197 try:
198 key,val = entry.split('=')
199 except ValueError:
200 break
201 (key,val) = (key.strip(),val.strip())
202 d[key] = val
203 return d
204
205 # class attributes accessed via properties:
206 properties = property(_getProperties)
207 jobs = property(_getJobs)
208 status = property(_getStatus)
209 freeCpu = property(_getFreeCpu)
210 numCpu = property(_getNumCpu)
211
212 class MOMNode(object):
213 """
214 The reprentation of a MOM process as reported by momctl.
215 This class must be instantiated with the node name as parameter.
216
217 Public attributes:
218 name (node name:string)
219 server (torque server name:string)
220 version (torque version:string)
221 jobs (jobs running on node:string)
222 directory (torque directory:string)
223 active(seconds since process active:string)
224 """
225 def __init__(self,node):
226 # horrible parsing due to lack of structure in the output
227 self.jobs = []
228 for line in os.popen("%s -d 0 -h %s" % (MOMCTL,node)):
229 if line.startswith("Host"):
230 m = re.search('Host:\s+(\S+)\s+Server:\s+(\S+)\s+Version:\s+(\S+)',line)
231 self.name = m.group(1)
232 self.server = m.group(2)
233 self.version = m.group(3)
234 elif line.startswith("HomeDirectory"):
235 m = re.search('HomeDirectory:\s+(\S+)',line)
236 self.directory = m.group(1)
237 elif line.startswith("MOM"):
238 m = re.search('MOM active:\s+(.*)',line)
239 self.active = m.group(1)
240 elif line.startswith("Job"):
241 m = re.search('.*\[(.*)\]',line)
242 self.jobs.append(m.group(1))
243
244 class TorqueError(Exception):
245 pass
246
247 def _checkProgram(*args):
248 """
249 Check if the programs passed as argument are found in the current PATH
250 and return a list with corresponding True or False values.
251 """
252
253 ret = []
254 dirnames = os.environ["PATH"].split(os.pathsep)
255 for app in args:
256 for dirname in dirnames:
257 filename = os.path.join(dirname, app)
258 if (os.path.exists(filename) and
259 os.path.isfile(filename) and
260 os.access(filename, os.X_OK)):
261 ret.append(True)
262 break
263 else:
264 ret.append(False)
265
266 return ret
267
268 def _paragraphs(file, separator=None):
269 if not callable(separator):
270 def separator(line): return re.match('^\s+$',line)
271 paragraph = []
272 for line in file:
273 if separator(line):
274 if paragraph:
275 yield ''.join(paragraph)
276 paragraph = []
277 else:
278 paragraph.append(line)
279 if paragraph: yield ''.join(paragraph)
280
281 (_pbsnodesOK, _momctlOK) = _checkProgram(PBSNODES, MOMCTL)

Properties

Name Value
svn:eol-style native
svn:keywords Id URL

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28