Please note that the CVS and issue trackers have moved to GitHub. These Trac pages are no longer kept up-to-date.

root/seattle/trunk/deploymentscripts/custom.py@5637

Revision 2810, 8.6 KB (checked in by konp, 10 years ago)

Fixed high cpu usage, added summary tool, a number of other fixes and improvements as well. Note: blackbox node keys have not been added yet.

Line 
1"""
2<Program Name>
3  custom.py
4
5<Started>
6  May 2009
7
8<Author>
9  n2k8000@u.washington.edu
10  Konstantin Pik
11
12<Purpose>
13  This is a sample of what type of custom scripts can be added to be deployed
14  on machines. This script in particular grabs the version from nmmain.py by
15  executing a grep through the shell, then prints it to console and all output
16  is logged from this script.
17
18<Usage>
19  This script is called by runlocaltests.py and takes in the path to the
20  seattle install.  Not intended to run as a stand-alone.
21 
22"""
23
24import subprocess
25import sys
26import os
27import deploy_helper
28
29def shellexec_each_line(cmd_list):
30  final_out = ''
31  final_err = ''
32  for each_command in cmd_list:
33    print each_command
34    out, err = shellexec(each_command)
35    final_out += out
36    final_err += err
37    format_print(out, err)
38  return final_out, final_err
39   
40   
41   
42def shellexec(cmd_str):
43  """
44  <Purpose>
45    Uses subprocess to execute the command string in the shell.
46     
47  <Arguments>
48    cmd_str:  The string to be treated as a command (or set of commands,
49                separated by ;).
50   
51  <Exceptions>
52    None.
53
54  <Side Effects>
55    None.
56
57  <Returns>
58    A tuple containing (stdout, strerr)
59
60    Detailed:
61    stdout: stdout printed to console during command execution.
62    strerr: error (note: some programs print to strerr instead of stdout)
63  """
64  # get a handle to the subprocess we're creating..
65  handle = subprocess.Popen(cmd_str, shell=True, 
66      stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
67
68  # execute and grab the stdout and err
69  stdoutdata, strerrdata = handle.communicate("")
70
71  return stdoutdata, strerrdata
72
73 
74 
75def format_print(out, err):
76  """
77  <Purpose>
78    Will print out the non-empty out/err strings once they're properly
79    formatted. Intended to format stdout and stderr
80
81  <Arguments>
82    out:
83      stdout
84    err:
85      std error
86
87  <Exceptions>
88    None.
89
90  <Side Effects>
91    None.
92
93  <Returns>
94    None.
95  """
96  out = out.strip('\n\r ')
97  err = err.strip('\n\r ')
98  if out:
99    print out
100  if err:
101    print err
102
103   
104   
105def main(installpath):
106  """
107  <Purpose>
108     Entry point into the script. Shell executes a grep command that will
109     grab the version from nmmain.py. Then prints err&output of grep to stdout
110
111  <Arguments>
112    installpath:
113      The seattle install path.
114
115  <Exceptions>
116    None.
117
118  <Side Effects>
119    None.
120
121  <Returns>
122    None.
123  """
124 
125  # this'll get the version
126  out, err = shellexec('grep ^version '+installpath+'/nmmain.py')
127  format_print(out, err)
128 
129  # check if we need to upgrade
130  check_if_update(out)
131 
132  #shellexec('rm -rf '+installpath+'/seattle_repy')
133 
134  # check for memory hogs
135  check_highest_mem_use()
136 
137  if not is_cron_running():
138    start_cron()
139 
140  try:
141    # try and print all file content of /v2 folder
142    print 'The following files exist in /v2:'+str(os.listdir(installpath+'/v2'))
143    # now dump any log files we may have that would be located in /v2
144    for each_file in os.listdir(installpath+'/v2'):
145      # check that it's a file and not a dir, just in case.
146      if os.path.isfile(installpath+'/v2/'+each_file):
147        # yup, dump it so we can see it
148        print '\nFile contents of '+each_file
149        out, err = shellexec('cat '+installpath+'/v2/'+each_file)
150        # summarize each file
151        final_out = deploy_helper.summarize_all_blocks(out)
152        format_print(final_out, err)
153        print 'End contents of '+each_file
154       
155    # We're also gonna try and print the content of the vesseldict file
156    print '\nGrabbing vesseldict from seattle_repy.'
157    print '\nFile contents of vesseldict:'
158   
159    # we also need to dump the vesseldict file
160    if os.path.isfile(installpath+'/vesseldict'):
161      out, err = shellexec('cat '+installpath+'/vesseldict')
162      format_print(out, err)
163    else:
164      print '\tvesseldict is missing!!!'
165     
166    print 'End contents of vesseldict'
167  except Exception, e:
168    print e
169    return
170
171   
172   
173def stop_seattle():
174  """
175  <Purpose>
176    Makes a call to stop seattle on current node.
177
178  <Arguments>
179    None.
180
181  <Exceptions>
182    None.
183
184  <Side Effects>
185    None.
186
187  <Returns>
188    None.
189  """
190 
191  installpath = sys.argv[1]
192  out, err = shellexec(installpath+'/stop_seattle.sh')
193  format_print(out, err)
194  return
195
196 
197 
198def start_seattle():
199  """
200  <Purpose>
201    Makes a call to start seattle on current node.
202
203  <Arguments>
204    None.
205
206  <Exceptions>
207    None.
208
209  <Side Effects>
210    None.
211
212  <Returns>
213    None.
214  """ 
215  installpath = sys.argv[1]
216  out, err = shellexec('chmod +x '+installpath+'/start_seattle.sh')
217  format_print(out, err)
218  out, err = shellexec(installpath+'/start_seattle.sh')
219  format_print(out, err)
220  return
221
222 
223def check_highest_mem_use():
224  """
225  <Purpose>
226     Checks for the processes that have the highest memory usage
227
228  <Arguments>
229    None.
230
231  <Exceptions>
232    None.
233
234  <Side Effects>
235    None.
236
237  <Returns>
238    None.
239  """
240  # this'll get the top 3 memory hogs on the system
241  out, err = shellexec('ps -eo pcpu,pid,user,args | sort -r -k1 | head -4')
242  print 'Memory usage information:'
243  print out
244  print err
245
246
247def is_nm_running():
248  out, err = shellexec('ps -ef | grep nmmain.py | grep -v grep ')
249  return out.find('nmmain.py') > -1
250
251def is_su_running():
252  out, err = shellexec('ps -ef | grep softwareupdater.py | grep -v grep ')
253  return out.find('softwareupdater.py') > -1
254 
255def upgrade_node():
256  """
257  <Purpose>
258     Executes shell commands to download an install file and then
259     install the seattle node.
260
261  <Arguments>
262    None.
263
264  <Exceptions>
265    None.
266
267  <Side Effects>
268    None.
269
270  <Returns>
271    None.
272  """
273 
274  # stop seattle first.
275  # stop_seattle()
276  # we need to 'cd ~/' since the cwd will be the current install directory
277  # but we want to do everything from ~/ to make it easier.
278 
279  #print 'before'
280  out, err = shellexec('tar -xf deploy.tar upgrade_nodes.sh; chmod +x upgrade_nodes.sh; cp -fr upgrade_nodes.sh '+sys.argv[1])
281  if out.find('ERROR 404'):
282    upgrade_node()
283    return
284  else:
285    format_print(out, err)
286 
287  out, err = shellexec('cd '+sys.argv[1]+'; ./upgrade_nodes.sh; rm upgrade_nodes.sh; cd ~/; rm upgrade_nodes.sh')
288  format_print(out, err)
289  #print 'after'
290 
291
292
293  return
294 
295 
296
297def is_cron_running():
298  """
299  <Purpose>
300     Checks to see if cron is running.
301
302  <Arguments>
303    None.
304
305  <Exceptions>
306    None.
307
308  <Side Effects>
309    None.
310
311  <Returns>
312    Boolean. True if cron is running, False if not.
313  """
314 
315  # we'll build up commands in a list, then join with ;'s.
316  cmd_list = []
317 
318  # command that we're going to execute
319  cmd_list.append('ps auwx | grep crond | grep -v grep')
320 
321  # build the command string
322  cmd_str = '; '.join(cmd_list)
323 
324  print '\nChecking if crond is running'
325 
326  # execute the string
327  out, err = shellexec(cmd_str)
328 
329  # check to see if cron* was listed in the ps command
330  crond_running = out.find('cron') > -1
331 
332  format_print(out, err)
333  print 'crond is running: '+str(crond_running)+'\n'
334  return crond_running
335 
336 
337 
338def start_cron():
339  """
340  <Purpose>
341     Starts up crond on this machine
342
343  <Arguments>
344    None.
345
346  <Exceptions>
347    None.
348
349  <Side Effects>
350    Dumps any stderr/out produced when starting crond to stdout which in turn
351    will be logged.
352
353  <Returns>
354    None.
355  """
356 
357  # build up a list of commands
358  cmd_list = []
359  cmd_list.append('sudo mkdir /usr/lib/cron')
360  cmd_list.append('sudo touch /usr/lib/cron/cron.deny')
361  cmd_list.append('sudo /sbin/chkconfig --level 12345 crond on')
362  cmd_list.append('sudo /etc/init.d/crond start')
363 
364  # make it a string and execute it
365  cmd_str = '; '.join(cmd_list)
366  out, err = shellexec(cmd_str)
367 
368  # dump to stdout
369  format_print(out, err)
370  return
371 
372 
373 
374def check_if_update(out):
375  """
376  <Purpose>
377     Checks whether a node needs to be updated or not.
378
379  <Arguments>
380    out:
381      output from shell grabbing the version info from nmmain.
382
383  <Exceptions>
384    None.
385
386  <Side Effects>
387    None.
388
389  <Returns>
390    None.
391  """
392 
393  valid_versions = ['0.1l', '0.1m']
394 
395  # flag that'll keep track whether the node needs to be upgraded
396  has_valid_version = False
397 
398  # check all the versions possible
399  for each_valid_version in valid_versions:
400    if out.find(each_valid_version) > -1:
401      has_valid_version = True 
402 
403 
404  if not has_valid_version:
405    print "\n\nStarting node update"
406    upgrade_node()
407    # get the possibly new version from the file and dump it.
408    out, err = shellexec('grep ^version '+sys.argv[1]+'/nmmain.py') 
409    format_print(out, err)
410    print "Node update complete\n\n"
411  else:
412    if not is_su_running() or not is_nm_running():
413      print 'Restarting seattle on this node'
414      stop_seattle()
415      start_seattle()
416  return
417
418if __name__ == "__main__":
419  main(sys.argv[1])
Note: See TracBrowser for help on using the browser.