Please note that the CVS and issue trackers have moved to GitHub. These Trac pages are no longer kept up-to-date.

root/seattle/trunk/deploymentscripts/deploy_helper.py@5637

Revision 2810, 13.3 KB (checked in by konp, 10 years ago)

Fixed high cpu usage, added summary tool, a number of other fixes and improvements as well. Note: blackbox node keys have not been added yet.

Line 
1"""
2<Program Name>
3  deploy_helper.py
4
5<Started>
6  May 2009
7
8<Author>
9  n2k8000@u.washington.edu
10  Konstantin Pik
11
12<Purpose>
13  File has helper methods to be used across all the deploy_* libraries.
14"""
15
16import subprocess
17import time
18
19# Assorted helper functions that don't really go anywhere else
20
21def summarize_all_blocks(text):
22  """
23  <Purpose>
24    If new patterns need to be added to be summarized (aka they're spamming
25    the logs, they'll go here)
26
27  <Arguments>
28    text:
29      the text that shall be summarized, typically the contents of the log file.
30
31  <Exceptions>
32    None.
33
34  <Side Effects>
35    None.
36
37  <Returns>
38    String. Returns text but with all the changes.
39  """
40 
41 
42 
43 
44  text = summarize_block(['Traceback', 'line 151', 'line 89', 'line 222', 
45      'line 190', 'line 325', 'line 856', 'line 728', 'line 695', 
46      'line 663', 'Temporary failure in name resolution'], text)
47     
48  """
49  1251419423.49:PID-11142:Traceback (most recent call last):
50  File "softwareupdater.py", line 151, in safe_download
51  File "/vservers/.vref/planetlab-f8-i386/usr/lib/python2.5/urllib.py", line 89, in urlretrieve
52  File "/vservers/.vref/planetlab-f8-i386/usr/lib/python2.5/urllib.py", line 222, in retrieve
53  File "/vservers/.vref/planetlab-f8-i386/usr/lib/python2.5/urllib.py", line 190, in open
54  File "/vservers/.vref/planetlab-f8-i386/usr/lib/python2.5/urllib.py", line 325, in open_http
55  File "/vservers/.vref/planetlab-f8-i386/usr/lib/python2.5/httplib.py", line 856, in endheaders
56  File "/vservers/.vref/planetlab-f8-i386/usr/lib/python2.5/httplib.py", line 728, in _send_output
57  File "/vservers/.vref/planetlab-f8-i386/usr/lib/python2.5/httplib.py", line 695, in send
58  File "/vservers/.vref/planetlab-f8-i386/usr/lib/python2.5/httplib.py", line 663, in connect
59  IOError: [Errno socket error] (-3, 'Temporary failure in name resolution')
60  """
61 
62  text = summarize_block(["['Timestamps match']"], text)
63  text = summarize_block(["['Expired signature']"], text)
64 
65  text = summarize_block(['Traceback', 'line 75', 'line 49', 'ValueError'], text)
66 
67  """
68  1251205775.07:PID-14482:Traceback (most recent call last):
69  File "/home/uw_seattle/seattle_repy/nmrequesthandler.py", line 75, in handle_request
70  File "nodemanager.repyhelpercache/session_repy.py", line 49, in session_recvmessage
71  ValueError: Bad message size
72  """
73 
74  text = summarize_block(['restarting advert'], text)
75 
76  text = summarize_block(['[do_rsync] New metainfo not signed correctly. Not updating.'], text)
77 
78 
79  text = summarize_block(['[safe_download] Failed to download http://seattle.cs.washington.edu/couvb/updatesite/0.1/metainfo'], text)
80 
81 
82  text = summarize_block(["The metainfo indicates no update is needed: ['Timestamps match'"], text)
83  """
84 
85  """
86  text = summarize_block(['Traceback', 'line 5942', 'line 5823', 'openDHT announce error: Socket closed'], text)
87  text = summarize_block(['Traceback', 'line 5942', 'line 5823', "openDHT announce error: (111, 'Connection refused'"], text)
88  text = summarize_block(['Traceback', 'line 5942', 'line 5823', 'openDHT announce error: timed out'], text)
89  text = summarize_block(['Traceback', 'line 5942', 'line 5823', 'centralized announce error: timed out'], text)
90  text = summarize_block(['Traceback', 'line 5942', 'line 5823', "centralized announce error: (111, 'Connection refused')"], text)
91  text = summarize_block(['Traceback', 'line 5942', 'line 5823', "centralized announce error: Socket Timeout"], text)
92  text = summarize_block(['Traceback', 'line 5942', 'line 5823', "centralized announce error: (-2, 'Name or service not known')"], text)
93
94
95
96  """
97  1249514361.81:PID-11141:Traceback (most recent call last):
98  File "/home/uw_seattle/seattle_repy/nmadvertise.py", line 5942, in run
99  File "/home/uw_seattle/seattle_repy/nmadvertise.py", line 5823, in advertise_announce
100  AdvertiseError: openDHT announce error: (111, 'Connection refused')
101  """
102
103
104  text = summarize_block(['Traceback', 'line 92', 'line 217', 'line 260',
105      'line 215', "NameError: global name 'sha_hash' is not defined"], text)
106  """
107  1252386102.32:PID-6552:Traceback (most recent call last):
108  File "/home/uw_seattle/seattle_repy/nmrequesthandler.py", line 92, in handle_request
109  File "/home/uw_seattle/seattle_repy/nmrequesthandler.py", line 217, in process_API_call
110  File "/home/uw_seattle/seattle_repy/nmrequesthandler.py", line 260, in ensure_is_correctly_signed
111  File "nodemanager.repyhelpercache/signeddata_repy.py", line 215, in signeddata_issignedcorrectly
112  NameError: global name 'sha_hash' is not defined
113  """
114
115 
116  text = summarize_block(['[fresh_software_updater] Fresh software updater started'], text)
117 
118  text = summarize_block(['Something is wrong with the metainfo'], text)
119 
120  text = summarize_block(['Traceback', 'line 151', 'line 89', "line 222",
121      'line 190', 'line 325', 'line 856', 'line 728', 'line 695', 'line 679', 
122      'IOError: [Errno socket error] timed out'], text)
123     
124  """
125  1252011627.59:PID-7346:Traceback (most recent call last):
126  File "softwareupdater.py", line 151, in safe_download
127  File "/vservers/.vref/planetlab-f8-i386/usr/lib/python2.5/urllib.py", line 89, in urlretrieve
128  File "/vservers/.vref/planetlab-f8-i386/usr/lib/python2.5/urllib.py", line 222, in retrieve
129  File "/vservers/.vref/planetlab-f8-i386/usr/lib/python2.5/urllib.py", line 190, in open
130  File "/vservers/.vref/planetlab-f8-i386/usr/lib/python2.5/urllib.py", line 325, in open_http
131  File "/vservers/.vref/planetlab-f8-i386/usr/lib/python2.5/httplib.py", line 856, in endheaders
132  File "/vservers/.vref/planetlab-f8-i386/usr/lib/python2.5/httplib.py", line 728, in _send_output
133  File "/vservers/.vref/planetlab-f8-i386/usr/lib/python2.5/httplib.py", line 695, in send
134  File "/vservers/.vref/planetlab-f8-i386/usr/lib/python2.5/httplib.py", line 679, in connect
135  IOError: [Errno socket error] timed out
136  """
137 
138 
139  text = summarize_block(['Another software updater old process (pid: '], text)
140  text = summarize_block(['Loading config', 'Traceback', 'line 491', 'line 419', 'KeyError'], text)
141  text = summarize_block([':need more than 1 value to unpack'], text)
142 
143  text = summarize_block(['Another node manager process'], text)
144  """
145  710:[ERROR]:Another node manager process (pid: 11141) is running
146  """
147 
148  text = summarize_block(['node manager is alive'], text)
149 
150 
151  # the below are fixes if the same file is processed multiple times.
152  text = summarize_block(['_____', '', '______'], text, False)
153 
154  text = summarize_block(['Replaced 0 in', '', '___'], text, False)
155 
156  text = summarize_block(['_____', 'Replaced 0 in', '', '___'], text, False)
157 
158  text = trim_newlines(text)
159 
160  text = trim_newlines(text)
161 
162  return text
163
164
165
166def trim_newlines(text):
167  """
168  <Purpose>
169    Replaces \n\n\ns with single \n repeatedly.
170
171  <Arguments>
172    text:
173      the text to replace.
174
175  <Exceptions>
176    None.
177
178  <Side Effects>
179    None.
180
181  <Returns>
182    String. Returns text but with all the changes.
183  """
184 
185  while text != text.replace('\n\n\n', '\n'):
186    text = text.replace('\n\n\n', '\n')
187    time.sleep(.1)
188  text = text.replace('\n\n\n', '\n')
189  return text
190
191
192def summarize_block(pattern_array, text, log = True):
193  """
194  <Purpose>
195    The purpose is to summarize a file that has multiple similar entries
196    into something concise
197
198  <Arguments>
199    pattern_array:
200      array of strings.  The array is a pattern to look for, where each element corresponds to
201      what the (n+1)th line should look like. e.g.:
202       
203          ['Traceback', 'line 1234', 'line 3333', 'KeyError']
204         
205      will look for 'Traceback' on a line, once it finds it it will look for 'line 1234' on the
206      following line and then 'line 3333' on the following line, and 'KeyError' on the next line.
207      If it is found the "counter" for instances of this pattern will be increased, and those lines
208      will be removed from the argument text
209    text:
210      This is the text which will be searched, line by line for the pattern
211
212  <Exceptions>
213    None.
214
215  <Side Effects>
216    Careless pattern-making can cause unwanted truncation of the text
217
218  <Returns>
219    String.  The original text, but with the truncation and changes noted.
220  """
221 
222  # the number of current matches/replacements made
223  number_of_matches = 0
224 
225  # flag for whether we matched something or not
226  matched_flag = False
227 
228  # conver to array as we're going to enumerate through it by line #s
229  text_array = text.splitlines()
230 
231  # will hold a sample of what was actually replaced and what matched the pattern
232  sample_replace = []
233 
234 
235  for each_index in range(len(text_array)):
236 
237    # grab each lien from the text array
238    each_line = text_array[each_index]
239   
240    # we found the first pattern
241    if pattern_array[0] in each_line:
242      if each_index >= 2 and not text_array[each_index - 1].startswith('Replaced ') and not text_array[each_index - 2].startswith('___'):
243        matched_flag = True
244     
245        # do we have enough lines to match to the pattern?
246        if len(pattern_array) - 1 + each_index < len(text_array):
247         
248          for i in range(len(pattern_array)):
249            # try to match the other patterns now
250
251            if matched_flag:
252
253              if len(text_array) > each_index + i:
254                next_line = text_array[each_index + i]
255              else:
256                temp = '\n'.join(text_array)
257               
258                temp = trim_newlines(temp)
259               
260                if number_of_matches > 0 and log:
261                  temp =  temp + '\n______________________________\n'+\
262                      'Replaced '+str(number_of_matches)+' instances of the following pattern: \n'+\
263                      '\n'.join(sample_replace)+'\n'+\
264                      '______________________________\n'
265                return temp
266               
267
268              if pattern_array[i] in next_line:
269                if pattern_array[i] == '' and next_line != '':
270                  matched_flag = False
271                else:
272                  matched_flag = True
273              else:
274                matched_flag = False
275
276           
277          if matched_flag:
278            number_of_matches += 1
279            # we matched all the flags, time to replace by grabbing n indexes
280            # and replacing them
281           
282            # do we have a sample of what we replaced?
283            if not sample_replace:
284              sample_replace = text_array[each_index:each_index+len(pattern_array)]
285             
286            for i in range(len(pattern_array)):
287              text_array[each_index + i] = ''
288           
289            each_index += len(pattern_array)
290            matched_flag = False
291         
292  temp = '\n'.join(text_array)
293  temp = trim_newlines(temp)
294  if number_of_matches > 0 and log:
295    temp =  temp + '\n______________________________\n'+\
296        'Replaced '+str(number_of_matches)+' instances of the following pattern: \n'+\
297        '\n'.join(sample_replace)+'\n'+\
298        '______________________________\n'
299  return temp
300
301
302
303
304# TODO: what if www resolves to multiple IPs? ex: google.com
305# TODO: handle 'is an alias' case, ex: google.com
306def dnslookup(ip_or_hostname):
307  """
308  <Purpose>
309    Changes IPs to hostnames, and hostnames to IPs using the 'host' program on linux machines.
310
311  <Arguments>
312    ip_or_hostname:
313      a string that is either the IP or a hostname
314     
315  <Exceptions>
316    Raises an exception if host returns an unexpected value, or if there is
317    some error in the lookup.
318
319  <Side Effects>
320    None.
321
322  <Returns>
323    String.  Returns either the hostname or the IP or same string if a
324      reverse lookup can't be done.
325  """
326
327  # ip -> hostname and
328  # hostname -> ip
329  out, err, retcode = shellexec2('host '+ip_or_hostname)
330  # sample strings:
331  # 1. [ip] domain name pointer [hostname].
332  # 2. [hostname] has address [ip]
333  if retcode == 0:
334    if out.find('domain name pointer') > -1:
335      # this is of type 1, so return the hostname
336      hostname = out.split(' ')[-1].strip('\r\n .')
337      return hostname.lower()
338    elif out.find('has address') > -1:
339      # type 2, so return the IP
340      ip = out.split(' ')[-1].strip('\r\n .')
341      return ip
342      # error
343    else:
344      raise Exception('Error in dnslookup: unexpected return value ('+str(out.strip('\n\r '))+')')
345  else:
346    if out.find('not found') > -1:
347      # probably networkip
348      return ip_or_hostname.lower()
349    raise Exception('Error in dnslookup ('+str(retcode)+'): '+str(out)+', '+str(err)+' .')
350
351
352def shellexec2(cmd_str):
353  """
354  <Purpose>
355    Uses subprocess to execute the command string in the shell.
356     
357  <Arguments>
358    cmd_str:  The string to be treated as a command (or set of commands,
359                deploy_logging.separated by ;).
360   
361  <Exceptions>
362    None.
363
364  <Side Effects>
365    None.
366
367  <Returns>
368    A tuple containing (stdout, strerr, returncode)
369
370    Detailed:
371    stdout: stdout printed to console during command execution.
372    strerr: error (note: some programs print to strerr instead of stdout)
373    returncode: the return code of our call. If there are multiple commands,
374                then this is the return code of the last command executed.
375  """
376
377  # get a handle to the subprocess we're creating..
378  handle = subprocess.Popen(cmd_str, shell=True, 
379      stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
380
381  # execute and grab the stdout and err
382  stdoutdata, strerrdata = handle.communicate("")
383
384  # The return code... 
385  returncode = handle.returncode
386 
387  return stdoutdata, strerrdata, returncode
Note: See TracBrowser for help on using the browser.