Please note that the CVS and issue trackers have moved to GitHub. These Trac pages are no longer kept up-to-date.

root/seattle/trunk/deploymentscripts/missing_seattle_install_p.py@5637

Revision 2810, 7.3 KB (checked in by konp, 10 years ago)

Fixed high cpu usage, added summary tool, a number of other fixes and improvements as well. Note: blackbox node keys have not been added yet.

Line 
1"""
2<Program Name>
3  missing_seattle_install_p.py
4
5<Started>
6  June 2009
7
8<Author>
9  n2k8000@u.washington.edu
10  Konstantin Pik
11
12<Purpose>
13  This file will read in a list file passed into it, and from that list it
14  will install seattle on all of those nodes.  The list file is to be in the
15  file format specified for .LIST files (!user:[username], followed by list of
16  IPs).
17
18<Usage>
19  python missing_seattle_install.py missing.list
20 
21  Note: missing.list is the default file name.
22 
23"""
24
25import thread
26import time
27import sys
28
29# for remote_shellexec
30import deploy_network
31import deploy_threading
32import parallelize_repy
33
34# the running thread counter
35thread_counter = 0
36
37# the lock on the thread_counter, just to make sure add/sub is atomic
38thread_lock = thread.allocate_lock()
39
40
41
42def get_remote_hosts_from_file(fname = 'missing.list'):
43  """
44  <Purpose>
45    Returns a list of the IP as read from file specified.
46
47    File format is:
48 
49    !user:[username]
50    [IPs]
51
52    [username] is the username that will be used until a new $username is
53      specified in the same format. NOTE: Username is case sensitive.
54    [IPs] are a list of IPs/hostname (one per line) associated with that
55      username
56
57  <Arguments>
58    fname:
59      Optional. The filename containing the IPs of the remote machines.  File
60      must be in the same directory as this script.
61   
62  <Exceptions>
63    Catches a thrown exception if the IP file is not found.
64
65  <Side Effects>
66    None.
67
68  <Returns>
69    Returns a list of tuples with (username, ip) on success, False on failure
70  """
71
72  # IP file must be in the same dir as this script
73  try:
74    file_of_ips = open(fname, 'r')
75  except Exception, e:
76    print 'Error: Are you missing your list of remote hosts? ('+str(e)+')'
77    file_of_ips.close()
78    return False
79  else:
80    # flag on whether we have any remote hosts (there are users, and comments
81    # in the file as well
82    have_one_ip = False
83
84    # initialize dict   
85    users_ip_tuple_list = []
86
87    current_username = ''
88
89    # Python docs suggest doing this instead of reading in whole file into mem:
90    for line in file_of_ips:
91
92      # if first chars match what we want ('!user:' is 6 chars long)
93      if line[0:6].lower() == '!user:':
94        # grab everything after the '!user:' string
95        # -1 so we drop the \n and leading/trailing spaces
96        current_username = line[6:-1].strip()
97      else:
98        # ignore blank lines and spaces
99        if line.strip('\n '):
100          # and ignore comments (lines starting with #)
101          if line.strip('\n ')[0] != '#':
102            # if we get here, then we have an IP so we need to  check that
103            # user is not empty.. log err if it is and complain.
104            if not current_username:
105              print 'Critical Error: No username specified for remote host group!'
106              file_of_ips.close()
107              return False
108
109            # add (username, remote_host) pair
110            users_ip_tuple_list.append((current_username, line.rstrip('\n ')))
111            # set flag that we have at least one ip
112            have_one_ip = True
113
114    # return true only if we have at least ONE ip that we added to the list
115    # and not just a bunch of users
116    if have_one_ip:
117      # lets make the list a set, which is a cheap way of getting rid of
118      # duplicates, then cast back to list.
119      finalized_list = list(set(users_ip_tuple_list))
120      print "Found "+str(len(finalized_list))+" unique hosts to connect to."
121      file_of_ips.close()
122      return finalized_list
123    file_of_ips.close()
124    return False
125
126 
127
128def format_print(out, err):
129  """
130  <Purpose>
131    Will print out the non-empty out/err strings once they're properly
132    formatted. Intended to format stdout and stderr. Also will print to
133    missing.log
134
135  <Arguments>
136    out:
137      stdout
138    err:
139      std error
140
141  <Exceptions>
142    None.
143
144  <Side Effects>
145    None.
146
147  <Returns>
148    None.
149  """
150  try:
151    out = out.strip('\n\r ')
152    err = err.strip('\n\r ')
153   
154    logfilehandle = open('missing.log', 'a')
155   
156    if out:
157      print out
158      logfilehandle.write(out+'\n')
159    if err:
160      print err
161      logfilehandle.write(err+'\n')
162     
163    logfilehandle.close()
164  except Exception, e:
165    print 'Error while writing file and/or formatting data'
166    print e
167   
168  return
169
170
171
172
173def worker(username_host_tuple):
174  """
175  <Purpose>
176    Worker thread that makes calls to remote_shellexec
177
178  <Arguments>
179    username_host_tuple:
180      username_host_tuple[0] is
181      username:
182        the username to log in as
183       
184      username_host_tuple[1] is
185      host:
186        the remote hostname/ip to install on.
187
188  <Exceptions>
189    None.
190
191  <Side Effects>
192    None.
193
194  <Returns>
195    None.
196  """
197  username = 'uw_seattle' #username_host_tuple[0]
198  host = username_host_tuple
199 
200 
201  # build up a command string that'll download and install seattle
202  cmd_list = []
203 
204  # 1. Remove old file, and download the file
205  cmd_list.append('cd seattle_repy; ./uninstall.sh; cd ~; rm -rf seattle_repy')
206 
207  cmd_list.append('rm -rf seattle_linux.tgz')
208 
209  cmd_list.append('wget https://seattlegeni.cs.washington.edu/geni/download/flibble/seattle_linux.tgz')
210  #cmd_list.append('wget --no-check-certificate https://blackbox.cs.washington.edu/geni/html/tukwila/seattle_linux.tgz')
211
212  # 2. Untar
213  cmd_list.append('tar -xf seattle_linux.tgz')
214 
215  # 3. Change into seattle_repy directory and execute python install.sh to start seattle
216  cmd_list.append('cd seattle_repy; ./install.sh > /dev/null 2> /dev/null < /dev/null&')
217 
218  # merge into a command string
219  cmd_str = '; '.join(cmd_list)
220 
221  out, err, retcode = deploy_network.remote_shellexec(cmd_str, username, host)
222  format_print(out, err)
223
224 
225 
226def main():
227  """
228  <Purpose>
229    Entry point into the program.  Reads the hosts that need installing
230    from file and then starts the threads that will take care of downloading
231    and installing seattle.  Then waits for all threads to finish.  This takes
232    a while as an RSA key needs to be generated during each install.
233
234  <Arguments>
235    None
236
237  <Exceptions>
238    Possible exception when launching new threads.
239
240  <Side Effects>
241    None.
242
243  <Returns>
244    None.
245  """
246 
247  # start the timeout monitor thread
248  deploy_threading.init()
249 
250  # the fn of the file that contains the list of nodes we'll be using
251  nodelist_fn = ''
252 
253  # did we get a parameter passed in? if so that's our fn
254  if len(sys.argv) > 1:
255    nodelist_fn = sys.argv[1]
256    print 'Using '+nodelist_fn+' filename to read in hostnames'
257  else:
258    print 'Using default missing.list filename to read in hostnames'
259   
260  # get hosts from file
261  #if nodelist_fn:
262  #  hosts = get_remote_hosts_from_file(nodelist_fn)
263  #else: # use default fn
264  #  hosts = get_remote_hosts_from_file()
265  # '128.208.1.130', 128.208.1.217
266 
267  # or manually type in hosts here
268  hosts = [ ]
269 
270  # if we have hostnames
271  if hosts:
272   
273   
274    # BEGIN
275    func_handle = parallelize_repy.parallelize_initfunction(hosts, worker, 10)
276   
277    size = str(len(hosts))
278   
279    while not parallelize_repy.parallelize_isfunctionfinished(func_handle):
280      results_dict = parallelize_repy.parallelize_getresults(func_handle)
281      print str(len(results_dict['aborted']))+' aborted, '+str(len(results_dict['exception']))+\
282          ' exceptioned, '+str(len(results_dict['returned']))+' finished of '+size+' total.'
283      time.sleep(5)
284     
285  deploy_threading.destroy()
286    # END
287     
288     
289if __name__ == "__main__":
290  main()
Note: See TracBrowser for help on using the browser.