Please note that the CVS and issue trackers have moved to GitHub. These Trac pages are no longer kept up-to-date.

root/seattle/trunk/preparetest.py@5637

Revision 5637, 10.3 KB (checked in by justinc, 7 years ago)

This is a fix for #1081. I didn't fix things like make_base_installers or
dist that don't need to run on different platforms.

Line 
1"""
2<Author>
3  Cosmin Barsan
4 
5  Edited to add an optional argument to also copy the repy tests by
6  Brent Couvrette on November 13, 2008.
7
8  Conrad Meyer, Thu Nov 26 2009: Move dynamic ports code from run_tests.py
9  to preparetest.py.
10<Start Date>
11  October 3, 2008
12
13<Description>
14  This script has been adapted from the bash script preparetest.  The
15  script first erases all the files in the target folder, then copies
16  the necessary test files to it. Afterwards, the .mix files in the
17  target folder are run through the preprocessor.  It is required that
18  the folder passed in as an argument to the script exists.
19
20<Usage>
21  preparetest.py <target_folder> <-t>
22
23  if -t is specified, the repy tests will also be included, otherwise, they will not
24
25<Notes>
26  This file is used as a library by trunk/www/deploy_state_transitions.py
27  If you make ANY changes to this file please let Ivan know so that the
28  other script can continue to function correctly. Thanks. (IB 01/19/09)
29
30  This file is also used directly by trunk/dist/make_base_installers.py. Also
31  let Zack know if any adaptions are made to this file so the base installers
32  can continue to be created correctly.  See ticket #501 about removing or
33  adapting make_base_installer.py's dependence on this file.  (Zack 7/2/09)
34
35"""
36
37import sys
38import glob
39import os
40import random
41import shutil
42import optparse
43import subprocess
44
45sys.path.insert(0, os.path.join(os.getcwd(), "repy", "tests"))
46import testportfiller
47sys.path = sys.path[1:]
48
49# Whether to copy files that enable shims. In case of major bugs in shims,
50# simply turn it off to revert back to the shim-less architecture. Added by
51# Danny Yuxing Huang.
52USE_SHIMS = False
53
54#define a function to use for copying the files matching the file expression to the target folder
55#file_expr may contain wildcards
56#target must specify an existing directory with no wildcards
57def copy_to_target(file_expr, target):
58  files_to_copy = glob.glob(file_expr)
59  for file_path in files_to_copy:
60    if os.path.isfile(file_path):
61      shutil.copyfile(file_path,target +"/"+os.path.basename(file_path))
62
63
64#iterate through the .mix files in current folder and run them through the preprocessor
65#script_path must specify the name of the preprocessor script
66#the working directory must be set to the directory containing the preprocessor script prior to executing this function.
67def process_mix(script_path, verbose):
68  mix_files = glob.glob("*.mix")
69  error_list = []
70
71  for file_path in mix_files:
72    #generate a .py file for the .mix file specified by file_path
73    processed_file_path = (os.path.basename(file_path)).replace(".mix",".py")
74    (theout, theerr) =  exec_command(sys.executable+" " + script_path + " " + file_path + " " + processed_file_path)
75
76    # If there was any problem processing the files, then notify the user.
77    if theerr:
78      print "Unable to process the file: " + file_path
79      error_list.append((file_path, theerr))
80      # If the verbose option is on then print the error.
81     
82  if verbose:
83    print "\n" + '#'*50 + "\nPrinting all the exceptions (verbose option)\n" + '#'*50
84    if len(error_list) == 0:
85      print "NONE!!"
86    for file_name, error in error_list:
87      print "\n" + file_name + ":"
88      print error
89      print '-'*80
90
91
92def exec_command(command):
93# Windows does not like close_fds and we shouldn't need it so...
94  p =  subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
95
96  # get the output and close
97  theout = p.stdout.read()
98  p.stdout.close()
99
100  # get the errput and close
101  theerr = p.stderr.read()
102  p.stderr.close()
103
104  # FreeBSD prints on stdout, when it gets a signal...
105  # I want to look at the last line.   it ends in \n, so I use index -2
106  if len(theout.split('\n')) > 1 and theout.split('\n')[-2].strip() == 'Terminated':
107    # lose the last line
108    theout = '\n'.join(theout.split('\n')[:-2])
109   
110    # however we threw away an extra '\n' if anything remains, let's replace it
111    if theout != '':
112      theout = theout + '\n'
113
114  # everyone but FreeBSD uses stderr
115  if theerr.strip() == 'Terminated':
116    theerr = ''
117
118  # Windows isn't fond of this either...
119  # clean up after the child
120#  os.waitpid(p.pid,0)
121
122  return (theout, theerr)
123
124
125helpstring = """python preparetest.py [-t | -v] <foldername>"""
126
127# Prints the given error message and the help string, then exits
128def help_exit(errMsg, parser):
129  print errMsg
130  parser.print_help()
131  sys.exit(1)
132
133def main():
134  repytest = False
135  RANDOMPORTS = False
136  verbose = False
137       
138  target_dir = None
139
140  # Parse the options provided.
141  parser = optparse.OptionParser()
142
143  parser.add_option("-t", "--include-test-files", action="store_true",
144                    dest="include_tests", help="Include the test " +
145                    "files in the output directory.")
146  parser.add_option("-v", "--verbose", action="store_true",
147                    dest="verbose", help="Show more output on failure")
148  parser.add_option("-r", "--randomports", action="store_true", 
149                    dest="randomports", help="Fill in the ports randomly")
150
151  (options, args) = parser.parse_args()
152
153  # Set certain variables according to the options provided.
154  if options.include_tests:
155    repytest = True
156  if options.randomports:
157    RANDOMPORTS = True
158  if options.verbose:
159    verbose = True
160
161  # Extract the target directory if available.
162  if len(args) == 0:
163    help_exit("Please pass the target directory as a parameter.", parser)
164  else:
165    target_dir = args[0]
166
167
168  #store root directory
169  current_dir = os.getcwd()
170
171  # Make sure they gave us a valid directory
172  if not( os.path.isdir(target_dir) ):
173    help_exit("given foldername is not a directory", parser)
174
175  #set working directory to the test folder
176  os.chdir(target_dir) 
177  files_to_remove = glob.glob("*")
178
179  #clean the test folder
180  for f in files_to_remove: 
181    if os.path.isdir(f):
182      shutil.rmtree(f)         
183    else:
184      os.remove(f)
185
186  #go back to root project directory
187  os.chdir(current_dir) 
188
189  #now we copy the necessary files to the test folder
190  copy_to_target("repy/*", target_dir)
191  copy_to_target("nodemanager/*", target_dir)
192  copy_to_target("portability/*", target_dir)
193  copy_to_target("seattlelib/*", target_dir)
194  copy_to_target("seash/*", target_dir)
195  copy_to_target("softwareupdater/*", target_dir)
196  copy_to_target("autograder/nm_remote_api.mix", target_dir)
197  copy_to_target("keydaemon/*", target_dir)
198  # The license must be included in anything we distribute.
199  copy_to_target("LICENSE.TXT", target_dir)
200
201  if USE_SHIMS:
202    # Copy over the files needed for using shim.
203    copy_to_target("production_nat_new/src/*", target_dir)
204    copy_to_target("production_nat_new/src/nmpatch/nmmain.py", target_dir)
205    copy_to_target("production_nat_new/src/nmpatch/nminit.mix", target_dir)
206    copy_to_target("production_nat_new/src/nmpatch/nmclient.repy", target_dir)
207    copy_to_target("production_nat_new/src/nmpatch/nmclient_shims.mix", target_dir)
208    copy_to_target("production_nat_new/src/nmpatch/sockettimeout.repy", target_dir)
209
210 
211  # Only copy the tests if they were requested.
212  if repytest:
213    # The test framework itself.
214    copy_to_target("utf/*.py", target_dir)
215    # The various tests.
216    copy_to_target("repy/tests/*", target_dir)
217    copy_to_target("nodemanager/tests/*", target_dir)
218    copy_to_target("portability/tests/*", target_dir)   
219    copy_to_target("seash/tests/*", target_dir)
220    copy_to_target("oddball/tests/*", target_dir)
221    copy_to_target("seattlelib/tests/*", target_dir)
222    copy_to_target("keydaemon/tests/*", target_dir)
223    copy_to_target("utf/tests/*", target_dir)
224    # jsamuel: This file, dist/update_crontab_entry.py, is directly included by
225    # make_base_installers and appears to be a candidate for removal someday.
226    # I assume zackrb needed this for installer testing.
227    copy_to_target("dist/update_crontab_entry.py", target_dir)
228
229    if USE_SHIMS:
230      # Unit tests for shims
231      copy_to_target("production_nat_new/src/unit_tests/*", target_dir)
232
233  #set working directory to the test folder
234  os.chdir(target_dir)
235
236
237  # set up dynamic port information
238  if RANDOMPORTS:
239    print "\n[ Randomports option was chosen ]\n"+'-'*50
240    portstouseasints = random.sample(range(52000, 53000), 3)
241    portstouseasstr = []
242    for portint in portstouseasints:
243      portstouseasstr.append(str(portint))
244   
245    print "Randomly chosen ports: ",portstouseasstr
246    testportfiller.replace_ports(portstouseasstr, portstouseasstr)
247
248    # Replace the string <nodemanager_port> with a random port
249    random_nodemanager_port = random.randint(53000, 54000)
250    print "Chosen random nodemanager port: " + str(random_nodemanager_port)
251    print '-'*50 + "\n"
252    replace_string("<nodemanager_port>", str(random_nodemanager_port), "*nm*")
253    replace_string("<nodemanager_port>", str(random_nodemanager_port), "*securitylayers*")
254   
255  else:
256    # if this isn't specified, just use the default ports...
257    testportfiller.replace_ports(['12345','12346','12347'], ['12345','12346','12347'])
258
259    # Use default port 1224 for the nodemanager port if --random flag is not provided.
260    replace_string("<nodemanager_port>", '1224', "*nm*")
261    replace_string("<nodemanager_port>", '1224', "*securitylayers*")
262
263
264
265  #call the process_mix function to process all mix files in the target directory
266  process_mix("repypp.py", verbose)
267
268  #go back to root project directory
269  os.chdir(current_dir) 
270
271
272
273
274def replace_string(old_string, new_string, file_name_pattern="*"):
275  """
276  <Purpose>
277    Go through all the files in the current folder and replace
278    every match of the old string in the file with the new
279    string.
280
281  <Arguments>
282    old_string - The string we want to replace.
283 
284    new_string - The new string we want to replace the old string
285      with.
286
287    file_name_pattern - The pattern of the file name if you want
288      to reduce the number of files we look at. By default the
289      function looks at all files.
290
291  <Exceptions>
292    None.
293
294  <Side Effects>
295    Many files may get modified.
296
297  <Return>
298    None
299  """
300
301  for testfile in glob.glob(file_name_pattern):
302    # Read in the initial file.
303    inFile = file(testfile, 'r')
304    filestring = inFile.read()
305    inFile.close()
306
307    # Replace any form of the matched old string with
308    # the new string.
309    filestring = filestring.replace(old_string, new_string)
310
311    # Write the file back.
312    outFile = file(testfile, 'w')
313    outFile.write(filestring)
314    outFile.close()
315
316
317
318
319
320
321if __name__ == '__main__':
322  main()
Note: See TracBrowser for help on using the browser.