Please note that the CVS and issue trackers have moved to GitHub. These Trac pages are no longer kept up-to-date.

root/seattle/trunk/utf/utf.py@5649

Revision 5649, 17.9 KB (checked in by albert, 7 years ago)

Partial fix for #1084. Additionally, a number of individual test files need to be fixed.

RevLine 
[3490]1#!/usr/bin/python
2"""
3<Program>
4  Seattle Test Framework
5
6
7<Author>
8  Vjekoslav Brajkovic
[3640]9  Stephen Sievers
[3490]10
11<Started>
12  2009-07-11
13 
14
15<Requirements>
16
17  <Naming Convention>
18
19    Required naming convention for every test file:
20
21    ut_MODULE{_DESCRIPTOR].py
22
23    There can be multiple descriptors associated with a module.
24
25    <Example>
26      ut_semaphore_simple_create.py
27      ut_semaphore_simple_destroy.py
28   
29  <Pragma Directives>
30
31    The token-string is a series of characters that gives a specific framework
32    instruction and arguments, if any. The number sign (#) must be the first
33    non-white-space character on the line containing the pragma; white-space
34    characters can separate the number sign and the word pragma. Following
35    #pragma, write any text that the translator can parse as preprocessing
36    tokens.
37   
38    <Example>
[4038]39      #pragma repy [RESTRICTIONS]
[3490]40      #pragma out [TEXT]
41      #pragma error [TEXT]
42
[4038]43    The parser throws an exception on unrecognized pragmas.
[4101]44
45<Modified>
46  Modified on Nov. 16, 2010 by Monzur Muhammad to add functionality for the
47  verbose option. Now if verbose is on, it display the time taken for the test.
[3490]48"""
49
50
51import glob
52import optparse
53import os
54import signal
55import subprocess
56import sys
57import time
58
59import utfutil
60
61# Valid prefix and suffix.
62SYNTAX_PREFIX = 'ut_'
63SYNTAX_SUFFIX = '.py'
64
65
66
67
68# Acceptable pragma directives.
69REPY_PRAGMA = 'repy'
70ERROR_PRAGMA = 'error'
71OUT_PRAGMA = 'out'
72
73
[4101]74# Verbose Option
75VERBOSE = False
[3490]76
[4104]77SHOW_TIME = False
78
[3490]79# UTF Exceptions.
80class InvalidTestFileError(Exception): 
81  pass
82class InvalidPragmaError(Exception): 
83  pass
84
85
86
87
88def main():
89  """
90  <Purpose>
91    Executes the main program that is the unit testing framework.
92    Tests different modules, files, capabilities, dependent on command
93    line arguments.
94   
95  <Arguments>
96    None
97
98  <Exceptions>
99    None.
100   
101  <Side Effects>
102    None
103
104  <Returns>
105    None
106  """
107  ###
108  ### Define allowed arguments.
109  ###
110  parser = optparse.OptionParser()
111
112  ### Generic Option Category.
113  group_generic = optparse.OptionGroup(parser, "Generic")
114 
115  # Verbose flag.
116  group_generic.add_option("-v", "--verbose",
117                    action="store_true", dest="verbose", default=False,
118                    help="verbose output")
[4104]119
120  # Verbose flag.
121  group_generic.add_option("-T", "--show-time",
122                    action="store_true", dest="show_time", default=False,
123                    help="display the time taken to execute a test.")
124
[3490]125  parser.add_option_group(group_generic)
126
127  ### Testing Option Category.
128  group_test = optparse.OptionGroup(parser, "Testing")
129 
130  # Test for a specific module.
131  group_test.add_option("-m", "--module", dest="module",
132                        help="run tests for a specific module", 
133                        metavar="MODULE")
134
135  # Run a specific test file.
136  group_test.add_option("-f", "--file", dest="file",
137                        help="execute a specific test file", 
138                        metavar="FILE")
139
[3640]140  # Run all tests in current directory
141  group_test.add_option("-a", "--all", dest="all", action="store_true",
142                        help="execute all test files",
143                        metavar="ALL")
144
[3490]145  parser.add_option_group(group_test)
146 
147  # All files in the current working directory.
148  all_files = glob.glob("*")
149
150  # Valid test files in the current working directory.
151  valid_files = filter_files(all_files)
152
153  ###
154  # Parse the arguments.
155  ###
156  (options, args) = parser.parse_args()
157
[3640]158
159  #Count number of args for legal number of args test
160  i = 0
161  if (options.module):
162    i = i + 1
163  if (options.all):
164    i = i + 1
165  if (options.file):
166    i = i + 1
167
[3490]168  # Test for mutual exclusion.
[3640]169  if i > 1:
[3490]170    parser.error("Options are mutually exclusive!")
171   
172
[4104]173  # Check if the show_time option is on.
174  if (options.show_time):
175    global SHOW_TIME
176    SHOW_TIME = True
[4101]177
178
[3490]179  if (options.file): # Single file.
180
181    file_path = options.file
182
[4150]183    # I need to derive the module name...
184    if not file_path.startswith('ut_') or len(file_path.split('_'))<3:
185      print "Error, cannot determine module name from filename '"+file_path+"'"
186      return
187    else:
188      module_name = file_path.split('_')[1]
189
190    # the test_module code is really poorly structured.   I need to tell it to
191    # consider the shutdown, setup, and subprocess scripts...
192    files_to_use = [file_path]
193    module_file_list = filter_files(valid_files, module = module_name)
194   
195    files_to_use = files_to_use + filter_files(module_file_list, descriptor = 'setup')
196    files_to_use = files_to_use + filter_files(module_file_list, descriptor = 'shutdown')
197    files_to_use = files_to_use + filter_files(module_file_list, descriptor = 'subprocess')
198
199    test_module(module_name, files_to_use)
200
[3490]201  elif (options.module): # Entire module.
202   
203    # Retrieve the module name.
204    module_name = options.module
205   
206    module_file_list = filter_files(valid_files, module = module_name)
[5633]207
208    # is the module empty? we should print something to indicate this is likely
209    # an error...
210    if len(module_file_list) == 0:
211      print "No module with name:",module_name
212      return
213
[3490]214    test_module(module_name, module_file_list)
215   
[3640]216  elif (options.all): #all test files
[3490]217    test_all(valid_files)
218
[3640]219  else: # If no options are present, print the usage
220   
221    print "Usage: python utf.py (-f filename | -m modulename | -a)"
222    print "-f -- test a specific filename"
223    print "-m -- test a module of modulename"
224    print "-a -- run all tests in current directory"
[3490]225
226
227
[3640]228
[4150]229def execute_and_check_program(file_path):
[3490]230  """
231  <Purpose>
[4150]232    Given the test file path, this function will execute the program and
233    monitor its behavior
[3490]234   
235  <Arguments>
236    Test file path.
237
238  <Exceptions>
239    None.
240   
241  <Side Effects>
242    None
243
244  <Returns>
245    None
246  """
247  file_path = os.path.normpath(file_path)
248  testing_monitor(file_path)
249
250
251
252
253def test_module(module_name, module_file_list):
254  """
255  <Purpose>
256    Execute all test files contained within module_file_list matching the
257    module_name in the form of each test file name 'ut_<module_name>_<descriptor>.py'
258
259  <Arguments>
260    module_name: module name to be tested
261    module_file_list: a list of files to be filtered by module name and ran through
262      the testing framework
263
264  <Exceptions>
265    None
266
267  <Side Effects>
268    None
269 
270  <Returns>
271    None
272  """
273  print 'Testing module:', module_name
274 
275  setup_file = None
276 
277  # Given all test files for the specified module name, find the file whose
278  # descriptor equals 'setup' (there can be only one such file name).
279  filtered_files = filter_files(module_file_list, descriptor = 'setup')
280 
281  if filtered_files:
282    setup_file = filtered_files.pop()
283    module_file_list.remove(setup_file)
284
285  subprocess_file = None
286 
287  filtered_files = filter_files(module_file_list, descriptor = 'subprocess')
288  if filtered_files:
289    subprocess_file = filtered_files.pop()
290    module_file_list.remove(subprocess_file)
291 
292  shutdown_file = None
293 
294  # Given all test files for the specified module name, find the file whose
295  # descriptor equals 'shutdown' (there can be only one such file name).
296 
297  filtered_files = filter_files(module_file_list, descriptor = 'shutdown')
298  if filtered_files:
299    shutdown_file = filtered_files.pop()
300    module_file_list.remove(shutdown_file)
301
302 
303  sub = None
[4181]304  # If we must open a process to run concurrently with the tests, we will use
305  # its stdin to indicate when to stop...
[3490]306  if subprocess_file:
307    print "Now starting subprocess: " + subprocess_file
[5637]308    sub = subprocess.Popen([sys.executable, subprocess_file], stdin=subprocess.PIPE)
[3523]309    # Give the process time to start
310    time.sleep(30)
[3490]311
312  if setup_file:
313    print "Now running setup script: " + setup_file
[4150]314    execute_and_check_program(setup_file)   
[3490]315
[4101]316  start_time = time.time()
317
318  # Run the module tests
[3490]319  for test_file in module_file_list: 
[4150]320    execute_and_check_program(test_file)
[3490]321
[4101]322  end_time = time.time()
323
324
[3490]325  if shutdown_file:
326    print "Now running shutdown script: " + shutdown_file
[4150]327    execute_and_check_program(shutdown_file)
[3490]328
[4181]329  #If we opened a subprocess, we need to stop it by shutting its stdin
[3490]330  if sub:
[4181]331    print "Now stopping subprocess: " + subprocess_file   
332    sub.stdin.close()
333    sub.wait()
[4012]334
[4181]335
[4104]336  if SHOW_TIME:
337    print "Total time taken to run tests on module %s is: %s" % (module_name, str(end_time-start_time)[:6])
[3490]338
[4101]339
340
341
[3490]342def test_all(file_list):
343  """
344  <Purpose>
345    Given the list of valid test files, this function will test each module
346    within the test file list
347   
348  <Arguments>
349    file_list: List of test files to be ran
350
351  <Exceptions>
352    None
353   
354  <Side Effects>
355    None
356
357  <Returns>
358    None
359  """
360  module_dictionary = { }
361
362  # Map test files to their respective modules.
363  # dictionary[module name] -> list(test files)
364  for test_file in file_list: 
365    (module, descriptor) = parse_file_name(test_file)
366   
367    if module_dictionary.has_key(module):
368      module_dictionary[module].append(test_file)
369    else:
370      module_dictionary[module] = [test_file]
371 
372  # Test each module.
373  for module_name, module_file_list in module_dictionary.iteritems():
374    test_module(module_name, module_file_list)
375
376
377
378
379def testing_monitor(file_path):
380  """
381  <Purpose>
382    Executes and prints the results of the unit test contained within 'file_path'
383   
384 
385  <Arguments>
386    file_path: File to be used within the testing_monitor
387
388  <Exceptions>
389    InvalidPragmaError: if there is an invalid pragma within the source file
390
391  <Side Effects>
392    None
393
394  <Returns>
395    None
396  """
397
398  # Source the file.
399  file_object = open(file_path)
400  source = file_object.read()
401
402  (tail, head) = os.path.split(file_path)
403
404  (module, descriptor) = parse_file_name(head)
405  print "\tRunning: %-50s" % head,
[3941]406  # flush output in case the test hangs...
407  sys.stdout.flush()
[3490]408
409  # Parse all pragma directives for that file.
410  try: 
411    pragmas = parse_pragma(source)
412  except InvalidPragmaError:
413    print '[ ERROR ]'
414    print_dashes()
415    print 'Invalid pragma directive.'
416    print_dashes()
417   
418    return
419
420  # Now, execute the test file.
[4101]421  start_time = time.time()
[3490]422  report = execution_monitor(file_path, pragmas)
423
[4101]424  # Calculate the time taken for the test
425  end_time = time.time()
426  time_taken = str(end_time - start_time)[:5]
427  if len(time_taken) < 5:
428    time_taken = " "*(5-len(time_taken)) + time_taken
429
[3490]430  if report:
[4104]431    if SHOW_TIME:
[4101]432      print '[ FAIL ] [ %ss ]' % time_taken
433
434    else:
435      print '[ FAIL ]'
436
437
438
[3490]439    print_dashes()
440   
441    for key, value in report.items():
[4055]442      print 'Standard', key, ':'
443      produced_val, expected_val = value
444      print "."*30 + "Produced" + "."*30 + "\n" + str(produced_val)
445      print "."*30 + "Expected" + "."*30 + "\n" + str(expected_val)
[3490]446      print_dashes()
447   
448  else:
[4104]449    if SHOW_TIME:
[4101]450      print '[ PASS ] [ %ss ]' % time_taken
451    else:
452      print '[ PASS ]'
[3490]453
454
455
456
457def execution_monitor(file_path, pragma_dictionary):
458  """
459  <Purpose>
460    Executes a unit test written with a source contained in file_path. If the source
461    contains any pragmas (#pragma out, #pragma repy, #pragma err), the unit testing
462    framework creates the report differently. If there is a repy pragma, the test
463    executes in repy, not python. If there is an out or err pragma, the unit testing
464    framework will include that there was to be output in the report.
465 
466  <Arguments>
467    file_path: file to be executed under the framework
468    pragma_dictionary: dictionary of pragmas within this test file
469
470  <Exceptions>
471    None
472
473  <Side Effects>
474    None
475
476  <Returns>
477    A report containing information about any unexpected output:
478    { Pragma Type : (Produced, Expected), ... }
479  """
480
481  # Status report.
482  report = { }
483
[5637]484  executable = sys.executable
[3490]485  popen_args = [ executable ]
486
487  if pragma_dictionary.has_key(REPY_PRAGMA):
488    repy = 'repy.py'
489    default_restriction = 'restrictions.default'
490   
491    # Did the user specify a non-default restrictions file?
[3551]492    repyArgs = pragma_dictionary[REPY_PRAGMA]
493    if not repyArgs: 
494      repyArgs = default_restriction
[3490]495   
496    popen_args.append(repy)
[3551]497
498    # For tests requiring repy arguments besides restrictions.default
499    # the user must specify them after the pragma
500    arguments = repyArgs.split(" ")
501    for element in arguments:
502      popen_args.append(element)
[3490]503 
504  popen_args.append(file_path)
505
506  # Execute the program.
[5649]507  (rawout, error) = utfutil.execute(popen_args)
508
509  # Get rid of debug output on Android, #1084.
510  out = strip_android_debug_messages(rawout)
[3490]511 
512  # Is this executable suppose to produce any output on standard out?
513  if pragma_dictionary.has_key(OUT_PRAGMA):
514    expected_out = pragma_dictionary[OUT_PRAGMA]
515   
516    if not expected_out and not out: # pragma out
517      report[OUT_PRAGMA] = (None, expected_out)
518    elif not expected_out in out: # pragma out [ARGUMENT]
519      report[OUT_PRAGMA] = (out, expected_out)
520   
521  elif out: # If not, make sure the standard out is empty.
522    report[ERROR_PRAGMA] = (out, None)
523
524
525  # Is this executable suppose to produce any output on standard error?
526  if pragma_dictionary.has_key(ERROR_PRAGMA):
527    expected_error = pragma_dictionary[ERROR_PRAGMA]
528   
529    if not expected_error and not  error: # pragma error
530      report[ERROR_PRAGMA] = (None, expected_error)
531    elif not expected_error in error: # pragma error [ARGUMENT]
532      report[ERROR_PRAGMA] = (error, expected_error)
533     
534  elif error: # If not, make sure the standard error is empty.
535    report[ERROR_PRAGMA] = (error, None)
536
537
538  return report
539
540 
541
542def parse_pragma(source_text):
543  """
544  <Purpose>
545    Parses pragma directives which are contained within a source code file.
546    Possible pragmas inside of a source code include:
547      #pragma out arg
548      #pragma err arg
549      #pragma repy arg
550 
551  <Arguments>
552    source_text: the content of a particular file
553
554  <Exceptions>
555    InvalidPragmaError: When a pragma of an unrecognizable format is used
556      (i.e. none of the above)
557
558  <Side Effects>
559    None
560
561  <Returns>
562    Parsed pragma directives:
563    { Pragma Type : Argument, ... }
564  """
565  directive  = 'pragma'
566  pragma_directives = utfutil.parse_directive(source_text, directive)
567  pragma_dictionary = { }
568
569  for (directive, pragma_type, arg) in pragma_directives:
570    if pragma_type in (OUT_PRAGMA, ERROR_PRAGMA, REPY_PRAGMA):
571      pragma_dictionary[pragma_type] = arg
572    else: 
573      raise InvalidPragmaError(pragma_type)
574
575  return pragma_dictionary
576
577
578
579
580def parse_file_name(file_name):
581  """
582  <Purpose>
583    Parses a file name to identify its module and its descriptor
584 
585  <Arguments>
586    file_name: the name of the test file
587
588  <Exceptions>
589    InvalidTestFileError: if you provide a test file which does not follow the
590      naming convention of 'ut_<module>_<descriptor>.py'
591     
592  <Side Effects>
593    None
594
595  <Returns>
596    A tuple containing (module, descriptor) in the file's naming convention of
597      'ut_<module>_<descriptor>.py'
598   
599  """
600  if not file_name.startswith(SYNTAX_PREFIX):
601    raise InvalidTestFileError(file_name)
602  if not file_name.endswith(SYNTAX_SUFFIX):
603    raise InvalidTestFileError(file_name)
604 
605  # Remove prefix and suffix.
606  stripped = file_name[len(SYNTAX_PREFIX):-len(SYNTAX_SUFFIX)]
607  # Partition the string.
608  (module, separator, descriptor) = stripped.partition('_')
609
610  # Empty module name is not allowed.
611  if not module:
612    raise InvalidTestFileError(file_name)
613
614  return (module, descriptor)
615
616
617
618
619def filter_files(file_list, module = None, descriptor = None):
620  """
621  <Purpose>
622    Given the list of files 'file_list', filter out all invalid test files, that is,
623    test files which do not have a module name of 'module' or a descriptor of 'descriptor'
624    in the form of 'ut_<module>_<descriptor>.py.
625 
626  <Arguments>
627    Module name.
628    Descriptor.
629
630  <Exceptions>
631    InvalidTestFileError--Raised if you provide a test file of incorrect format
632      (all files must follow name convention of ut_<module>_<descriptor>.py).
633
634  <Side Effects>
635    None
636
637  <Returns>
638    A list of all filtered file names:
639    [Filtered File Name list]
640  """
641  result = []
642 
643  for file_name in file_list:
644   
645    try:
646      (file_module, file_descrriptor) = parse_file_name(file_name)
647    except InvalidTestFileError: # This is not a valid test file.
648      continue
649   
650    # Filter based on the module name.
651    if module and file_module != module:
652      continue
653    # Filter based on the descriptor.
654    if descriptor and file_descrriptor != descriptor:
655      continue
656   
657    result.append(file_name) 
658 
659  return result
660
661
662
663
664def print_dashes(): 
665  print '-' * 80
666
667
668
669
[5649]670def strip_android_debug_messages(rawoutput):
671  """
672  Interim fix for #1084: Get rid of stray debugging output on Android
673  of the form "dlopen libpython2.6.so" and "dlopen /system/lib/libc.so",
674  yet preserve all of the other output (including empty lines).
675 
676  Note that almost the same code is used in trunk/repy/safe.py as of r5639.
677  """
678
679  # Are we running on Android?
680  try:
681    import android
682  except ImportError:
683    # If not, then we are done!
684    return rawoutput
685
686  # If we are, then there is work to do (as of Py4A 2.6.2)
687  output = ""
688  for line in rawoutput.split("\n"):
689    # Preserve empty lines
690    if line == "":
691      output += "\n"
692      continue
693    # Suppress debug messages we know can turn up
694    wordlist = line.split()
695    if wordlist[0]=="dlopen":
696      if wordlist[-1]=="/system/lib/libc.so":
697        continue
698      if wordlist[-1].startswith("libpython") and \
699        wordlist[-1].endswith(".so"):
700        # We expect "libpython" + version number + ".so".
701        # The version number should be a string convertible to float.
702        # If it's not, raise an exception.
703        try:
704          versionstring = (wordlist[-1].replace("libpython", 
705            "")).replace(".so", "")
706          junk = float(versionstring)
707        except TypeError, ValueError:
708          raise Exception("Unexpected debug output '" + line + 
709            "' while evaluating code safety!")
710    else:
711      output += line + "\n"
712
713  # Strip off the last newline character we added
714  return output[0:-1]
715
716
717
718
[3490]719if __name__ == "__main__":
720  try:
721    main()
722  except IOError:
723    raise
724  except InvalidTestFileError, e:
725    print 'Invalid file name syntax:', e
726  except:
727    print 'Internal error. Trace:'
728    print_dashes()
729    raise
Note: See TracBrowser for help on using the browser.