Changeset 3314

Show
Ignore:
Timestamp:
12/23/09 11:46:38 (10 years ago)
Author:
yemuru
Message:

Updated tests to work with a newer version of httpretrieve and registerhttpcallback

Location:
seattle/trunk/seattlelib/tests/httpretrieve_test
Files:
7 modified

Legend:

Unmodified
Added
Removed
  • seattle/trunk/seattlelib/tests/httpretrieve_test/ut_httpretrieve_content.py

    r3253 r3314  
    1111 
    1212 
    13 def server_test_content(httprequest_dictionary): 
     13def server_test_content(httprequest_dictionary, http_query, http_post): 
    1414  # build temp server that sends http content normaly  
    1515 
    1616  # return the content to check if the httpretrieve gets the same content  
    17   return mycontext['httpcontent'] 
     17  return [mycontext['httpcontent'], None] 
    1818   
    1919   
     
    3434    raise Exception('Server failed internally ' + str(e))   
    3535 
    36   # use httpretrieve to retrieve the content form the server.   
    37   try: 
    38     recv_msg = httpretrieve_get_string('http://127.0.0.1:12345')   
    39   except Exception, e: 
    40     print 'Http retrieve failed on receiving content, Raised: ' + str(e) 
    41      
    42   # check if the content sent form the server is the same as received by the http retrieve 
    43   else:   
    44     if mycontext['httpcontent'] != recv_msg: 
    45       print 'Failed: http response sent and received doesnt match' 
    46       print 'Server SENT MESSAGE: ' + mycontext['httpcontent']   
    47       print 'Httpretrieve RECIEVED MESSAGE: ' + recv_msg 
     36  else:  
     37    # use httpretrieve to retrieve the content form the server.   
     38    try: 
     39      recv_msg = httpretrieve_get_string('http://127.0.0.1:12345')   
     40    except Exception, e: 
     41      print 'Http retrieve failed on receiving content, Raised: ' + str(e) 
     42       
     43    # check if the content sent form the server is the same as received by the http retrieve 
     44    else:   
     45      if mycontext['httpcontent'] != recv_msg: 
     46        print 'Failed: http response sent and received doesnt match' 
     47        print 'Server SENT MESSAGE: ' + mycontext['httpcontent']   
     48        print 'Httpretrieve RECIEVED MESSAGE: ' + recv_msg 
    4849 
    49   finally: 
    50     # stop the server from waiting for other requests 
    51     stop_registerhttpcallback(handle) 
     50    finally: 
     51      # stop the server from waiting for other requests 
     52      stop_registerhttpcallback(handle) 
    5253 
  • seattle/trunk/seattlelib/tests/httpretrieve_test/ut_httpretrieve_filelikeobj_closed.py

    r3253 r3314  
    1212 
    1313 
    14 def server_test_filelikeobj(httprequest_dictionary): 
     14def server_test_filelikeobj(httprequest_dictionary, http_query, http_post): 
    1515  # for this test the server should just act normal because we are testing if the http retrieve raises an exception 
    1616  # when called after closed.  
    17   return 'normal server'      
     17  return ['normal server', None]      
    1818 
    1919 
  • seattle/trunk/seattlelib/tests/httpretrieve_test/ut_httpretrieve_header_timeout.py

    r3276 r3314  
    1010 
    1111 
    12 def server_test_header_timeout(httprequest_dictionary): 
     12def server_test_header_timeout(httprequest_dictionary, http_query, http_post): 
    1313  # build a server that takes too long to response to the httpretrieve 
    1414 
  • seattle/trunk/seattlelib/tests/httpretrieve_test/ut_httpretrieve_post.py

    r3276 r3314  
    66 
    77 
    8 include httpretrieve.repy 
    9 include registerhttpcallback.repy 
    10 include urllib.repy 
    11  
    12  
    13  
    14 def server_test_content(httprequest_dictionary): 
     8""" 
     9<Program Name> 
     10  registerhttpcallback.repy 
     11 
     12<Started> 
     13  July 29, 2009 
     14 
     15<Author> 
     16  Yafete Yemuru 
     17 
     18<Purpose> 
     19  This program is Web server that a user can use without understanding http protocol. 
     20  Given a URL this program acts like a web server that places a call back callbackfunc when a 
     21  client(web browser) makes a connection to a given URL. The call back function is meant to 
     22  modify a dynamic page using query and posted data. Once the call back return the modified 
     23  version of the website the registerhttpcallback will complete the process by sending the 
     24  web site content to the client(web browser). 
     25 
     26""" 
     27 
     28 
     29#begin include urllib.repy 
     30def urllib_quote(string, safe="/"): 
     31  """ 
     32  <Purpose> 
     33    Encode a string such that it can be used safely in a URL or XML 
     34    document. 
     35 
     36  <Arguments> 
     37    string: 
     38           The string to urlencode. 
     39 
     40    safe (optional): 
     41           Specifies additional characters that should not be quoted -- 
     42           defaults to "/". 
     43 
     44  <Exceptions> 
     45    TypeError if the safe parameter isn't an enumerable. 
     46 
     47  <Side Effects> 
     48    None. 
     49 
     50  <Returns> 
     51    Urlencoded version of the passed string. 
     52  """ 
     53 
     54  resultstr = "" 
     55 
     56  # We go through each character in the string; if it's not in [0-9a-zA-Z] 
     57  # we wrap it. 
     58 
     59  safeset = set(safe) 
     60 
     61  for char in string: 
     62    asciicode = ord(char) 
     63    if (asciicode >= ord("0") and asciicode <= ord("9")) or \ 
     64        (asciicode >= ord("A") and asciicode <= ord("Z")) or \ 
     65        (asciicode >= ord("a") and asciicode <= ord("z")) or \ 
     66        asciicode == ord("_") or asciicode == ord(".") or \ 
     67        asciicode == ord("-") or char in safeset: 
     68      resultstr += char 
     69    else: 
     70      resultstr += "%%%02X" % asciicode 
     71 
     72  return resultstr 
     73 
     74 
     75 
     76 
     77def urllib_quote_plus(string, safe=""): 
     78  """ 
     79  <Purpose> 
     80    Encode a string to go in the query fragment of a URL. 
     81 
     82  <Arguments> 
     83    string: 
     84           The string to urlencode. 
     85 
     86    safe (optional): 
     87           Specifies additional characters that should not be quoted -- 
     88           defaults to the empty string. 
     89 
     90  <Exceptions> 
     91    TypeError if the safe parameter isn't a string. 
     92 
     93  <Side Effects> 
     94    None. 
     95 
     96  <Returns> 
     97    Urlencoded version of the passed string. 
     98  """ 
     99 
     100  return urllib_quote(string, safe + " ").replace(" ", "+") 
     101 
     102 
     103 
     104 
     105def urllib_unquote(string): 
     106  """ 
     107  <Purpose> 
     108    Unquote a urlencoded string. 
     109 
     110  <Arguments> 
     111    string: 
     112           The string to unquote. 
     113 
     114  <Exceptions> 
     115    ValueError thrown if the last wrapped octet isn't a valid wrapped octet 
     116    (i.e. if the string ends in "%" or "%x" rather than "%xx". Also throws 
     117    ValueError if the nibbles aren't valid hex digits. 
     118 
     119  <Side Effects> 
     120    None. 
     121 
     122  <Returns> 
     123    The decoded string. 
     124  """ 
     125 
     126  resultstr = "" 
     127 
     128  # We go through the string from end to beginning, looking for wrapped 
     129  # octets. When one is found we add it (unwrapped) and the following 
     130  # string to the resultant string, and shorten the original string. 
     131 
     132  while True: 
     133    lastpercentlocation = string.rfind("%") 
     134    if lastpercentlocation < 0: 
     135      break 
     136 
     137    wrappedoctetstr = string[lastpercentlocation+1:lastpercentlocation+3] 
     138    if len(wrappedoctetstr) != 2: 
     139      raise ValueError("Quoted string is poorly formed") 
     140 
     141    resultstr = \ 
     142        chr(int(wrappedoctetstr, 16)) + \ 
     143        string[lastpercentlocation+3:] + \ 
     144        resultstr 
     145    string = string[:lastpercentlocation] 
     146 
     147  resultstr = string + resultstr 
     148  return resultstr 
     149 
     150 
     151 
     152 
     153def urllib_unquote_plus(string): 
     154  """ 
     155  <Purpose> 
     156    Unquote the urlencoded query fragment of a URL. 
     157 
     158  <Arguments> 
     159    string: 
     160           The string to unquote. 
     161 
     162  <Exceptions> 
     163    ValueError thrown if the last wrapped octet isn't a valid wrapped octet 
     164    (i.e. if the string ends in "%" or "%x" rather than "%xx". Also throws 
     165    ValueError if the nibbles aren't valid hex digits. 
     166 
     167  <Side Effects> 
     168    None. 
     169 
     170  <Returns> 
     171    The decoded string. 
     172  """ 
     173 
     174  return urllib_unquote(string.replace("+", " ")) 
     175 
     176 
     177 
     178 
     179def urllib_quote_parameters(dictionary): 
     180  """ 
     181  <Purpose> 
     182    Encode a dictionary of (key, value) pairs into an HTTP query string or 
     183    POST body (same form). 
     184 
     185  <Arguments> 
     186    dictionary: 
     187           The dictionary to quote. 
     188 
     189  <Exceptions> 
     190    None. 
     191 
     192  <Side Effects> 
     193    None. 
     194 
     195  <Returns> 
     196    The quoted dictionary. 
     197  """ 
     198 
     199  quoted_keyvals = [] 
     200  for key, val in dictionary.items(): 
     201    quoted_keyvals.append("%s=%s" % (urllib_quote(key), urllib_quote(val))) 
     202 
     203  return "&".join(quoted_keyvals) 
     204 
     205 
     206 
     207 
     208def urllib_unquote_parameters(string): 
     209  """ 
     210  <Purpose> 
     211    Decode a urlencoded query string or POST body. 
     212 
     213  <Arguments> 
     214    string: 
     215           The string to decode. 
     216 
     217  <Exceptions> 
     218    ValueError if the string is poorly formed. 
     219 
     220  <Side Effects> 
     221    None. 
     222 
     223  <Returns> 
     224    A dictionary mapping keys to values. 
     225  """ 
     226 
     227  keyvalpairs = string.split("&") 
     228  res = {} 
     229 
     230  for quotedkeyval in keyvalpairs: 
     231    # Throw ValueError if there is more or less than one '='. 
     232    quotedkey, quotedval = quotedkeyval.split("=") 
     233    key = urllib_unquote(quotedkey) 
     234    val = urllib_unquote(quotedval) 
     235    res[key] = val 
     236 
     237  return res 
     238 
     239#end include urllib.repy 
     240#begin include urlparse.repy   
     241""" 
     242<Program Name> 
     243  urlparse.repy 
     244 
     245<Started> 
     246  May 15, 2009 
     247 
     248<Author> 
     249  Michael Phan-Ba 
     250 
     251<Purpose> 
     252  Provides utilities for parsing URLs, based on the Python 2.6.1 module urlparse. 
     253 
     254""" 
     255 
     256 
     257def urlparse_urlsplit(urlstring, default_scheme="", allow_fragments=True): 
     258  """ 
     259  <Purpose> 
     260    Parse a URL into five components, returning a dictionary.  This corresponds 
     261    to the general structure of a URL: 
     262    scheme://netloc/path;parameters?query#fragment.  The parameters are not 
     263    split from the URL and individual componenets are not separated. 
     264 
     265    Only absolute server-based URIs are currently supported (all URLs will be 
     266    parsed into the components listed, regardless of the scheme). 
     267 
     268  <Arguments> 
     269    default_scheme: 
     270      Optional: defaults to the empty string.  If specified, gives the default 
     271      addressing scheme, to be used only if the URL does not specify one. 
     272 
     273    allow_fragments: 
     274      Optional: defaults to True.  If False, fragment identifiers are not 
     275      allowed, even if the URL's addressing scheme normally does support them. 
     276 
     277  <Exceptions> 
     278    ValueError on parsing a non-numeric port value. 
     279 
     280  <Side Effects> 
     281    None. 
     282 
     283  <Returns> 
     284    A dictionary containing: 
     285 
     286    Key         Value                               Value if not present 
     287    ============================================================================ 
     288    scheme      URL scheme specifier                empty string 
     289    netloc      Network location part               empty string 
     290    path        Hierarchical path                   empty string 
     291    query       Query component                     empty string 
     292    fragment    Fragment identifier                 empty string 
     293    username    User name                           None 
     294    password    Password                            None 
     295    hostname    Host name (lower case)              None 
     296    port        Port number as integer, if present  None 
     297 
     298  """ 
     299 
     300  components = {"scheme": default_scheme, "netloc": "", "path": "", "query": "", 
     301    "fragment": "", "username": None, "password": None, "hostname": None, 
     302    "port": None } 
     303 
     304  # Extract the scheme, if present. 
     305  (lpart, rpart) = _urlparse_splitscheme(urlstring) 
     306  if lpart: 
     307    components["scheme"] = lpart 
     308 
     309  # Extract the server information, if present. 
     310  if rpart.startswith("//"): 
     311    (lpart, rpart) = _urlparse_splitnetloc(rpart, 2) 
     312    components["netloc"] = lpart 
     313 
     314    (components["username"], components["password"], components["hostname"], 
     315      components["port"]) = _urlparse_splitauthority(lpart) 
     316 
     317  # Extract the fragment. 
     318  if allow_fragments: 
     319    (rpart, components["fragment"]) = _urlparse_splitfragment(rpart) 
     320 
     321 
     322  # Extract the query. 
     323  (components["path"], components["query"]) = _urlparse_splitquery(rpart) 
     324 
     325  return components 
     326 
     327 
     328def _urlparse_splitscheme(url): 
     329  """Parse the scheme portion of the URL""" 
     330  # The scheme is valid only if it contains these characters. 
     331  scheme_chars = \ 
     332    "abcdefghijklmnopqrstuvwxyz0123456789+-." 
     333 
     334  scheme = "" 
     335  rest = url 
     336 
     337  spart = url.split(":", 1) 
     338  if len(spart) == 2: 
     339 
     340    # Normalize the scheme. 
     341    spart[0] = spart[0].lower() 
     342 
     343    # A scheme is valid only if it starts with an alpha character. 
     344    if spart[0] and spart[0][0].isalpha(): 
     345      for char in spart[0]: 
     346        if char not in scheme_chars: 
     347          break 
     348      (scheme, rest) = spart 
     349 
     350  return scheme, rest 
     351 
     352 
     353def _urlparse_splitnetloc(url, start=0): 
     354  """Parse the netloc portion of the URL""" 
     355 
     356  # By default, the netloc is delimited by the end of the URL. 
     357  delim = len(url) 
     358 
     359  # Find the left-most delimiter. 
     360  for char in "/?#": 
     361    xdelim = url.find(char, start) 
     362    if xdelim >= 0: 
     363      delim = min(delim, xdelim) 
     364 
     365  # Return the netloc and the rest of the URL. 
     366  return url[start:delim], url[delim:] 
     367 
     368 
     369def _urlparse_splitauthority(netloc): 
     370  """Parse the authority portion of the netloc""" 
     371 
     372  # The authority can have a userinfo portion delimited by "@". 
     373  authority = netloc.split("@", 1) 
     374 
     375  # Default values. 
     376  username = None 
     377  password = None 
     378  hostname = None 
     379  port = None 
     380 
     381  # Is there a userinfo portion? 
     382  if len(authority) == 2: 
     383 
     384    # userinfo can be split into username:password 
     385    userinfo = authority[0].split(":", 1) 
     386 
     387    # hostport can be split into hostname:port 
     388    hostport = authority[1].split(":", 1) 
     389 
     390    if userinfo[0]: 
     391      username = userinfo[0] 
     392    if len(userinfo) == 2: 
     393      password = userinfo[1] 
     394 
     395  # No userinfo portion found. 
     396  else: 
     397 
     398    # hostport can be split into hostname:port 
     399    hostport = netloc.split(":", 1) 
     400 
     401  # Is there a port value? 
     402  if hostport[0]: 
     403    hostname = hostport[0] 
     404  if len(hostport) == 2: 
     405    port = int(hostport[1], 10) 
     406 
     407  # Return the values. 
     408  return username, password, hostname, port 
     409 
     410 
     411def _urlparse_splitquery(url): 
     412  """Parse the query portion of the url""" 
     413 
     414  qpart = url.split("?", 1) 
     415  if len(qpart) == 2: 
     416    query = qpart[1] 
     417  else: 
     418    query = "" 
     419 
     420  return qpart[0], query 
     421 
     422 
     423def _urlparse_splitfragment(url): 
     424  """Parse the query portion of the url""" 
     425 
     426  fpart = url.split("#", 1) 
     427  if len(fpart) == 2: 
     428    fragment = fpart[1] 
     429  else: 
     430    fragment = "" 
     431 
     432  return fpart[0], fragment 
     433 
     434#end include urlparse.repy   
     435#begin include sockettimeout.repy 
     436""" 
     437<Description> 
     438  Puts back in Python's non-blocking functionality. 
     439 
     440  send(): 
     441    Raises SocketTimeout Error if the send call lasts 
     442    longer than the set timeout. 
     443 
     444  recv(): 
     445    Guarentees the receipt of a message.   Raises SocketTimeoutError if it does not 
     446    receive any message before a given timeout. 
     447    If actually receives the message, returns the message and continues. 
     448 
     449<Usage> 
     450  Text-replacable for Repy Sockets: 
     451    timeout_openconn(desthost, destport, localip=None, localport=None, timeout = 5) 
     452    timeout_waitforconn(localip, localport, function) 
     453 
     454  Object: 
     455    sockobj.settimeout(seconds) 
     456    sockobj.send(data) 
     457    sockobj.recv(bytes) 
     458    sockobj.close() 
     459 
     460<Date> 
     461  Sun Mar  1 10:27:35 PST 2009 
     462 
     463<Example> 
     464  # hello world 
     465  include sockettimer.repy 
     466 
     467  def callback(ip, port, timeout_sockobj, commhandle, listenhandle): 
     468    hw_message = timeout_sockobj.recv(1047) 
     469 
     470    # cleanup 
     471    stopcomm(commhandle) 
     472    stopcomm(listenhandle) 
     473    timeout_sockobj.close() 
     474 
     475    print hw_message # => "hello world!" 
     476   
     477  def server(): 
     478    sockobj = timeout_waitforconn(getmyip(), 12345, callback) 
     479 
     480  def client(): 
     481    sockobj = timeout_openconn(getmyip(), 12345) 
     482    sockobj.send("hello world!") 
     483 
     484  def main(): 
     485    server() 
     486    client() 
     487    exitall() 
     488 
     489  if callfunc == 'initialize': 
     490    main()  
     491""" 
     492 
     493class SocketTimeoutError(Exception): 
     494  """The socket timed out before receiving a response""" 
     495 
     496def timeout_openconn(desthost, destport, localip=None, localport=None, timeout = 5): 
     497  """ 
     498  <Purpose>  
     499    Wrapper for Repy like socket interface 
     500 
     501  <Args> 
     502    Same as Repy openconn 
     503 
     504  <Exception> 
     505    Timeout exception if the dest address doesnt respond. 
     506 
     507  <Returns> 
     508    socket obj on success 
     509  """ 
     510 
     511  tsock = TimeoutSocket() 
     512  tsock.settimeout(timeout) 
     513  if localip and localport: 
     514    tsock.bind((localip, localport)) 
     515  tsock.connect((desthost, destport)) 
     516  return tsock 
     517 
     518def timeout_waitforconn(localip, localport, function): 
     519  """ 
     520  <Purpose>  
     521    Wrapper for Repy like socket interface 
     522 
     523  <Args> 
     524    Same as Repy waitforconn 
     525 
     526  <Side Effects> 
     527    Sets up event listener which calls function on messages. 
     528 
     529  <Returns> 
     530    Handle to listener. 
     531  """ 
     532 
     533  tsock = TimeoutSocket() 
     534  tsock.bind((localip, localport)) 
     535  tsock.setcallback(function) 
     536  return tsock.listen() 
     537 
     538class TimeoutSocket: 
     539  """ 
     540  <Purpose> 
     541    Provide an socket object like the Repy usual one. 
     542 
     543  <Side Effects> 
     544    Uses a getlock() to watch for a timeout 
     545    Uses waitforconn and openconn to simulate socket 
     546  """ 
     547 
     548  ################ 
     549  # Constructors 
     550  ################ 
     551 
     552  def __init__(self): 
     553    """ Constructor for socket """ 
     554#    self.lock = getlock() # general lock BUG: Do we need to lock everything? 
     555    self.timeout_lock = getlock() # special lock for Timeout condition 
     556    self.timeout = 5 # seconds to wait 
     557    self.bytes_sent = None # used to check if send() timed out 
     558 
     559    # user vars    
     560    self.local_address = None # ip, port 
     561    self.remote_address = None # ip, port 
     562    self.callback = None # the user's function to call 
     563 
     564    # repy socket vars 
     565    self.sockobj = None #  the Repy socket 
     566    self.commhandle = None # the current comm 
     567    self.listencommhandle = None # the listener comm 
     568 
     569    #error tracking vars 
     570     
     571    #if any exceptions are thrown in the separate thread executing _send_and_release,  
     572    #they are caught and stored in this variable, then raised in _send_or_close 
     573    self.TCPSendError = None 
     574 
     575  ################ 
     576  # Mutator methods 
     577  ################# 
     578 
     579  def settimeout(self, value): 
     580    """ Setter for timeout""" 
     581    self.timeout = value 
     582 
     583  def setcallback(self, function): 
     584    """ Setter for callback function""" 
     585    self.callback = function 
     586 
     587  #################### 
     588  # Public Methods 
     589  #################### 
     590 
     591  def bind(self, local_address = None): 
     592    """ 
     593    <Purpose> 
     594      Set local address 
     595 
     596    <Args> 
     597      Tuple of (ip, port) local. 
     598    """ 
     599    self.local_address = local_address 
     600 
     601  def listen(self): 
     602    """ 
     603    <Purpose> 
     604      Listen for peer 
     605     
     606    <Side Effects> 
     607      Calls Repy waitforconn() 
     608    """ 
     609    return self._waitforconn() 
     610 
     611  def connect(self, remote_address): 
     612    """ 
     613    <Purpose> 
     614      Connect to peer. 
     615 
     616    <Args> 
     617      Tuple of (ip, port) remote. 
     618    
     619    <Side Effects> 
     620      Calls Repy openconn. 
     621    """ 
     622    self.remote_address = remote_address 
     623    self._openconn() 
     624 
     625  def recv(self, maxLen): # timeout as optional arg ??? 
     626    """ 
     627    <Purpose> 
     628      If it fails to finish within the timeout, I close the socket and raise a 
     629      TimeoutError exception. I.e. if there's no message, we call it an error 
     630      and raise it. 
     631       
     632    <Arguments> 
     633      maxLen - bytes to recv 
     634 
     635    <Exception> 
     636      Raises TimeoutError exception if the recv times out 
     637      without receiving a message. 
     638 
     639    <Side Effects> 
     640      Closes the connection if times out. 
     641 
     642    <Returns> 
     643      The message. 
     644    """ 
     645    return self._recv_or_close(maxLen) 
     646 
     647  def send(self, data): 
     648    """ 
     649    <Purpose> 
     650      Just like normal Repy socket.  Sends messages. 
     651       
     652    <Arguments> 
     653      data - the string message 
     654 
     655    <Exception> 
     656      Same as Repy socket. 
     657  
     658    <Returns> 
     659      The bytes sent. 
     660    """ 
     661    return self._send_or_close(data) 
     662 
     663  def close(self): 
     664    self.local_address = None # ip, port 
     665    self.remote_address = None # ip, port 
     666    self.callback = None # the user's function to call 
     667 
     668    if self.sockobj: 
     669      self.sockobj.close() 
     670    self.sockobj = None #  the Repy socket 
     671     
     672    # Armon: As part of the semantics, stopcomm will raise an  
     673    # exception given an invalid handle, e.g. None. Thus, 
     674    # we need to check for this. 
     675    if self.commhandle:  
     676      stopcomm(self.commhandle) 
     677      self.commhandle = None # the current comm 
     678     
     679    # Armon: Same as above. 
     680    if self.listencommhandle: 
     681      stopcomm(self.listencommhandle) 
     682      self.listencommhandle = None # the listener comm 
     683 
     684 
     685  ######################## 
     686  # Private 
     687  ######################### 
     688 
     689  def _openconn(self): 
     690    """Handle current state variables and call Repy openconn.""" 
     691 
     692    destip, destport = self.remote_address 
     693    if self.local_address: 
     694      srcip, srcport = self.local_address 
     695      self.sockobj = openconn(destip, destport, srcip, srcport, self.timeout) 
     696    else: 
     697      self.sockobj = openconn(destip, destport) 
     698 
     699  def _waitforconn(self): 
     700    """Setup way between Repy waitforconn event""" 
     701    localip, localport = self.local_address 
     702    self.listencommhandle = waitforconn(localip, localport, self._callback) 
     703    return self.listencommhandle 
     704 
     705  def _callback(self, ip, port, sockobj, ch, lh): 
     706    """Pass on through to user callback""" 
     707    self.sockobj = sockobj 
     708    self.listencommhandle = lh # same as the 1st from wait for comm, right? 
     709    self.commhandle = ch # should we care? 
     710     
     711    if not self.remote_address: 
     712      self.remote_address = (ip, port) 
     713    else:  
     714      raise Exception("what! peer does not match?") 
     715 
     716    self.callback(ip, port, self, ch, lh) 
     717 
     718  def _send(self, data): 
     719    """Send data""" 
     720    return self.sockobj.send(data) 
     721 
     722  def _recv(self, maxLen): 
     723    """Recv data of length maxLen""" 
     724    return self.sockobj.recv(maxLen) 
     725 
     726  def _send_and_release(self, data): 
     727    """Send data then release the timeout lock""" 
     728     
     729    #Bug Fix (Cosmin): exceptions thrown in separate thread _send_and_release could not be caught 
     730    #now we store the exception if it is thrown and raise it back in send_or_close 
     731    try: 
     732      self.bytes_sent = self._send(data) 
     733    except Exception, e: 
     734      self.TCPSendError = e 
     735     
     736    self._quietly_release() # release the lock 
     737  
     738  def _quietly_release(self): 
     739    """Release the timeout lock and ignore if already released""" 
     740    try: 
     741      self.timeout_lock.release() 
     742    except: 
     743      pass 
     744    
     745  def _send_or_close(self, data): 
     746    """Raise the Timeout Error if no receipt.  Keep track by timeout_lock.""" 
     747 
     748    # acquire the lock, when it's release we'll carry on 
     749    self.timeout_lock.acquire() 
     750 
     751    # fork off a lock that'll release the lock at the timeout 
     752    timerhandle = settimer(self.timeout, self._quietly_release, ()) 
     753 
     754    # fork off a send call so we can raise the exception in the main thread 
     755    # the send call will also release our lock 
     756    settimer(0, self._send_and_release, (data,)) 
     757 
     758    # block until either the timeout or _send finishes 
     759    self.timeout_lock.acquire() 
     760    self.timeout_lock.release() 
     761 
     762    if self.bytes_sent: # send finished 
     763      canceltimer(timerhandle) 
     764      retdata = self.bytes_sent 
     765      self.bytes_sent = None 
     766      return retdata 
     767    elif self.TCPSendError != None: 
     768      #Bug Fix (Cosmin): exceptions thrown in separate thread _send_and_release could not be caught 
     769       
     770      #we got an error within the separate thread that performed the send operation 
     771      exception_to_throw = self.TCPSendError 
     772      self.TCPSendError = None 
     773      raise exception_to_throw 
     774    else: # it timed out 
     775      self.close() 
     776      raise SocketTimeoutError 
     777 
     778  def _recv_or_close(self, amount): 
     779    """Raise the Timeout Error if no receipt.  Keep track by timeout_lock.""" 
     780    timerhandle = settimer(self.timeout, self._clobbersocket, ()) 
     781    try: 
     782      retdata = self._recv(amount) 
     783    except Exception, e: 
     784      # if it's not the timeout, reraise... 
     785      if self.timeout_lock.acquire(False): 
     786        raise 
     787      raise SocketTimeoutError 
     788     
     789    # I acquired the lock, I should stop the timer because I succeeded... 
     790    if self.timeout_lock.acquire(False): 
     791      # even if this isn't in time, the lock prevents a race condition  
     792      # this is merely an optimization to prevent the timer from ever firing... 
     793      canceltimer(timerhandle) 
     794      self.timeout_lock.release() # Alper's bug 3/10/09 
     795      return retdata 
     796    else: 
     797      raise SocketTimeoutError 
     798 
     799  def _clobbersocket(self): 
     800    """If I can acquire the lock without blocking, then close the socket to abort""" 
     801    if self.timeout_lock.acquire(False): 
     802      self.close() 
     803 
     804 
     805############################ 
     806# Deprecated functions 
     807############################## 
     808 
     809# private function... 
     810def sockettimeout_clobbersocket(sockobj,mylock): 
     811  # if I can acquire the lock without blocking, then close the socket to abort 
     812  if mylock.acquire(False): 
     813    sockobj.close() 
     814 
     815# if it fails to finish within the timeout, I close the socket and raise a 
     816# SocketTimeout exception... 
     817def sockettimeout_recv_or_close(sockobj, amount, timeout): 
     818  # A lock I'll use for this attempt 
     819  mylock = getlock() 
     820  timerhandle = settimer(timeout,clobbersocket, (sockobj, mylock)) 
     821  try: 
     822    retdata = sockobj.recv(amount) 
     823  except Exception, e: 
     824    # if it's not the timeout, reraise... 
     825    if mylock.acquire(False): 
     826      raise 
     827    raise SocketTimeout 
     828     
     829  # I acquired the lock, I should stop the timer because I succeeded... 
     830  if mylock.acquire(False): 
     831    # even if this isn't in time, the lock prevents a race condition  
     832    # this is merely an optimization to prevent the timer from ever firing... 
     833    canceltimer(timerhandle) 
     834    return retdata 
     835  else: 
     836    raise SocketTimeout 
     837 
     838 
     839#end include sockettimeout.repy 
     840# used for hierarchy exception  
     841#begin include http_hierarchy_error.repy 
     842""" 
     843<Program Name> 
     844  http_hierarchy_error.repy 
     845 
     846<Started> 
     847  Oct 05, 2009 
     848 
     849<Author> 
     850  Yafete Yemuru 
     851 
     852<Purpose> 
     853  provides a hierachy http error using status code including client and server errors 
     854  classes.   
     855""" 
     856 
     857 
     858''' 
     859http hierarchy error exception classes 
     860 
     861-> HttpError 
     862   -> HttpRetrieveClientError 
     863      -> HttpUserInputError 
     864      -> HttpConnectionError 
     865 
     866   -> HttpServerError 
     867      -> HttpResponseError 
     868         -> HttpHeaderError 
     869            -> HttpHeaderReceivingError 
     870            -> HttpHeaderFormatError 
     871         -> HttpContentError 
     872            -> HttpContentReceivingError 
     873            -> HttpContentLengthError 
     874        
     875   -> HttpStatuscodeError 
     876     -> HttpError1xx 
     877        -> followed by all http status code error number HttpError(number) 
     878         
     879     -> HttpError2xx 
     880        -> followed by all http status code error number HttpError(number) 
     881         
     882     -> HttpError3xx 
     883        -> followed by all http status code error number HttpError(number) 
     884 
     885     -> HttpError4xx 
     886        -> followed by all http status code error number HttpError(number) 
     887         
     888     -> HttpError5xx 
     889        -> followed by all http status code error number HttpError(number) 
     890 
     891''' 
     892 
     893class HttpError(Exception): 
     894  pass 
     895 
     896# raises an exception for http client error  
     897class HttpRetrieveClientError(HttpError):# extend HttpError  
     898  pass 
     899class HttpUserInputError(HttpRetrieveClientError): 
     900  pass 
     901class HttpConnectionError(HttpRetrieveClientError): 
     902  pass 
     903 
     904 
     905# raises an exception for any http server failure   
     906class HttpServerError(HttpError):# extend HttpError  
     907  pass 
     908class HttpResponseError(HttpServerError): 
     909  pass 
     910class HttpHeaderError(HttpResponseError): 
     911  pass 
     912class HttpHeaderReceivingError(HttpHeaderError): 
     913  pass 
     914class HttpHeaderFormatError(HttpHeaderError): 
     915  pass 
     916class HttpContentError(HttpResponseError): 
     917  pass 
     918class HttpContentReceivingError(HttpContentError): 
     919  pass 
     920class HttpContentLengthError(HttpContentError): 
     921  pass 
     922 
     923 
     924class HttpStatusCodeError(HttpError):# extend HttpError 
     925  pass 
     926class HttpError1xx(HttpStatusCodeError): 
     927  pass 
     928class HttpError100(HttpError1xx): 
     929  pass 
     930class HttpError101(HttpError1xx):  
     931  pass 
     932class HttpError102(HttpError1xx):  
     933  pass 
     934 
     935 
     936class HttpError2xx(HttpStatusCodeError): 
     937  pass 
     938class HttpError201(HttpError2xx):  
     939  pass 
     940class HttpError202(HttpError2xx):    
     941  pass 
     942class HttpError203(HttpError2xx):  
     943  pass 
     944class HttpError204(HttpError2xx):  
     945  pass 
     946class HttpError205(HttpError2xx):  
     947  pass 
     948class HttpError206(HttpError2xx):  
     949  pass 
     950class HttpError207(HttpError2xx):  
     951  pass 
     952class HttpError226(HttpError2xx):  
     953  pass                  
     954 
     955 
     956class HttpError3xx(HttpStatusCodeError): 
     957  pass 
     958class HttpError300(HttpError3xx):  
     959  pass 
     960class HttpError301(HttpError3xx):  
     961  pass 
     962class HttpError302(HttpError3xx):  
     963  pass 
     964class HttpError303(HttpError3xx):  
     965  pass 
     966class HttpError304(HttpError3xx): 
     967  pass 
     968class HttpError305(HttpError3xx):  
     969  pass 
     970class HttpError306(HttpError3xx):  
     971  pass 
     972class HttpError307(HttpError3xx):  
     973  pass 
     974                     
     975 
     976class HttpError4xx(HttpStatusCodeError): 
     977  pass 
     978class HttpError400(HttpError4xx): 
     979  pass   
     980class HttpError401(HttpError4xx):  
     981  pass 
     982class HttpError402(HttpError4xx):  
     983  pass 
     984class HttpError403(HttpError4xx): 
     985  pass   
     986class HttpError404(HttpError4xx):  
     987  pass 
     988class HttpError405(HttpError4xx):  
     989  pass 
     990class HttpError406(HttpError4xx):  
     991  pass 
     992class HttpError407(HttpError4xx):  
     993  pass 
     994class HttpError408(HttpError4xx):  
     995  pass 
     996class HttpError409(HttpError4xx):  
     997  pass 
     998class HttpError410(HttpError4xx):  
     999  pass 
     1000class HttpError411(HttpError4xx):  
     1001  pass 
     1002class HttpError412(HttpError4xx):  
     1003  pass 
     1004class HttpError413(HttpError4xx):  
     1005  pass 
     1006class HttpError414(HttpError4xx):  
     1007  pass 
     1008class HttpError415(HttpError4xx):  
     1009  pass 
     1010class HttpError416(HttpError4xx):  
     1011  pass 
     1012class HttpError417(HttpError4xx):  
     1013  pass 
     1014class HttpError418(HttpError4xx):  
     1015  pass 
     1016class HttpError422(HttpError4xx):  
     1017  pass 
     1018class HttpError423(HttpError4xx):  
     1019  pass 
     1020class HttpError424(HttpError4xx):  
     1021  pass 
     1022class HttpError425(HttpError4xx):  
     1023  pass 
     1024class HttpError426(HttpError4xx):  
     1025  pass 
     1026 
     1027 
     1028class HttpError5xx(HttpStatusCodeError): 
     1029  pass 
     1030class HttpError500(HttpError5xx):  
     1031  pass 
     1032class HttpError501(HttpError5xx):  
     1033  pass 
     1034class HttpError502(HttpError5xx):  
     1035  pass 
     1036class HttpError503(HttpError5xx):  
     1037  pass 
     1038class HttpError504(HttpError5xx):  
     1039  pass 
     1040class HttpError505(HttpError5xx):  
     1041  pass 
     1042class HttpError506(HttpError5xx): 
     1043  pass   
     1044class HttpError507(HttpError5xx): 
     1045  pass   
     1046class HttpError510(HttpError5xx): 
     1047  pass 
     1048 
     1049#end include http_hierarchy_error.repy 
     1050 
     1051 
     1052 
     1053 
     1054 
     1055def registerhttpcallback(url, callbackfunc, httprequest_limit = 131072, httppost_limit = 2048): 
     1056  """ 
     1057  <Purpose> 
     1058     Waits for a connection to the given url(host). Calls callbackfunc with a argument of http request 
     1059     dictionary up on success.  
     1060   
     1061      
     1062  <Arguments> 
     1063     url: 
     1064           String of a http web server-URL  
     1065 
     1066     callbackfunc: 
     1067            The callbackfunc to be called. It can take up to three argument . 
     1068            First argument- The http request dictionary keys are as follows: 
     1069              http_command - for the http request method GET or POST 
     1070              http_version - which http version the client  used HTTP\1.1 or HTTP\1.0   
     1071              path - the exact request path parsed  
     1072              query - the exact request query parsed  
     1073              posted_data - if there is any posted data returns None if there isnt one 
     1074 
     1075              And all the http requests headers the client provided. eg. Content-Length: 22   
     1076              will incude Content-Length as a key and 22 as a value 
     1077 
     1078            Second argument - httprequest query; this returns a unencoded dictionary of the query 
     1079            Third argument - httprequest posted data; this returns a unencoded dictionary of the posted data 
     1080 
     1081            RESTRICTIONS: 
     1082              -> Follow the http_hierarchy_error(HttpStatusCodeError)to raise an exception. eg. raise 
     1083                  HttpError404 for file not found 
     1084              -> To redirect raise HttpError301(url) or HttpError302(url). The url has to be valid.   
     1085              -> server only excepts tuple type of [httpcontent, httpheader] where the httpcontent is a 
     1086                  string of the content intended to display. httpheader is a dictionary used only if you have a 
     1087                  extra header you want to add on the http response.  
     1088 
     1089     httprequest_limit: 
     1090            -> used to to limit the http request a client can make(default value set to 128kb or 131072 charactors) 
     1091 
     1092     httppost_limit: 
     1093            -> used to to limit the http post a client can make(default value set to 2mb or 2048 charactors)      
     1094 
     1095 
     1096  <Exceptions> 
     1097          HttpConnectionError-> if the server fails on waiting for a connection 
     1098          HttpUserInputError -> if server fails on making a connection to client(web browser) 
     1099          HttpServerError -> If server fails internally 
     1100 
     1101          HttpError408 -> If client(web browser) takes to long to send request   
     1102          HttpError413 -> If the http posted data length is too long 
     1103                       -> If the http request length is too long 
     1104          HttpError417 -> If there is the http request format problem 
     1105                       -> If the content length doesnt match the actual posted data length 
     1106          HttpError501 -> If the given http command is not supported or valid (supported 'GET' and 'POST') 
     1107          HttpError411 -> If the http command is POST and content length is not given 
     1108          HttpError500 -> If server fails internally on sending the http content OR HTTP header,sending error and 
     1109                       -> If the callback fucntion doesnt follow restrictions   
     1110                       -> if the users input for url is not in correct format or unsuported 
     1111                          protocol(this program only supports Http) 
     1112 
     1113 
     1114  <Side Effects> 
     1115     None  
     1116 
     1117  <Returns> 
     1118     A handle to the listener. This can be used to stop the server from waiting for a connection.   
     1119  """   
     1120  def run_webserver(ip, port, sock, thiscommhandle, listencommhandle): 
     1121  # this function is defined inside the registerhttpcallback fuction so callback function name that is 
     1122  # given by a user is placed a call on when there is a connection.  
     1123    try: 
     1124      # receive the client(web browser) request and return a list of the request 
     1125      client_request_lines = _registerhttpcallback_receive_client_request(sock, httprequest_limit) 
     1126 
     1127      # check if the received request meets http requsest format standards and return a 
     1128      # dictionary of the http request headers.  
     1129      httprequest_dictionary = _registerhttpcallback_make_httprequsest_dictionary(client_request_lines)     
     1130 
     1131      # check if there is posted data and add to the httprequest_dictionary with a key posted_data. 
     1132      # returns None if there isnt a posted data 
     1133      _registerhttpcallback_receive_httpposted_data(sock, httprequest_dictionary, httppost_limit) 
     1134       
     1135      # get dictionary decode from the given query  
     1136      httprequest_query = _registerhttpcallback_get_decode_dict(httprequest_dictionary, 'query') 
     1137 
     1138      # get dictionary decode from the given post  
     1139      httprequest_posted_data = _registerhttpcallback_get_decode_dict(httprequest_dictionary, 'posted_data') 
     1140 
     1141      # place the call back callbackfunc with dictionary that includes http_command, 
     1142      # http_version, path, query, posted_data, and all the http requests headers 
     1143      webpage_content = callbackfunc(httprequest_dictionary, httprequest_query, httprequest_posted_data) 
     1144         
     1145      # callback callbackfunc excecuted, send the processed dynamic web page data to client(web browser) 
     1146      _registerhttpcallback_send_httpresponse(sock, webpage_content) 
     1147 
     1148       
     1149    except HttpStatusCodeError, e: 
     1150      # send any error that occur during processing server to the client(web browser) 
     1151      _registerhttpcallback_send_httpformat_error(sock, e) 
     1152       
     1153       
     1154    except Exception, e: 
     1155      # if the program failed to catch an internal error raise an exception and send it to client(web browser) 
     1156      try: 
     1157        raise HttpError500('Server failed internally: ' + str(e)) 
     1158      except Exception, e: 
     1159        _registerhttpcallback_send_httpformat_error(sock, e) 
     1160 
     1161 
     1162 
     1163   
     1164  # get the host and port from the given url to use for connection to listen on 
     1165  (host, port) = _registerhttpcallback_get_host_port(url) 
     1166 
     1167  try: 
     1168    # waits for a client(web browser) to make a connetion and run the web server  
     1169    listencommhandle = waitforconn(host, port, run_webserver) 
     1170 
     1171  except Exception, e: 
     1172    # the waiting for a connection failed, stop waiting and raise an exception  
     1173    stopcomm(listencommhandle) 
     1174    raise HttpConnectionError('Web server failed on waiting for connection ' + str(e)) 
     1175 
     1176  else: 
     1177    # store the listencommhandle, to stop server if needed 
     1178    return listencommhandle 
     1179 
     1180 
     1181 
     1182 
     1183 
     1184 
     1185 
     1186 
     1187def stop_registerhttpcallback(handle):    
     1188  """ 
     1189    <Purpose> 
     1190          Deregister a callback for a commhandle.  
     1191 
     1192    <Arguments> 
     1193          commhandle: 
     1194              A commhandle as returned by registerhttpcallback. 
     1195 
     1196    <Exceptions> 
     1197          None. 
     1198 
     1199    <Side Effects> 
     1200          This has an undefined effect on a socket-like object if it is currently in use. 
     1201 
     1202    <Returns> 
     1203          Returns True if commhandle was successfully closed, False if the handle cannot be closed  
     1204  """ 
     1205  #close the connection 
     1206  return stopcomm(handle) 
     1207 
     1208    
     1209 
     1210 
     1211   
     1212 
     1213def _registerhttpcallback_get_host_port(url): 
     1214  # get host and path from the given url 
     1215  try: 
     1216    # returns a dictionary of {scheme, netloc, path, quer, fragment, username, password, hostname and port} form the url 
     1217    urlparse = urlparse_urlsplit(url)   
     1218  except Exception, e: 
     1219    raise HttpUserInputError('Server URL format error:' + str(e)) 
     1220  else: 
     1221    # check if the given url is valid using the url parse  
     1222    if urlparse['scheme'] != 'http': 
     1223      raise HttpUserInputError('The given protocol type isnt suported: Given ' + urlparse['scheme'])        
     1224    if urlparse['hostname'] == None: 
     1225      raise HttpUserInputError('Server URL format error: host name is not given')  
     1226 
     1227    host = urlparse['hostname'] 
     1228     
     1229    # use default port 80 if the port isnt given 
     1230    if urlparse['port'] == None: 
     1231      port = 80 
     1232    else: 
     1233      # if given use the given port 
     1234      port = urlparse['port'] 
     1235 
     1236    return host, port 
     1237 
     1238 
     1239 
     1240       
     1241def _registerhttpcallback_receive_client_request(sock, httprequest_limit): 
     1242  # receive request from the client using the socket connection 
     1243  if not type(httprequest_limit) == int: 
     1244    # check if the given httprequest limit is valid 
     1245    raise HttpServerError('The given http request limit isnt int ' + str(e)) 
     1246   
     1247   
     1248  # empty line is used as a signal for when the http request is done, and we set a 
     1249  # default request length limit to be 131072 character(128kb) 
     1250  client_request = _registerhttpcallback_receive_untilemptyline(sock, httprequest_limit) 
     1251         
     1252   
     1253  # http request format requires the request to be line by line 
     1254  # build a list of the received request split line by line to check the formating of the request 
     1255  client_request_lines = '' 
     1256  try:  
     1257    # split the entire message line by line  
     1258    client_request_lines = client_request.splitlines() 
     1259  except Exception, e: 
     1260    # raise an exception if the request doenst follow the http protocol format  
     1261    raise HttpError417('Error on the http request format ' + str(e)) 
     1262   
     1263  # the http request has to be at least one line including the http header request 
     1264  if len(client_request_lines) == 0: 
     1265    raise HttpError417('The received request doesnt follow http protocol requirments: Given ' + client_request) 
     1266 
     1267  # returns a list of client request 
     1268  return client_request_lines 
     1269 
     1270 
     1271 
     1272 
     1273 
     1274def _registerhttpcallback_make_httprequsest_dictionary(client_request_lines): 
     1275  # builds up a dictionary from the received request or raises an exception if the 
     1276  # request format isnt http protocol. The dictionary also includes the http header request 
     1277  # parsed with custome keys (http_command - for http comand methed, path - parsed form 
     1278  # the request url, query - query string parsed form the request url, http_version - 
     1279  # for the http protocol version) 
     1280   
     1281  httpheader_request = True 
     1282  httprequest_dictionary = {} 
     1283   
     1284  for request_line in client_request_lines:   
     1285    if httpheader_request: 
     1286      # acording to the http protocol format the first line is different because it is includes comand url and version 
     1287 
     1288      # check if the http request is valid and parse the http request method, URL and http version 
     1289      (http_command, path, query, http_version) = _registerhttpcallback_httprequest_header_parse(request_line) 
     1290 
     1291      # use custom names for the parsed data of the first request line and add to the dictionary  
     1292      httprequest_dictionary['http_command'] = http_command 
     1293      httprequest_dictionary['http_version'] = http_version 
     1294      httprequest_dictionary['path'] = path 
     1295      httprequest_dictionary['query'] = query   
     1296 
     1297      # used to flag the for the upcoming lines, because they have stationary keys 
     1298      httpheader_request = False  
     1299       
     1300    elif request_line == '': 
     1301      # last line is empty line, return the http dictionary built so far 
     1302      return httprequest_dictionary   
     1303 
     1304    else: 
     1305      # the rest of the lines should be formated 'httpheader_key: httpheader_value'  eg.'Content-Length: 34' 
     1306      try: 
     1307        modified_request = request_line.split(': ') 
     1308      except Exception, e: 
     1309        raise HttpError417('Error on the http request format; Given: ' + request_line + 'in the request line ' + str(e)) 
     1310 
     1311      # raise an exception if the request line doesnt contain 2 contents for 'httpheader_key: httpheader_value'  
     1312      if len(modified_request) != 2: 
     1313        raise HttpError417('Error on the http request format; Given: ' + request_line + 'in the request line') 
     1314 
     1315      httprequest_key = modified_request[0] 
     1316      httprequest_value = modified_request[1] 
     1317       
     1318      httprequest_dictionary[httprequest_key] = httprequest_value 
     1319 
     1320  return httprequest_dictionary 
     1321   
     1322 
     1323 
     1324 
     1325 
     1326def _registerhttpcallback_get_decode_dict(httprequest_dictionary , decode_type): 
     1327  # returns a decode dictionary of post or query depending up on the encoded style    
     1328 
     1329  # get the data from the request dictionary  
     1330  data_to_decode = httprequest_dictionary[decode_type] 
     1331       
     1332  if decode_type == 'posted_data': 
     1333    # inorder to check if the post is empty use None    
     1334    empty_check = None     
     1335  else: 
     1336    # inorder to check if the query is empty use empty string    
     1337     empty_check = '' 
     1338         
     1339  if data_to_decode != empty_check: 
     1340    try: 
     1341      # decode the given data depending up on the style it was encoded   
     1342      return urllib_unquote_parameters(data_to_decode) 
     1343 
     1344    except Exception, e: 
     1345      # raise an exception if the fails decoding the given data   
     1346      raise HttpUserInputError('Invalid ' + decode_type + ' Raised ' + str(e) + ' on decoding') 
     1347 
     1348  # if the data is empty return None  
     1349  return None     
     1350 
     1351 
     1352 
     1353 
     1354   
     1355def _registerhttpcallback_httprequest_header_parse(http_header_request): 
     1356  # varifiy's if the http request header format is valid and returns the http command, path, query, 
     1357  # and http version parsed from the http header 
     1358 
     1359  # http request header should include RequestMethod <url> HTTP<version> or RequestMethod HTTP<version> 
     1360  # and is located at the top of the http request 
     1361  try: 
     1362    http_command_url_version = http_header_request.split() 
     1363  except Exception, e: 
     1364    raise HttpError417('Http header request needs spacing in between: Given: ' + http_header_request + str(e)) 
     1365 
     1366  # Check that the first line at least contains 3  words: RequestMethod <url> HTTP<version> 
     1367  if len(http_command_url_version) >= 3:  
     1368    url = http_command_url_version[1] 
     1369    http_version = http_command_url_version[2] 
     1370   
     1371  else: 
     1372    # http request header cant have any more data than RequestMethod <url> HTTP<version> or RequestMethod HTTP<version> 
     1373    raise HttpError417('The request header should contain  RequestMethod <url> HTTP<version>, Given: ' + http_header_request) 
     1374 
     1375  # check the http comand is valid or if its even suported(suported comands include GET and POST) 
     1376  if http_command_url_version[0] == 'GET' or http_command_url_version[0].lower() == 'post': 
     1377     http_command = http_command_url_version[0] 
     1378  else: 
     1379    raise HttpError501('The given http comand is not suported or valid. Given: ' + str(http_command_url_version[0])) 
     1380   
     1381 
     1382  # check the if the http version is valid 
     1383  if not http_version.startswith('HTTP'): 
     1384    raise HttpError417('Http header request version should start of with HTTP then <version>, Given: ' +  httpversion + ' as a http version')  
     1385 
     1386  # (query used to modify the dynamic page) http header includes the path and query, pasrse the given url to path and query  
     1387  (path, query) = _registerhttpcallback_parse_httpheader_url(url)   
     1388 
     1389  return http_command, path, query, http_version  
     1390 
     1391 
     1392 
     1393 
     1394def _registerhttpcallback_parse_httpheader_url(url): 
     1395  # parse out the query and path from the url 
     1396  path = '' 
     1397  query = '' 
     1398 
     1399  # if url isnt given return empty strings   
     1400  if url != '': 
     1401    # if url is given parse the query and path using url parse 
     1402    try: 
     1403     # returns a dictionary of {scheme, netloc, path, query, fragment, username, 
     1404     # password, hostname and port} parsing the url                       
     1405      urlparse = urlparse_urlsplit(url)   
     1406    except Exception, e: 
     1407      raise HttpError417('Http request given url doesnt meet the http protocol standards ' + str(e) + 'Given url: ' + url) 
     1408 
     1409    # retrieve only the path and query  
     1410    try: 
     1411      path = urlparse['path'] 
     1412      query = urlparse['query'] 
     1413    except Exception, e: 
     1414      raise HttpError417('Http request given url doesnt meet the http protocol standards ' + str(e) + 'Given url: ' + url) 
     1415         
     1416    # if there is a url there has to be a path so raise an exception because the given url format is wrong  
     1417    if path == '': 
     1418      raise HttpError417('Error on parsing the http request header url: Given ' + url) 
     1419     
     1420  return path, query 
     1421 
     1422 
     1423 
     1424 
     1425def _registerhttpcallback_receive_httpposted_data(sock, httprequest_dictionary, httppoost_limit): 
     1426  # receive the posted data which sent right after the http request with a empty line 
     1427  # indicating the end of the posted data(this is if the http comand is only a POST) 
     1428 
     1429  if not type(httppoost_limit) == int: 
     1430    # check if the given http post limit is valid 
     1431    raise HttpServerError('The given http post limit is not a int, given: ' + str(type(httppoost_limit))) 
     1432 
     1433  # if the http comand method isnt post theres is no posted data 
     1434  posted_data = None 
     1435 
     1436  # Bug pointed out by Albert Rafetseder: not all browsers send post with caps  
     1437  if httprequest_dictionary['http_command'].lower() == 'post': 
     1438    # get the posted data length or raise an exception if not given  
     1439    try: 
     1440      posted_data_length = int(httprequest_dictionary['Content-Length']) 
     1441    except Exception, e:    
     1442      raise HttpError411('content length is required on a http POST comand') 
     1443    else: 
     1444      # Bug pointed out by Albert Rafetseder: post doesnt send a empty line after the posted data 
     1445      # recieve the posted data using the posted data length 
     1446      posted_data = _registerhttpcallback_receive_httppost(sock, posted_data_length, httppoost_limit) 
     1447 
     1448    # check if there is a posted data and return it 
     1449    if len(posted_data) == 0: 
     1450      raise HttpError417('The request included a http POST comand with no data posted') 
     1451     
     1452  # adds the posted data or None to the httprequest dictionary  
     1453  httprequest_dictionary['posted_data'] = posted_data 
     1454 
     1455 
     1456 
     1457 
     1458 
     1459def _registerhttpcallback_receive_httppost(sock, http_post_length, length_limit): 
     1460  # receives posted data from the given connection untill the given content length field matchs the received amount  
     1461  total_recvd_post = '' 
     1462   
     1463  while True: 
     1464    if len(total_recvd_post) == http_post_length: 
     1465      # if the content length matchs the received posted data return it  
     1466      return total_recvd_post 
     1467 
     1468    # raise an exception if the received posted data length is greater than the given http header content length    
     1469    if len(total_recvd_post) > http_post_length: 
     1470      raise HttpError417('The http posted data didnt match the http content length header, given content length: ' + str(http_post_length) + ' while posted data length is ' + str(len(total_recvd_post))) 
     1471 
     1472    # raise an exception if the total received length has exceeded the given length limit 
     1473    if len(total_recvd_post) > length_limit:                   
     1474      raise HttpError413('The http posted data length exceeded length limit of ' + str(length_limit)) 
     1475 
     1476                        
     1477    try: 
     1478      # receive one character at a time inorder to check for the empty line 
     1479      content = sock.recv(512) 
     1480 
     1481    # catch any error that happens while receiving content              
     1482    except SocketTimeoutError, e: 
     1483      raise HttpError408('The server timed out while waiting to receive the post data ' + str(e)) 
     1484    except Exception, e:  
     1485      raise HttpError500('Error while receiving request posted data ' + str(e)) 
     1486 
     1487    else: 
     1488      # if there was not receiving error, keep on adding the receieved content  
     1489      total_recvd_post += content 
     1490 
     1491 
     1492 
     1493   
     1494 
     1495def _registerhttpcallback_receive_untilemptyline(sock, length_limit): 
     1496  # receives data from socket connection until it a empty line or until the given 
     1497  # length limit is exceeded                       
     1498  total_recvd_content = '' 
     1499   
     1500  while True: 
     1501    # receive until a empty line (\n\n or \r\n\r\n because new line is different by computer)  
     1502    if '\r\n\r\n' in total_recvd_content or '\n\n' in total_recvd_content: 
     1503      # found a empty line return the total received content 
     1504      return total_recvd_content.strip() 
     1505 
     1506    # raise an exception if the total received length has exceeded the given length limit 
     1507    if len(total_recvd_content) > length_limit:                   
     1508      raise HttpError413('The http request length exceeded the given length limit ' + str(length_limit)) 
     1509                         
     1510    try: 
     1511      # receive one character at a time inorder to check for the empty line 
     1512      content = sock.recv(1) 
     1513 
     1514    # catch any error that happens while receiving content              
     1515    except SocketTimeoutError, e: 
     1516      raise HttpError408('The server timed out while waiting to receive the request ' + str(e)) 
     1517    except Exception, e:  
     1518      raise HttpError500('Error while receiving http request ' + str(e)) 
     1519 
     1520    else: 
     1521      # if there was not receiving error, keep on adding the receieved content  
     1522      total_recvd_content += content 
     1523 
     1524 
     1525 
     1526 
     1527 
     1528def _registerhttpcallback_send_httpresponse(sock, callbackfunc_val): 
     1529  # sends a response to the client(web browser) with a ok http header and the http web page content 
     1530  if not type(callbackfunc_val) == list: 
     1531    raise HttpUserInputError('Callback func didnt return list, returned ' + str(type(callbackfunc_val))) 
     1532 
     1533  try: 
     1534    webpage_content = callbackfunc_val[0] 
     1535    callbckfunc_httpheader = callbackfunc_val[1] 
     1536  except Exception, e: 
     1537    raise HttpUserInputError('Callback func returned data failed ' + str(e)) 
     1538 
     1539  # check the given web page content  
     1540  if not type(webpage_content) == str: 
     1541    raise HttpUserInputError('Callback func didnt return str for the content, returned ' + str(type(webpage_content))) 
     1542  if len(webpage_content) == 0: 
     1543    raise HttpUserInputError('Callback func didnt return any content') 
     1544   
     1545   
     1546  # build the http ok response header  
     1547  httpheader = 'HTTP/1.0 200 OK\n' 
     1548  httpheader += 'Content-Length: ' + str(len(webpage_content)) + '\n' 
     1549  #check if there is a given http header and add it to the response 
     1550  httpheader += _registerhttpcallback_parse_callbckfunc_httpheader(callbckfunc_httpheader) 
     1551     
     1552  httpheader += 'Server: Seattle Testbed\n\n' 
     1553 
     1554  # http header followed by http content and close the connection 
     1555  try: 
     1556    sock.send(httpheader)  
     1557    sock.send(webpage_content) 
     1558    sock.close()  
     1559  except Exception, e: 
     1560    raise HttpConnectionError('server failed to send the http content ' + str(e))   
     1561 
     1562 
     1563 
     1564 
     1565 
     1566def _registerhttpcallback_parse_callbckfunc_httpheader(callbckfunc_httpheader): 
     1567  # builds a http header from the given http callback func list 
     1568  if callbckfunc_httpheader == None: 
     1569    # if the http header isnt given return a empty string 
     1570    return '' 
     1571  elif not type(callbckfunc_httpheader) == dict: 
     1572    # raise an exception if the http header isnt dictionary 
     1573    raise HttpUserInputError('The given http header is not a dictionary, given: ' + str(type(callbckfunc_httpheader))) 
     1574  else:  
     1575    # take the given key and val from the callbckfunc_httpheader dictionary and add them to the http header with 
     1576    # correct http format 
     1577    httpheaders = '' 
     1578    for key, val in callbckfunc_httpheader.items(): 
     1579      # make simple checks on the key and val  
     1580      if not type(key) == str: 
     1581        raise HttpUserInputError('The callback func given http header key isnt str given ' + str(type(key))) 
     1582      if not type(val) == str: 
     1583        raise HttpUserInputError('The callback func given http header value isnt str given ' + str( type(val))) 
     1584      if key == '' or val == '': 
     1585        # raise an exception if the key or value is a empty field 
     1586        raise HttpUserInputError('The callback func given empty http header feild of key or value') 
     1587      if key.capitalize() != key: 
     1588        raise HttpUserInputError('The callback func given http header key is not capitalized, given: ' + key) 
     1589 
     1590      # add the key and value to the http header field 
     1591      httpheaders += key + ' : ' + val + '\n'   
     1592 
     1593    # return the string of the http header   
     1594    return httpheaders 
     1595 
     1596   
     1597 
     1598 
     1599 
     1600def _registerhttpcallback_send_httpformat_error(sock, e): 
     1601  # send  correct format http header with a  http content that displays detailed error msg and close connection  
     1602 
     1603  # using the httpstatuscode dictionary get the statuscode number and statuscode constant from the given httperror 
     1604  (statuscode_numb, client_error_msg, statuscode_constant) = _registerhttpcallback_get_http_statuscode(e) 
     1605   
     1606  # build http body error msg to client(web browser) 
     1607  error_msg = client_error_msg 
     1608 
     1609  # error content body 
     1610  httpcontent = '<html>' 
     1611  httpcontent += '<head><title>' + str(statuscode_numb) + ' ' + statuscode_constant + '</title></head>' 
     1612  httpcontent += '<body><h1>' + str(statuscode_numb) + ' ' + statuscode_constant + '</h1>' 
     1613  httpcontent += '<p>' + error_msg + '</p></body>' 
     1614  httpcontent += '</html>' 
     1615  # to end the error content 
     1616  httpcontent += '\n\n' 
     1617   
     1618  # build the http header to send     
     1619  httpheader = 'HTTP/1.0 ' + str(statuscode_numb)  + ' ' + statuscode_constant + '\n' 
     1620 
     1621  # for redirect add the location of the redirection to the http header     
     1622  if statuscode_numb == 301 or statuscode_numb == 302: 
     1623    if client_error_msg == '': 
     1624      raise HttpUserInputError('Internal server error: callback func client should put the location on raising redirect') 
     1625    elif not client_error_msg.startswith('http://'): 
     1626      raise HttpUserInputError('Internal server error: calback func client redirect is invalid, Given: ' + client_error_msg) 
     1627    else: 
     1628      httpheader += 'Location: ' + str(client_error_msg) + '\n' 
     1629   
     1630  # finish up the http header 
     1631  httpheader += 'Content-Length: ' + str(len(httpcontent)) + '\n' 
     1632  httpheader += 'Server: Seattle Testbed\n\n' 
     1633   
     1634  # send the http response header and body to the client(web browser) and close connection 
     1635  try: 
     1636    sock.send(httpheader) 
     1637    sock.send(httpcontent) 
     1638    sock.close() 
     1639  except Exception, e: 
     1640    raise HttpConnectionError('server failed internally on send http error ' + str(statuscode_numb) + ' ' + statuscode_constant + ' ' + error_msg + ' Raised' + str(e))  
     1641 
     1642 
     1643 
     1644 
     1645def _registerhttpcallback_get_http_statuscode(e): 
     1646  # retrieves the status code number and constant given a exception class  
     1647   
     1648  # httpstatus code dictionary with the statuscode constant 
     1649  httpstatuscode_dict = { 
     1650      HttpError100: (100, 'Continue'), 
     1651      HttpError101: (101, 'Switching Protocols'), 
     1652      HttpError102: (102, 'Processing'), 
     1653      HttpError201: (201 ,'Created'), 
     1654      HttpError202: (202, 'Accepted'),   
     1655      HttpError203: (203, 'Non-Authoritative Information'), 
     1656      HttpError204: (204, 'No Content'), 
     1657      HttpError205: (205, 'Reset Content'), 
     1658      HttpError206: (206, 'Partial Content'), 
     1659      HttpError207: (207, 'Multi-Status'), 
     1660      HttpError226: (226, 'IM Used'), 
     1661      HttpError300: (300, 'Multiple Choices'), 
     1662      HttpError301: (301, 'Moved Permanently'), 
     1663      HttpError302: (302, 'Found'), 
     1664      HttpError303: (303, 'See Other'), 
     1665      HttpError304: (304, 'Not Modified'), 
     1666      HttpError305: (305, 'Use Proxy'), 
     1667      HttpError306: (306, 'Unused'), 
     1668      HttpError307: (307, 'Temporary Redirect'), 
     1669      HttpError400: (400, 'Bad Request'), 
     1670      HttpError401: (401, 'Unauthorized'), 
     1671      HttpError402: (402, 'Payment Required'), 
     1672      HttpError403: (403, 'Forbidden'), 
     1673      HttpError404: (404, 'Not Found'), 
     1674      HttpError405: (405, 'Method Not Allowed'), 
     1675      HttpError406: (406, 'Not Acceptable'), 
     1676      HttpError407: (407, 'Proxy Authentication Required'), 
     1677      HttpError408: (408, 'Request Timeout'), 
     1678      HttpError409: (409, 'Conflict'), 
     1679      HttpError410: (410, 'Gone'), 
     1680      HttpError411: (411, 'Length Required'), 
     1681      HttpError412: (412, 'Precondition Failed'), 
     1682      HttpError413: (413, 'Request Entity Too Large'), 
     1683      HttpError414: (414, 'Request-URI Too Long'), 
     1684      HttpError415: (415, 'Unsupported Media Type'), 
     1685      HttpError416: (416, 'Requested Range Not Satisfiable'), 
     1686      HttpError417: (417, 'Expectation Failed'), 
     1687      HttpError418: (418, 'Im a teapot'), 
     1688      HttpError422: (422, 'Unprocessable Entity'), 
     1689      HttpError423: (423, 'Locked'), 
     1690      HttpError424: (424, 'Failed Dependency'), 
     1691      HttpError425: (425, 'Unordered Collection'), 
     1692      HttpError426: (426, 'Upgrade Required'), 
     1693      HttpError500: (500, 'Internal Server Error'), 
     1694      HttpError501: (501, 'Not Implemented'), 
     1695      HttpError502: (502, 'Bad Gateway'), 
     1696      HttpError503: (503, 'Service Unavailable'), 
     1697      HttpError504: (504, 'Gateway Timeout'), 
     1698      HttpError505: (505, 'HTTP Version Not Supported'), 
     1699      HttpError506: (506, 'Variant Also Negotiates'), 
     1700      HttpError507: (507, 'Insufficient Storage'), 
     1701      HttpError510: (510, 'Not Extended')} 
     1702   
     1703  # retrieves the status number and constant from the given exception class using the dictionary  
     1704  try: 
     1705    (statuscode_numb, statuscode_constant) = httpstatuscode_dict[type(e)] 
     1706  except Exception, e: 
     1707    raise HttpServerError('Internal error on generating error msg: ' + str(e)) 
     1708 
     1709  # get any extra error msg that the callback fucntion raised  
     1710  client_error_msg = str(e) 
     1711 
     1712  # return what is retrieved 
     1713  return statuscode_numb, client_error_msg, statuscode_constant 
     1714 
     1715 
     1716 
     1717 
     1718 
     1719 
     1720 
     1721 
     1722 
     1723 
     1724 
     1725 
     1726 
     1727 
     1728 
     1729""" 
     1730<Program Name> 
     1731  httpretrieve.repy 
     1732 
     1733<Started> 
     1734  August 19, 2009 
     1735 
     1736<Author> 
     1737  Yafete Yemuru 
     1738 
     1739<Purpose> 
     1740  provides a http content from a web server using http protocol. It sends a http request to 
     1741  any http server through socket connection to get the http content. Then once the http server 
     1742  replies with http header and content, the http header is checked for any error message. 
     1743  Then provides http content in a format of string, saved in a file or as a file like object.  
     1744""" 
     1745 
     1746 
     1747 
     1748 
     1749#begin include urlparse.repy   
     1750""" 
     1751<Program Name> 
     1752  urlparse.repy 
     1753 
     1754<Started> 
     1755  May 15, 2009 
     1756 
     1757<Author> 
     1758  Michael Phan-Ba 
     1759 
     1760<Purpose> 
     1761  Provides utilities for parsing URLs, based on the Python 2.6.1 module urlparse. 
     1762 
     1763""" 
     1764 
     1765 
     1766def urlparse_urlsplit(urlstring, default_scheme="", allow_fragments=True): 
     1767  """ 
     1768  <Purpose> 
     1769    Parse a URL into five components, returning a dictionary.  This corresponds 
     1770    to the general structure of a URL: 
     1771    scheme://netloc/path;parameters?query#fragment.  The parameters are not 
     1772    split from the URL and individual componenets are not separated. 
     1773 
     1774    Only absolute server-based URIs are currently supported (all URLs will be 
     1775    parsed into the components listed, regardless of the scheme). 
     1776 
     1777  <Arguments> 
     1778    default_scheme: 
     1779      Optional: defaults to the empty string.  If specified, gives the default 
     1780      addressing scheme, to be used only if the URL does not specify one. 
     1781 
     1782    allow_fragments: 
     1783      Optional: defaults to True.  If False, fragment identifiers are not 
     1784      allowed, even if the URL's addressing scheme normally does support them. 
     1785 
     1786  <Exceptions> 
     1787    ValueError on parsing a non-numeric port value. 
     1788 
     1789  <Side Effects> 
     1790    None. 
     1791 
     1792  <Returns> 
     1793    A dictionary containing: 
     1794 
     1795    Key         Value                               Value if not present 
     1796    ============================================================================ 
     1797    scheme      URL scheme specifier                empty string 
     1798    netloc      Network location part               empty string 
     1799    path        Hierarchical path                   empty string 
     1800    query       Query component                     empty string 
     1801    fragment    Fragment identifier                 empty string 
     1802    username    User name                           None 
     1803    password    Password                            None 
     1804    hostname    Host name (lower case)              None 
     1805    port        Port number as integer, if present  None 
     1806 
     1807  """ 
     1808 
     1809  components = {"scheme": default_scheme, "netloc": "", "path": "", "query": "", 
     1810    "fragment": "", "username": None, "password": None, "hostname": None, 
     1811    "port": None } 
     1812 
     1813  # Extract the scheme, if present. 
     1814  (lpart, rpart) = _urlparse_splitscheme(urlstring) 
     1815  if lpart: 
     1816    components["scheme"] = lpart 
     1817 
     1818  # Extract the server information, if present. 
     1819  if rpart.startswith("//"): 
     1820    (lpart, rpart) = _urlparse_splitnetloc(rpart, 2) 
     1821    components["netloc"] = lpart 
     1822 
     1823    (components["username"], components["password"], components["hostname"], 
     1824      components["port"]) = _urlparse_splitauthority(lpart) 
     1825 
     1826  # Extract the fragment. 
     1827  if allow_fragments: 
     1828    (rpart, components["fragment"]) = _urlparse_splitfragment(rpart) 
     1829 
     1830 
     1831  # Extract the query. 
     1832  (components["path"], components["query"]) = _urlparse_splitquery(rpart) 
     1833 
     1834  return components 
     1835 
     1836 
     1837def _urlparse_splitscheme(url): 
     1838  """Parse the scheme portion of the URL""" 
     1839  # The scheme is valid only if it contains these characters. 
     1840  scheme_chars = \ 
     1841    "abcdefghijklmnopqrstuvwxyz0123456789+-." 
     1842 
     1843  scheme = "" 
     1844  rest = url 
     1845 
     1846  spart = url.split(":", 1) 
     1847  if len(spart) == 2: 
     1848 
     1849    # Normalize the scheme. 
     1850    spart[0] = spart[0].lower() 
     1851 
     1852    # A scheme is valid only if it starts with an alpha character. 
     1853    if spart[0] and spart[0][0].isalpha(): 
     1854      for char in spart[0]: 
     1855        if char not in scheme_chars: 
     1856          break 
     1857      (scheme, rest) = spart 
     1858 
     1859  return scheme, rest 
     1860 
     1861 
     1862def _urlparse_splitnetloc(url, start=0): 
     1863  """Parse the netloc portion of the URL""" 
     1864 
     1865  # By default, the netloc is delimited by the end of the URL. 
     1866  delim = len(url) 
     1867 
     1868  # Find the left-most delimiter. 
     1869  for char in "/?#": 
     1870    xdelim = url.find(char, start) 
     1871    if xdelim >= 0: 
     1872      delim = min(delim, xdelim) 
     1873 
     1874  # Return the netloc and the rest of the URL. 
     1875  return url[start:delim], url[delim:] 
     1876 
     1877 
     1878def _urlparse_splitauthority(netloc): 
     1879  """Parse the authority portion of the netloc""" 
     1880 
     1881  # The authority can have a userinfo portion delimited by "@". 
     1882  authority = netloc.split("@", 1) 
     1883 
     1884  # Default values. 
     1885  username = None 
     1886  password = None 
     1887  hostname = None 
     1888  port = None 
     1889 
     1890  # Is there a userinfo portion? 
     1891  if len(authority) == 2: 
     1892 
     1893    # userinfo can be split into username:password 
     1894    userinfo = authority[0].split(":", 1) 
     1895 
     1896    # hostport can be split into hostname:port 
     1897    hostport = authority[1].split(":", 1) 
     1898 
     1899    if userinfo[0]: 
     1900      username = userinfo[0] 
     1901    if len(userinfo) == 2: 
     1902      password = userinfo[1] 
     1903 
     1904  # No userinfo portion found. 
     1905  else: 
     1906 
     1907    # hostport can be split into hostname:port 
     1908    hostport = netloc.split(":", 1) 
     1909 
     1910  # Is there a port value? 
     1911  if hostport[0]: 
     1912    hostname = hostport[0] 
     1913  if len(hostport) == 2: 
     1914    port = int(hostport[1], 10) 
     1915 
     1916  # Return the values. 
     1917  return username, password, hostname, port 
     1918 
     1919 
     1920def _urlparse_splitquery(url): 
     1921  """Parse the query portion of the url""" 
     1922 
     1923  qpart = url.split("?", 1) 
     1924  if len(qpart) == 2: 
     1925    query = qpart[1] 
     1926  else: 
     1927    query = "" 
     1928 
     1929  return qpart[0], query 
     1930 
     1931 
     1932def _urlparse_splitfragment(url): 
     1933  """Parse the query portion of the url""" 
     1934 
     1935  fpart = url.split("#", 1) 
     1936  if len(fpart) == 2: 
     1937    fragment = fpart[1] 
     1938  else: 
     1939    fragment = "" 
     1940 
     1941  return fpart[0], fragment 
     1942 
     1943#end include urlparse.repy   
     1944#begin include sockettimeout.repy 
     1945""" 
     1946<Description> 
     1947  Puts back in Python's non-blocking functionality. 
     1948 
     1949  send(): 
     1950    Raises SocketTimeout Error if the send call lasts 
     1951    longer than the set timeout. 
     1952 
     1953  recv(): 
     1954    Guarentees the receipt of a message.   Raises SocketTimeoutError if it does not 
     1955    receive any message before a given timeout. 
     1956    If actually receives the message, returns the message and continues. 
     1957 
     1958<Usage> 
     1959  Text-replacable for Repy Sockets: 
     1960    timeout_openconn(desthost, destport, localip=None, localport=None, timeout = 5) 
     1961    timeout_waitforconn(localip, localport, function) 
     1962 
     1963  Object: 
     1964    sockobj.settimeout(seconds) 
     1965    sockobj.send(data) 
     1966    sockobj.recv(bytes) 
     1967    sockobj.close() 
     1968 
     1969<Date> 
     1970  Sun Mar  1 10:27:35 PST 2009 
     1971 
     1972<Example> 
     1973  # hello world 
     1974  include sockettimer.repy 
     1975 
     1976  def callback(ip, port, timeout_sockobj, commhandle, listenhandle): 
     1977    hw_message = timeout_sockobj.recv(1047) 
     1978 
     1979    # cleanup 
     1980    stopcomm(commhandle) 
     1981    stopcomm(listenhandle) 
     1982    timeout_sockobj.close() 
     1983 
     1984    print hw_message # => "hello world!" 
     1985   
     1986  def server(): 
     1987    sockobj = timeout_waitforconn(getmyip(), 12345, callback) 
     1988 
     1989  def client(): 
     1990    sockobj = timeout_openconn(getmyip(), 12345) 
     1991    sockobj.send("hello world!") 
     1992 
     1993  def main(): 
     1994    server() 
     1995    client() 
     1996    exitall() 
     1997 
     1998  if callfunc == 'initialize': 
     1999    main()  
     2000""" 
     2001 
     2002class SocketTimeoutError(Exception): 
     2003  """The socket timed out before receiving a response""" 
     2004 
     2005def timeout_openconn(desthost, destport, localip=None, localport=None, timeout = 5): 
     2006  """ 
     2007  <Purpose>  
     2008    Wrapper for Repy like socket interface 
     2009 
     2010  <Args> 
     2011    Same as Repy openconn 
     2012 
     2013  <Exception> 
     2014    Timeout exception if the dest address doesnt respond. 
     2015 
     2016  <Returns> 
     2017    socket obj on success 
     2018  """ 
     2019 
     2020  tsock = TimeoutSocket() 
     2021  tsock.settimeout(timeout) 
     2022  if localip and localport: 
     2023    tsock.bind((localip, localport)) 
     2024  tsock.connect((desthost, destport)) 
     2025  return tsock 
     2026 
     2027def timeout_waitforconn(localip, localport, function): 
     2028  """ 
     2029  <Purpose>  
     2030    Wrapper for Repy like socket interface 
     2031 
     2032  <Args> 
     2033    Same as Repy waitforconn 
     2034 
     2035  <Side Effects> 
     2036    Sets up event listener which calls function on messages. 
     2037 
     2038  <Returns> 
     2039    Handle to listener. 
     2040  """ 
     2041 
     2042  tsock = TimeoutSocket() 
     2043  tsock.bind((localip, localport)) 
     2044  tsock.setcallback(function) 
     2045  return tsock.listen() 
     2046 
     2047class TimeoutSocket: 
     2048  """ 
     2049  <Purpose> 
     2050    Provide an socket object like the Repy usual one. 
     2051 
     2052  <Side Effects> 
     2053    Uses a getlock() to watch for a timeout 
     2054    Uses waitforconn and openconn to simulate socket 
     2055  """ 
     2056 
     2057  ################ 
     2058  # Constructors 
     2059  ################ 
     2060 
     2061  def __init__(self): 
     2062    """ Constructor for socket """ 
     2063#    self.lock = getlock() # general lock BUG: Do we need to lock everything? 
     2064    self.timeout_lock = getlock() # special lock for Timeout condition 
     2065    self.timeout = 5 # seconds to wait 
     2066    self.bytes_sent = None # used to check if send() timed out 
     2067 
     2068    # user vars    
     2069    self.local_address = None # ip, port 
     2070    self.remote_address = None # ip, port 
     2071    self.callback = None # the user's function to call 
     2072 
     2073    # repy socket vars 
     2074    self.sockobj = None #  the Repy socket 
     2075    self.commhandle = None # the current comm 
     2076    self.listencommhandle = None # the listener comm 
     2077 
     2078    #error tracking vars 
     2079     
     2080    #if any exceptions are thrown in the separate thread executing _send_and_release,  
     2081    #they are caught and stored in this variable, then raised in _send_or_close 
     2082    self.TCPSendError = None 
     2083 
     2084  ################ 
     2085  # Mutator methods 
     2086  ################# 
     2087 
     2088  def settimeout(self, value): 
     2089    """ Setter for timeout""" 
     2090    self.timeout = value 
     2091 
     2092  def setcallback(self, function): 
     2093    """ Setter for callback function""" 
     2094    self.callback = function 
     2095 
     2096  #################### 
     2097  # Public Methods 
     2098  #################### 
     2099 
     2100  def bind(self, local_address = None): 
     2101    """ 
     2102    <Purpose> 
     2103      Set local address 
     2104 
     2105    <Args> 
     2106      Tuple of (ip, port) local. 
     2107    """ 
     2108    self.local_address = local_address 
     2109 
     2110  def listen(self): 
     2111    """ 
     2112    <Purpose> 
     2113      Listen for peer 
     2114     
     2115    <Side Effects> 
     2116      Calls Repy waitforconn() 
     2117    """ 
     2118    return self._waitforconn() 
     2119 
     2120  def connect(self, remote_address): 
     2121    """ 
     2122    <Purpose> 
     2123      Connect to peer. 
     2124 
     2125    <Args> 
     2126      Tuple of (ip, port) remote. 
     2127    
     2128    <Side Effects> 
     2129      Calls Repy openconn. 
     2130    """ 
     2131    self.remote_address = remote_address 
     2132    self._openconn() 
     2133 
     2134  def recv(self, maxLen): # timeout as optional arg ??? 
     2135    """ 
     2136    <Purpose> 
     2137      If it fails to finish within the timeout, I close the socket and raise a 
     2138      TimeoutError exception. I.e. if there's no message, we call it an error 
     2139      and raise it. 
     2140       
     2141    <Arguments> 
     2142      maxLen - bytes to recv 
     2143 
     2144    <Exception> 
     2145      Raises TimeoutError exception if the recv times out 
     2146      without receiving a message. 
     2147 
     2148    <Side Effects> 
     2149      Closes the connection if times out. 
     2150 
     2151    <Returns> 
     2152      The message. 
     2153    """ 
     2154    return self._recv_or_close(maxLen) 
     2155 
     2156  def send(self, data): 
     2157    """ 
     2158    <Purpose> 
     2159      Just like normal Repy socket.  Sends messages. 
     2160       
     2161    <Arguments> 
     2162      data - the string message 
     2163 
     2164    <Exception> 
     2165      Same as Repy socket. 
     2166  
     2167    <Returns> 
     2168      The bytes sent. 
     2169    """ 
     2170    return self._send_or_close(data) 
     2171 
     2172  def close(self): 
     2173    self.local_address = None # ip, port 
     2174    self.remote_address = None # ip, port 
     2175    self.callback = None # the user's function to call 
     2176 
     2177    if self.sockobj: 
     2178      self.sockobj.close() 
     2179    self.sockobj = None #  the Repy socket 
     2180     
     2181    # Armon: As part of the semantics, stopcomm will raise an  
     2182    # exception given an invalid handle, e.g. None. Thus, 
     2183    # we need to check for this. 
     2184    if self.commhandle:  
     2185      stopcomm(self.commhandle) 
     2186      self.commhandle = None # the current comm 
     2187     
     2188    # Armon: Same as above. 
     2189    if self.listencommhandle: 
     2190      stopcomm(self.listencommhandle) 
     2191      self.listencommhandle = None # the listener comm 
     2192 
     2193 
     2194  ######################## 
     2195  # Private 
     2196  ######################### 
     2197 
     2198  def _openconn(self): 
     2199    """Handle current state variables and call Repy openconn.""" 
     2200 
     2201    destip, destport = self.remote_address 
     2202    if self.local_address: 
     2203      srcip, srcport = self.local_address 
     2204      self.sockobj = openconn(destip, destport, srcip, srcport, self.timeout) 
     2205    else: 
     2206      self.sockobj = openconn(destip, destport) 
     2207 
     2208  def _waitforconn(self): 
     2209    """Setup way between Repy waitforconn event""" 
     2210    localip, localport = self.local_address 
     2211    self.listencommhandle = waitforconn(localip, localport, self._callback) 
     2212    return self.listencommhandle 
     2213 
     2214  def _callback(self, ip, port, sockobj, ch, lh): 
     2215    """Pass on through to user callback""" 
     2216    self.sockobj = sockobj 
     2217    self.listencommhandle = lh # same as the 1st from wait for comm, right? 
     2218    self.commhandle = ch # should we care? 
     2219     
     2220    if not self.remote_address: 
     2221      self.remote_address = (ip, port) 
     2222    else:  
     2223      raise Exception("what! peer does not match?") 
     2224 
     2225    self.callback(ip, port, self, ch, lh) 
     2226 
     2227  def _send(self, data): 
     2228    """Send data""" 
     2229    return self.sockobj.send(data) 
     2230 
     2231  def _recv(self, maxLen): 
     2232    """Recv data of length maxLen""" 
     2233    return self.sockobj.recv(maxLen) 
     2234 
     2235  def _send_and_release(self, data): 
     2236    """Send data then release the timeout lock""" 
     2237     
     2238    #Bug Fix (Cosmin): exceptions thrown in separate thread _send_and_release could not be caught 
     2239    #now we store the exception if it is thrown and raise it back in send_or_close 
     2240    try: 
     2241      self.bytes_sent = self._send(data) 
     2242    except Exception, e: 
     2243      self.TCPSendError = e 
     2244     
     2245    self._quietly_release() # release the lock 
     2246  
     2247  def _quietly_release(self): 
     2248    """Release the timeout lock and ignore if already released""" 
     2249    try: 
     2250      self.timeout_lock.release() 
     2251    except: 
     2252      pass 
     2253    
     2254  def _send_or_close(self, data): 
     2255    """Raise the Timeout Error if no receipt.  Keep track by timeout_lock.""" 
     2256 
     2257    # acquire the lock, when it's release we'll carry on 
     2258    self.timeout_lock.acquire() 
     2259 
     2260    # fork off a lock that'll release the lock at the timeout 
     2261    timerhandle = settimer(self.timeout, self._quietly_release, ()) 
     2262 
     2263    # fork off a send call so we can raise the exception in the main thread 
     2264    # the send call will also release our lock 
     2265    settimer(0, self._send_and_release, (data,)) 
     2266 
     2267    # block until either the timeout or _send finishes 
     2268    self.timeout_lock.acquire() 
     2269    self.timeout_lock.release() 
     2270 
     2271    if self.bytes_sent: # send finished 
     2272      canceltimer(timerhandle) 
     2273      retdata = self.bytes_sent 
     2274      self.bytes_sent = None 
     2275      return retdata 
     2276    elif self.TCPSendError != None: 
     2277      #Bug Fix (Cosmin): exceptions thrown in separate thread _send_and_release could not be caught 
     2278       
     2279      #we got an error within the separate thread that performed the send operation 
     2280      exception_to_throw = self.TCPSendError 
     2281      self.TCPSendError = None 
     2282      raise exception_to_throw 
     2283    else: # it timed out 
     2284      self.close() 
     2285      raise SocketTimeoutError 
     2286 
     2287  def _recv_or_close(self, amount): 
     2288    """Raise the Timeout Error if no receipt.  Keep track by timeout_lock.""" 
     2289    timerhandle = settimer(self.timeout, self._clobbersocket, ()) 
     2290    try: 
     2291      retdata = self._recv(amount) 
     2292    except Exception, e: 
     2293      # if it's not the timeout, reraise... 
     2294      if self.timeout_lock.acquire(False): 
     2295        raise 
     2296      raise SocketTimeoutError 
     2297     
     2298    # I acquired the lock, I should stop the timer because I succeeded... 
     2299    if self.timeout_lock.acquire(False): 
     2300      # even if this isn't in time, the lock prevents a race condition  
     2301      # this is merely an optimization to prevent the timer from ever firing... 
     2302      canceltimer(timerhandle) 
     2303      self.timeout_lock.release() # Alper's bug 3/10/09 
     2304      return retdata 
     2305    else: 
     2306      raise SocketTimeoutError 
     2307 
     2308  def _clobbersocket(self): 
     2309    """If I can acquire the lock without blocking, then close the socket to abort""" 
     2310    if self.timeout_lock.acquire(False): 
     2311      self.close() 
     2312 
     2313 
     2314############################ 
     2315# Deprecated functions 
     2316############################## 
     2317 
     2318# private function... 
     2319def sockettimeout_clobbersocket(sockobj,mylock): 
     2320  # if I can acquire the lock without blocking, then close the socket to abort 
     2321  if mylock.acquire(False): 
     2322    sockobj.close() 
     2323 
     2324# if it fails to finish within the timeout, I close the socket and raise a 
     2325# SocketTimeout exception... 
     2326def sockettimeout_recv_or_close(sockobj, amount, timeout): 
     2327  # A lock I'll use for this attempt 
     2328  mylock = getlock() 
     2329  timerhandle = settimer(timeout,clobbersocket, (sockobj, mylock)) 
     2330  try: 
     2331    retdata = sockobj.recv(amount) 
     2332  except Exception, e: 
     2333    # if it's not the timeout, reraise... 
     2334    if mylock.acquire(False): 
     2335      raise 
     2336    raise SocketTimeout 
     2337     
     2338  # I acquired the lock, I should stop the timer because I succeeded... 
     2339  if mylock.acquire(False): 
     2340    # even if this isn't in time, the lock prevents a race condition  
     2341    # this is merely an optimization to prevent the timer from ever firing... 
     2342    canceltimer(timerhandle) 
     2343    return retdata 
     2344  else: 
     2345    raise SocketTimeout 
     2346 
     2347 
     2348#end include sockettimeout.repy 
     2349#begin include http_hierarchy_error.repy 
     2350""" 
     2351<Program Name> 
     2352  http_hierarchy_error.repy 
     2353 
     2354<Started> 
     2355  Oct 05, 2009 
     2356 
     2357<Author> 
     2358  Yafete Yemuru 
     2359 
     2360<Purpose> 
     2361  provides a hierachy http error using status code including client and server errors 
     2362  classes.   
     2363""" 
     2364 
     2365 
     2366''' 
     2367http hierarchy error exception classes 
     2368 
     2369-> HttpError 
     2370   -> HttpRetrieveClientError 
     2371      -> HttpUserInputError 
     2372      -> HttpConnectionError 
     2373 
     2374   -> HttpServerError 
     2375      -> HttpResponseError 
     2376         -> HttpHeaderError 
     2377            -> HttpHeaderReceivingError 
     2378            -> HttpHeaderFormatError 
     2379         -> HttpContentError 
     2380            -> HttpContentReceivingError 
     2381            -> HttpContentLengthError 
     2382        
     2383   -> HttpStatuscodeError 
     2384     -> HttpError1xx 
     2385        -> followed by all http status code error number HttpError(number) 
     2386         
     2387     -> HttpError2xx 
     2388        -> followed by all http status code error number HttpError(number) 
     2389         
     2390     -> HttpError3xx 
     2391        -> followed by all http status code error number HttpError(number) 
     2392 
     2393     -> HttpError4xx 
     2394        -> followed by all http status code error number HttpError(number) 
     2395         
     2396     -> HttpError5xx 
     2397        -> followed by all http status code error number HttpError(number) 
     2398 
     2399''' 
     2400 
     2401class HttpError(Exception): 
     2402  pass 
     2403 
     2404# raises an exception for http client error  
     2405class HttpRetrieveClientError(HttpError):# extend HttpError  
     2406  pass 
     2407class HttpUserInputError(HttpRetrieveClientError): 
     2408  pass 
     2409class HttpConnectionError(HttpRetrieveClientError): 
     2410  pass 
     2411 
     2412 
     2413# raises an exception for any http server failure   
     2414class HttpServerError(HttpError):# extend HttpError  
     2415  pass 
     2416class HttpResponseError(HttpServerError): 
     2417  pass 
     2418class HttpHeaderError(HttpResponseError): 
     2419  pass 
     2420class HttpHeaderReceivingError(HttpHeaderError): 
     2421  pass 
     2422class HttpHeaderFormatError(HttpHeaderError): 
     2423  pass 
     2424class HttpContentError(HttpResponseError): 
     2425  pass 
     2426class HttpContentReceivingError(HttpContentError): 
     2427  pass 
     2428class HttpContentLengthError(HttpContentError): 
     2429  pass 
     2430 
     2431 
     2432class HttpStatusCodeError(HttpError):# extend HttpError 
     2433  pass 
     2434class HttpError1xx(HttpStatusCodeError): 
     2435  pass 
     2436class HttpError100(HttpError1xx): 
     2437  pass 
     2438class HttpError101(HttpError1xx):  
     2439  pass 
     2440class HttpError102(HttpError1xx):  
     2441  pass 
     2442 
     2443 
     2444class HttpError2xx(HttpStatusCodeError): 
     2445  pass 
     2446class HttpError201(HttpError2xx):  
     2447  pass 
     2448class HttpError202(HttpError2xx):    
     2449  pass 
     2450class HttpError203(HttpError2xx):  
     2451  pass 
     2452class HttpError204(HttpError2xx):  
     2453  pass 
     2454class HttpError205(HttpError2xx):  
     2455  pass 
     2456class HttpError206(HttpError2xx):  
     2457  pass 
     2458class HttpError207(HttpError2xx):  
     2459  pass 
     2460class HttpError226(HttpError2xx):  
     2461  pass                  
     2462 
     2463 
     2464class HttpError3xx(HttpStatusCodeError): 
     2465  pass 
     2466class HttpError300(HttpError3xx):  
     2467  pass 
     2468class HttpError301(HttpError3xx):  
     2469  pass 
     2470class HttpError302(HttpError3xx):  
     2471  pass 
     2472class HttpError303(HttpError3xx):  
     2473  pass 
     2474class HttpError304(HttpError3xx): 
     2475  pass 
     2476class HttpError305(HttpError3xx):  
     2477  pass 
     2478class HttpError306(HttpError3xx):  
     2479  pass 
     2480class HttpError307(HttpError3xx):  
     2481  pass 
     2482                     
     2483 
     2484class HttpError4xx(HttpStatusCodeError): 
     2485  pass 
     2486class HttpError400(HttpError4xx): 
     2487  pass   
     2488class HttpError401(HttpError4xx):  
     2489  pass 
     2490class HttpError402(HttpError4xx):  
     2491  pass 
     2492class HttpError403(HttpError4xx): 
     2493  pass   
     2494class HttpError404(HttpError4xx):  
     2495  pass 
     2496class HttpError405(HttpError4xx):  
     2497  pass 
     2498class HttpError406(HttpError4xx):  
     2499  pass 
     2500class HttpError407(HttpError4xx):  
     2501  pass 
     2502class HttpError408(HttpError4xx):  
     2503  pass 
     2504class HttpError409(HttpError4xx):  
     2505  pass 
     2506class HttpError410(HttpError4xx):  
     2507  pass 
     2508class HttpError411(HttpError4xx):  
     2509  pass 
     2510class HttpError412(HttpError4xx):  
     2511  pass 
     2512class HttpError413(HttpError4xx):  
     2513  pass 
     2514class HttpError414(HttpError4xx):  
     2515  pass 
     2516class HttpError415(HttpError4xx):  
     2517  pass 
     2518class HttpError416(HttpError4xx):  
     2519  pass 
     2520class HttpError417(HttpError4xx):  
     2521  pass 
     2522class HttpError418(HttpError4xx):  
     2523  pass 
     2524class HttpError422(HttpError4xx):  
     2525  pass 
     2526class HttpError423(HttpError4xx):  
     2527  pass 
     2528class HttpError424(HttpError4xx):  
     2529  pass 
     2530class HttpError425(HttpError4xx):  
     2531  pass 
     2532class HttpError426(HttpError4xx):  
     2533  pass 
     2534 
     2535 
     2536class HttpError5xx(HttpStatusCodeError): 
     2537  pass 
     2538class HttpError500(HttpError5xx):  
     2539  pass 
     2540class HttpError501(HttpError5xx):  
     2541  pass 
     2542class HttpError502(HttpError5xx):  
     2543  pass 
     2544class HttpError503(HttpError5xx):  
     2545  pass 
     2546class HttpError504(HttpError5xx):  
     2547  pass 
     2548class HttpError505(HttpError5xx):  
     2549  pass 
     2550class HttpError506(HttpError5xx): 
     2551  pass   
     2552class HttpError507(HttpError5xx): 
     2553  pass   
     2554class HttpError510(HttpError5xx): 
     2555  pass 
     2556 
     2557#end include http_hierarchy_error.repy 
     2558#begin include urllib.repy 
     2559def urllib_quote(string, safe="/"): 
     2560  """ 
     2561  <Purpose> 
     2562    Encode a string such that it can be used safely in a URL or XML 
     2563    document. 
     2564 
     2565  <Arguments> 
     2566    string: 
     2567           The string to urlencode. 
     2568 
     2569    safe (optional): 
     2570           Specifies additional characters that should not be quoted -- 
     2571           defaults to "/". 
     2572 
     2573  <Exceptions> 
     2574    TypeError if the safe parameter isn't an enumerable. 
     2575 
     2576  <Side Effects> 
     2577    None. 
     2578 
     2579  <Returns> 
     2580    Urlencoded version of the passed string. 
     2581  """ 
     2582 
     2583  resultstr = "" 
     2584 
     2585  # We go through each character in the string; if it's not in [0-9a-zA-Z] 
     2586  # we wrap it. 
     2587 
     2588  safeset = set(safe) 
     2589 
     2590  for char in string: 
     2591    asciicode = ord(char) 
     2592    if (asciicode >= ord("0") and asciicode <= ord("9")) or \ 
     2593        (asciicode >= ord("A") and asciicode <= ord("Z")) or \ 
     2594        (asciicode >= ord("a") and asciicode <= ord("z")) or \ 
     2595        asciicode == ord("_") or asciicode == ord(".") or \ 
     2596        asciicode == ord("-") or char in safeset: 
     2597      resultstr += char 
     2598    else: 
     2599      resultstr += "%%%02X" % asciicode 
     2600 
     2601  return resultstr 
     2602 
     2603 
     2604 
     2605 
     2606def urllib_quote_plus(string, safe=""): 
     2607  """ 
     2608  <Purpose> 
     2609    Encode a string to go in the query fragment of a URL. 
     2610 
     2611  <Arguments> 
     2612    string: 
     2613           The string to urlencode. 
     2614 
     2615    safe (optional): 
     2616           Specifies additional characters that should not be quoted -- 
     2617           defaults to the empty string. 
     2618 
     2619  <Exceptions> 
     2620    TypeError if the safe parameter isn't a string. 
     2621 
     2622  <Side Effects> 
     2623    None. 
     2624 
     2625  <Returns> 
     2626    Urlencoded version of the passed string. 
     2627  """ 
     2628 
     2629  return urllib_quote(string, safe + " ").replace(" ", "+") 
     2630 
     2631 
     2632 
     2633 
     2634def urllib_unquote(string): 
     2635  """ 
     2636  <Purpose> 
     2637    Unquote a urlencoded string. 
     2638 
     2639  <Arguments> 
     2640    string: 
     2641           The string to unquote. 
     2642 
     2643  <Exceptions> 
     2644    ValueError thrown if the last wrapped octet isn't a valid wrapped octet 
     2645    (i.e. if the string ends in "%" or "%x" rather than "%xx". Also throws 
     2646    ValueError if the nibbles aren't valid hex digits. 
     2647 
     2648  <Side Effects> 
     2649    None. 
     2650 
     2651  <Returns> 
     2652    The decoded string. 
     2653  """ 
     2654 
     2655  resultstr = "" 
     2656 
     2657  # We go through the string from end to beginning, looking for wrapped 
     2658  # octets. When one is found we add it (unwrapped) and the following 
     2659  # string to the resultant string, and shorten the original string. 
     2660 
     2661  while True: 
     2662    lastpercentlocation = string.rfind("%") 
     2663    if lastpercentlocation < 0: 
     2664      break 
     2665 
     2666    wrappedoctetstr = string[lastpercentlocation+1:lastpercentlocation+3] 
     2667    if len(wrappedoctetstr) != 2: 
     2668      raise ValueError("Quoted string is poorly formed") 
     2669 
     2670    resultstr = \ 
     2671        chr(int(wrappedoctetstr, 16)) + \ 
     2672        string[lastpercentlocation+3:] + \ 
     2673        resultstr 
     2674    string = string[:lastpercentlocation] 
     2675 
     2676  resultstr = string + resultstr 
     2677  return resultstr 
     2678 
     2679 
     2680 
     2681 
     2682def urllib_unquote_plus(string): 
     2683  """ 
     2684  <Purpose> 
     2685    Unquote the urlencoded query fragment of a URL. 
     2686 
     2687  <Arguments> 
     2688    string: 
     2689           The string to unquote. 
     2690 
     2691  <Exceptions> 
     2692    ValueError thrown if the last wrapped octet isn't a valid wrapped octet 
     2693    (i.e. if the string ends in "%" or "%x" rather than "%xx". Also throws 
     2694    ValueError if the nibbles aren't valid hex digits. 
     2695 
     2696  <Side Effects> 
     2697    None. 
     2698 
     2699  <Returns> 
     2700    The decoded string. 
     2701  """ 
     2702 
     2703  return urllib_unquote(string.replace("+", " ")) 
     2704 
     2705 
     2706 
     2707 
     2708def urllib_quote_parameters(dictionary): 
     2709  """ 
     2710  <Purpose> 
     2711    Encode a dictionary of (key, value) pairs into an HTTP query string or 
     2712    POST body (same form). 
     2713 
     2714  <Arguments> 
     2715    dictionary: 
     2716           The dictionary to quote. 
     2717 
     2718  <Exceptions> 
     2719    None. 
     2720 
     2721  <Side Effects> 
     2722    None. 
     2723 
     2724  <Returns> 
     2725    The quoted dictionary. 
     2726  """ 
     2727 
     2728  quoted_keyvals = [] 
     2729  for key, val in dictionary.items(): 
     2730    quoted_keyvals.append("%s=%s" % (urllib_quote(key), urllib_quote(val))) 
     2731 
     2732  return "&".join(quoted_keyvals) 
     2733 
     2734 
     2735 
     2736 
     2737def urllib_unquote_parameters(string): 
     2738  """ 
     2739  <Purpose> 
     2740    Decode a urlencoded query string or POST body. 
     2741 
     2742  <Arguments> 
     2743    string: 
     2744           The string to decode. 
     2745 
     2746  <Exceptions> 
     2747    ValueError if the string is poorly formed. 
     2748 
     2749  <Side Effects> 
     2750    None. 
     2751 
     2752  <Returns> 
     2753    A dictionary mapping keys to values. 
     2754  """ 
     2755 
     2756  keyvalpairs = string.split("&") 
     2757  res = {} 
     2758 
     2759  for quotedkeyval in keyvalpairs: 
     2760    # Throw ValueError if there is more or less than one '='. 
     2761    quotedkey, quotedval = quotedkeyval.split("=") 
     2762    key = urllib_unquote(quotedkey) 
     2763    val = urllib_unquote(quotedval) 
     2764    res[key] = val 
     2765 
     2766  return res 
     2767 
     2768#end include urllib.repy 
     2769 
     2770 
     2771 
     2772 
     2773 
     2774def httpretrieve_open(url, http_query=None, http_post=None, http_header=None, header_timeout=30, content_timeout=30, httpheader_limit=8192, httpcontent_limit=4194304): 
     2775  """ 
     2776  <Purpose> 
     2777     Returns file like object that that can read the http content form a http server. The file like 
     2778     object gets string from http server using read method.   
     2779 
     2780  <Arguments> 
     2781    url: 
     2782           String of a http web server-URL 
     2783    http_post: 
     2784           dictionary of data to post to server(unencoded string, the library encodes the post it self) 
     2785    http_query: 
     2786           dictionary of query to send to server(unencoded string, the library encodes the query it self) 
     2787    http_header: 
     2788           dictionary of http header to add the the http header request 
     2789    header_timeout: 
     2790           socket timeout for receiving header from server(default value set to 30 seconds) 
     2791    content_timeout: 
     2792           socket timeout for receiving content from server(default value set to 30 seconds) 
     2793    httpheader_limit: 
     2794           length limit for when a server sends http header(default value set to 8kb(8192 charactors))  
     2795    httpcontent_limit: 
     2796            limits the the amount of content a server can send to the retrieval  
     2797  
     2798  <Exceptions> 
     2799        HttpUserInputError 
     2800            ->  If given a invalid URL  
     2801            ->  If given a none http protocol server 
     2802            ->  if file like object read is given a negative number or a none int as a limit 
     2803            ->  if the file like object is called after it is closed 
     2804 
     2805        HttpConnectionError 
     2806            ->  if opening connection with server fails       
     2807            ->  if sending http request to http server fails   
     2808 
     2809        HttpHeaderReceivingError 
     2810            ->  If the timeout(default timeout set to 5 seconds) for receiving exceeds    
     2811            ->  If the http header length exceeds (default  
     2812             
     2813        HttpHeaderFormatError 
     2814            ->  If The http header is too long(default set to 8 kb(8192 charactors)) 
     2815            ->  If the http header statuscode format is invalid. The right http header 
     2816                    status code includes HTTP<version> http_status_number http_status_msg                  
     2817            ->  If the http server gives a http header with a invalid content length 
     2818                    or invalid redirect location 
     2819             
     2820        HttpContentReceivingError: 
     2821            ->  if the http server fails to send http content  
     2822            ->  if the timeout(default timeout set to 5 seconds) for receiving exceeds 
     2823            ->  if the server socket connection fails for any reason besides connection closing 
     2824                    during receiving http content 
     2825                         
     2826        HttpContentLengthError: 
     2827            ->  if content length exceeds(default set to 4096 kb(4194304 charactors))  
     2828                    (ONLY WORKS CONTENT LENGTH IS GIVEN BY THE HTTP SERVER) 
     2829            ->  If the total received length is not the same as the content length(this check will fail 
     2830                     if the content length isnt given) 
     2831            ->  If read is called with limits and it returns a empty string but the total read is                                       
     2832                     not equal to the content length(this check will fail if the content length isnt given)                 
     2833                                             
     2834        HttpStatuscodeError: 
     2835            -> if the http response status code isnt ok or redirect, it will raise an exception 
     2836                   depending up on the http protocol status number   
     2837         
     2838 
     2839  <Side Effects> 
     2840    None  
     2841 
     2842  <Returns> 
     2843    Returns file like obj which can read the http content from http web server.  
     2844  """ 
     2845   
     2846  # check if url is valid and get host, path, port and query from the given url 
     2847  (host, port, path, url_query) = _httpretrieve_parse_given_url(url) 
     2848 
     2849  # get connection to the http web server 
     2850  try: 
     2851    sock = timeout_openconn(host, port) 
     2852     
     2853  except Exception, e: 
     2854    raise HttpConnectionError('Error: opening a connection failed with given http server, Given: ' + str(url) + ' ' + str(e)) 
     2855     
     2856  # build a http format request using the given port, host, path and query 
     2857  httpheader = _httpretrieve_buildhttprequest(http_header, port, host, path, url_query, http_query, http_post) 
     2858   
     2859  # send http format request to http web server   
     2860  _httpretrieve_sendhttprequest(sock, httpheader)  
     2861   
     2862  # receive the http header lines in a form of list from the http web server 
     2863  httpheaderlines = _httpretrieve_receive_httpheader(sock, header_timeout, httpheader_limit) 
     2864   
     2865  # get the http status number and http status msg from http header response 
     2866  (http_status_number, http_status_msg) = _httpretrieve_get_httpstatuscode(httpheaderlines) 
     2867 
     2868  if http_status_number == '200':# ok message 
     2869    # gets the content length if given. 
     2870    contentlength = _httpretrieve_get_contentlength(httpheaderlines) 
     2871    # return a filelikeobj to read the http content from the http server  
     2872    return _httpretrieve_filelikeobject(sock, contentlength, httpcontent_limit, content_timeout)  
     2873   
     2874  elif http_status_number == '301' or http_status_number == '302': # redirect 
     2875    # redirect to the new location via recursion   
     2876    sock.close() 
     2877    # get the redirection location 
     2878    redirect_location = _httpretrieve_httpredirect(httpheaderlines)  
     2879    # redirect to the new location using recursion 
     2880    return httpretrieve_open(redirect_location)      
     2881 
     2882  else: 
     2883    # if given receive content length inorder to check http content error is received fully 
     2884    contentlength = _httpretrieve_get_contentlength(httpheaderlines) 
     2885    # receive the http content error  
     2886    http_errorcontent = _httpretrieve_receive_httperror_content(sock, contentlength) 
     2887    # raise exception depending up on the http status number and add on the http error content after a 
     2888    # discription that says 'Http error content: ' 
     2889    _httpretrieve_raise_httpstatuscode_error(http_status_number, http_status_msg, http_errorcontent)    
     2890 
     2891 
     2892   
     2893 
     2894 
     2895def httpretrieve_save_file(url, filename, http_query=None, http_post=None, http_header=None, header_timeout=30, content_timeout=30, httpheader_limit=8192, httpcontent_limit=4194304): 
     2896  """ 
     2897  <Purpose> 
     2898     Saves http content of the given URL to current directory   
     2899 
     2900  <Arguments> 
     2901    url: 
     2902           String of a http web server URL 
     2903    filename: 
     2904           The file name for the http content to be saved in 
     2905    http_post: 
     2906           dictionary of data to post to server(unencoded string, the library encodes the post it self) 
     2907    http_query: 
     2908           dictionary of query to send to server(unencoded string, the library encodes the query it self) 
     2909    http_header: 
     2910           dictionary of http header to add the the http header request 
     2911    header_timeout: 
     2912           socket timeout for receiving header from server(default value set to 30 seconds) 
     2913    content_timeout: 
     2914           socket timeout for receiving content from server(default value set to 30 seconds) 
     2915    httpheader_limit: 
     2916           length limit for when a server sends http header(default value set to 8kb(8192 charactors))  
     2917    httpcontent_limit: 
     2918            limits the the amount of content a server can send to the retrieval  
     2919            
     2920  <Exceptions> 
     2921   
     2922    HttpRetrieveClientError: cant create a file to save the http content too 
     2923    Also includes:  all the exception from httpretrieve_open  
     2924 
     2925  <Side Effects> 
     2926    same as httpretrieve_open   
     2927 
     2928  <Returns> 
     2929    None  
     2930  """ 
     2931   
     2932  httpcontent = '' 
     2933  try: 
     2934    # create a new file with the given filename 
     2935    newfile = open(filename, 'w') 
     2936  except Exception, e: 
     2937    raise HttpRetrieveClientError('Error on creating a file to saving http content' + str(e)) 
     2938 
     2939  http_obj = httpretrieve_open(url, http_query, http_post, http_header, header_timeout, content_timeout, httpheader_limit, httpcontent_limit) 
     2940 
     2941  # keep on reading 1024 and writing it into a file, until it receives an empty string 
     2942  # which means the http server content is completely read  
     2943  while True: 
     2944    httpcontent = http_obj.read(1024) 
     2945    if httpcontent == '': 
     2946      # done reading close file and file like obj and exit loop  
     2947      newfile.close()   
     2948      http_obj.close() 
     2949      break 
     2950    newfile.write(httpcontent) 
     2951 
     2952 
     2953   
     2954def httpretrieve_get_string(url, http_query=None, http_post=None, http_header=None, header_timeout=30, content_timeout=30, httpheader_limit=8192, httpcontent_limit=4194304): 
     2955  """ 
     2956  <Purpose> 
     2957     retruns string of the http content from a given URL 
     2958 
     2959  <Arguments> 
     2960    url: 
     2961           String of a http web server-URL 
     2962    http_post: 
     2963           dictionary of data to post to server(unencoded string, the library encodes the post it self) 
     2964    http_query: 
     2965           dictionary of query to send to server(unencoded string, the library encodes the query it self) 
     2966    http_header: 
     2967           dictionary of http header to add the the http header request 
     2968    header_timeout: 
     2969           socket timeout for receiving header from server(default value set to 30 seconds) 
     2970    content_timeout: 
     2971           socket timeout for receiving content from server(default value set to 30 seconds) 
     2972    httpheader_limit: 
     2973           length limit for when a server sends http header(default value set to 8kb(8192 charactors))  
     2974    httpcontent_limit: 
     2975            limits the the amount of content a server can send to the retrieval  
     2976  
     2977  <Exceptions> 
     2978     same as httpretrieve_open 
     2979 
     2980  <Side Effects> 
     2981     same as httpretrieve_open  
     2982 
     2983  <Returns> 
     2984     returns a string of the http content.  
     2985  """        
     2986   
     2987  http_obj = httpretrieve_open(url, http_query, http_post, http_header, header_timeout, content_timeout, httpheader_limit, httpcontent_limit) 
     2988  # retrieve the http content from server using httpretrieve file like object 
     2989  httpcontent = http_obj.read() 
     2990  http_obj.close() 
     2991  # return the http content in a form of string 
     2992  return httpcontent 
     2993 
     2994 
     2995 
     2996class _httpretrieve_filelikeobject: 
     2997  # file like object used to receive the http content with a length limit    
     2998  def __init__(self, sock, contentlength, httpcontent_limit, content_timeout): 
     2999    self.sock = sock 
     3000    if contentlength == None: 
     3001      self.contentlengthisknown = False 
     3002    else: 
     3003      self.contentlengthisknown = True 
     3004      self.contentlength = contentlength 
     3005    self.httpcontent_limit = httpcontent_limit   
     3006    self.content_timeout = content_timeout   
     3007    self.fileobjclosed = False 
     3008    self.totalcontentisreceived = False  
     3009    self.totalread = 0 
     3010     
     3011 
     3012  def read(self, limit = None): 
     3013    """ 
     3014    <Purpose> 
     3015        reads the http content from http server using the file like object    
     3016 
     3017    <Arguments> 
     3018        limit(optional): 
     3019             maximum number of bytes to read. If not specified the whole file is read. 
     3020             Can be 0 or any positive int 
     3021    
     3022    <Exceptions> 
     3023 
     3024        HttpContentReceivingError: 
     3025            ->  if the http server fails to send http content  
     3026            ->  if the timeout(default timeout set to 5 seconds) for receiving exceeds 
     3027            ->  if the server socket connection fails for any reason besides connection closing 
     3028                    during receiving http content 
     3029        HttpContentLengthError: 
     3030            ->  if content length exceeds(default set to 4096 kb(4194304 charactors)) 
     3031                    (ONLY WORKS CONTENT LENGTH IS GIVEN BY THE HTTP SERVER) 
     3032            ->  If the total received length is not the same as the content length(this check will fail 
     3033                     if the content length isnt given) 
     3034            ->  If read is called with limits and it returns a empty string but the total read is                                       
     3035                     not equal to the content length(this check will fail if the content length isnt given) 
     3036                      
     3037        HttpStatuscodeError: 
     3038            if the http response status code isnt ok or redirect, it will raise an exception 
     3039            depending up on the http protocol status number   
     3040 
     3041 
     3042    <Side Effects> 
     3043       None 
     3044 
     3045 
     3046    <Returns> 
     3047      returns the content of http server in a form of string or an empty string if the content is completely read. 
     3048    """ 
     3049     
     3050    # raises an exception, if read is called after the filelikeobj is closed  
     3051    if self.fileobjclosed == True: 
     3052      raise HttpUserInputError('Http Error: filelikeobj is closed') 
     3053 
     3054    # if read is called after all http content received return an empty string 
     3055    if self.totalcontentisreceived: 
     3056      return '' 
     3057    
     3058    # check if limit is given 
     3059    if limit == None:  
     3060      readhaslimit = False  
     3061      left_to_read = 1024 
     3062    else:  
     3063    # check if limit is a valid number 
     3064      if not type(limit) == int: 
     3065        raise HttpUserInputError('User input Error: given a none int to receive' + str(e))   
     3066      elif limit < 0: 
     3067        # raise an exception if limit is a negative number 
     3068        raise HttpUserInputError('User input Error: given a negative number to receive, given: ' + str(limit)) 
     3069      readhaslimit = True 
     3070      left_to_read = limit   
     3071 
     3072    # set a timeout for receiveing content from server  
     3073    self.sock.settimeout(self.content_timeout) 
     3074 
     3075    # if limit is given it will receiveby subtracting what is left until the limit is reached  
     3076    # if limit isnt given it will receive1024 until the server closes connection        
     3077    httpcontent = '' 
     3078    while True: 
     3079      try: 
     3080        content = self.sock.recv(left_to_read) 
     3081 
     3082      except SocketTimeoutError: 
     3083        # raise an exception if receiveis taking too long to respond 
     3084        self.sock.close()   
     3085        raise HttpContentReceivingError('Timeout Error on receiving content: server taking too long to send content')   
     3086 
     3087      except Exception , e: 
     3088        # socket closed - signal for when the server is done sending content 
     3089        # if there is any other exceptions close connection and raise an error   
     3090        if 'Socket closed' not in str(e):  
     3091          self.sock.close()             
     3092          raise HttpContentReceivingError('Error on receiving content:' + str(e)) 
     3093        self.totalcontentisreceived = True 
     3094        break 
     3095 
     3096      else: 
     3097        # By default, httpretrieve permits content length to be less than 4,096 kilobytes(4194304 charactors) 
     3098        if len(content) >= self.httpcontent_limit: 
     3099          raise HttpContentLengthError('content length exceeded ' + self.httpcontent_limit) 
     3100 
     3101        # add what is received 
     3102        httpcontent += content 
     3103        if readhaslimit: 
     3104          # keep subtracting what is left to receieve until it reachs the given limit amount 
     3105          self.totalread += len(content) 
     3106          if len(content) == left_to_read: 
     3107            break 
     3108          else: 
     3109            left_to_read -= len(content) 
     3110 
     3111    # check if there was an error during reciving http content 
     3112    self._check_recieving_error(readhaslimit, httpcontent) 
     3113 
     3114    return httpcontent 
     3115   
     3116 
     3117  def close(self): 
     3118    """ 
     3119    <Purpose> 
     3120      close the file like object  
     3121 
     3122    <Arguments> 
     3123      None 
     3124    
     3125    <Exceptions> 
     3126      None  
     3127 
     3128    <Side Effects> 
     3129      closes socket connection for the http client to http server 
     3130 
     3131    <Returns> 
     3132      Nothing  
     3133    """    
     3134    self.fileobjclosed = True# flag used to raise an exception if the file like object is called after closed 
     3135    self.sock.close() 
     3136 
     3137 
     3138  def _check_recieving_error(self, readhaslimit, httpcontent): 
     3139    if len(httpcontent) == 0: 
     3140      self.sock.close() 
     3141      raise HttpContentLengthError('Error on recieving content: received a http header but didnt receive any http content') 
     3142     
     3143    if self.contentlengthisknown:                     
     3144      # if limit is given and content length is given and total content is received, check the total read equals the content length     
     3145      if readhaslimit and self.totalcontentisreceived: 
     3146        if self.totalread != self.contentlength: 
     3147          self.sock.close()                 
     3148          raise HttpContentLengthError('Total length read with limit did not match the content length: total read: ' + str(self.totalread) + ' content length: ' + str(self.contentlength)) 
     3149 
     3150      # if called read without limit and content length is given; check if the received length is the same as the content length        
     3151      if readhaslimit == False: 
     3152        if len(httpcontent) != self.contentlength: 
     3153          self.sock.close() 
     3154          raise HttpContentLengthError('Total received length did not match the content length: received: ' + str(len(httpcontent)) + ' content length : ' + str(self.contentlength))      
     3155 
     3156 
     3157 
     3158 
     3159 
     3160def _httpretrieve_parse_given_url(url): 
     3161  # checks if the URL is in the right format and returns a string of host, port, path and query by parsing the URL   
     3162  try: 
     3163   # returns a dictionary of {scheme, netloc, path, quer, fragment, username, password, hostname and port} form the url                      
     3164    urlparse = urlparse_urlsplit(url)   
     3165  except Exception, e: 
     3166    raise HttpUserInputError('Given URL error: ' + str(e)) 
     3167  else: 
     3168    # check if the protocol is http  
     3169    if urlparse['scheme'] != 'http': 
     3170      raise HttpUserInputError('Given URL error: the given protocol ' + urlparse['scheme'] + ' isnt supported')        
     3171    if urlparse['hostname'] == None: 
     3172      raise HttpUserInputError('Given URL error: host name is not given')  
     3173 
     3174    # get only the host path, port, query from the urlparse dictionary 
     3175    host = urlparse['hostname'] 
     3176    path = urlparse['path'] 
     3177    query = urlparse['query'] 
     3178     
     3179    # use default port 80 if the port isnt given                     
     3180    if urlparse['port'] == None: 
     3181      port = 80  
     3182    else: 
     3183      port = urlparse['port'] 
     3184 
     3185    return host, port, path, query 
     3186 
     3187 
     3188 
     3189 
     3190 
     3191def _httpretrieve_buildhttprequest(http_header, port, host, path, url_query, dict_query, http_post): 
     3192  # send http request to the http web server using socket connection   
     3193   
     3194  if http_post != None: 
     3195    # there is a posted data, thus use http POST command 
     3196    
     3197    # check if the given post data is valid 
     3198    if not type(http_post) == dict: 
     3199      raise HttpUserInputError('The given http_post is not a dictionary, given: ' + str(type(http_post))) 
     3200 
     3201    # change the given http post dictionary into a encoded post data with a key and val   
     3202    try:  
     3203      http_post = urllib_quote_parameters(http_post) 
     3204    except Exception, e: 
     3205      raise HttpUserInputError('Error encoding the given http post dictionary ' +  str(http_post) + str(e)) 
     3206 
     3207 
     3208    # build the main http request header which includes the GET/POST and the Host name field 
     3209    httpheader = _httpretrieve_httprequestmain_header('POST', url_query, dict_query, path, host, port) 
     3210 
     3211    # if given add a client http header to the request 
     3212    httpheader += _httpretrieve_parse_clienthttpheader(http_header) 
     3213 
     3214    # indicate the http post content length 
     3215    httpheader += 'Content-Length: ' + str(len(http_post)) + '\r\n'   
     3216    # add a new line to indicate that the http header is done and the http post is followed. 
     3217    httpheader += '\r\n' 
     3218    # include the posted data after the http header empty line 
     3219    httpheader += http_post 
     3220         
     3221 
     3222  else: 
     3223    # there is no posted data, use http GET method    
     3224    httpheader = _httpretrieve_httprequestmain_header('GET', url_query, dict_query, path, host, port) 
     3225    # if http header is given, add client headers to the request  
     3226    httpheader += _httpretrieve_parse_clienthttpheader(http_header) 
     3227    # add a new line to indicate that the header request is complete 
     3228    httpheader += '\r\n' 
     3229 
     3230  # return header with a new line which is signal for http header is done  
     3231  return httpheader  
     3232 
     3233 
     3234 
     3235 
     3236def _httpretrieve_httprequestmain_header(http_command, url_query, dict_query, path, host, port): 
     3237  # builds the first two main http request headers which include the GET/POST and the HOST name 
     3238   
     3239  # before building the httprequest make sure there isnt two fields of query given by the client 
     3240  if url_query != '' and dict_query != None: 
     3241    # cant have two different fields with query 
     3242    raise HttpUserInputError('Cant input a http query with the url and an extra parameter dictionary with a http query') 
     3243 
     3244  elif dict_query != None: 
     3245    # user has given a http query  
     3246    try:  
     3247      encoded_query = '?' + urllib_quote_parameters(dict_query) 
     3248    except Exception, e: 
     3249      raise HttpUserInputError('Error encoding the given http query dictionary ' +  str(dict_query) + str(e)) 
     3250 
     3251  elif url_query != '': 
     3252    # if there is a query include the query on the main header('?' is used as a seperater between path) 
     3253    encoded_query = '?' + url_query 
     3254  else: 
     3255    # there is no query 
     3256    encoded_query = '' 
     3257     
     3258 
     3259  # if there is no path add a '/' on the request and if there is a path use the given path 
     3260  addpath = '/' 
     3261  if path != '': 
     3262    addpath = path 
     3263 
     3264  # FIRST header which includes the POST/GET request   
     3265  main_httpheader = http_command + ' ' + addpath + encoded_query + ' HTTP/1.0\r\n' 
     3266 
     3267 
     3268  # if port is 80, dont need to include the port upon request 
     3269  addport = '' 
     3270  if port != 80: 
     3271    # if the port is not 80 the host needs to include the port number on Host header 
     3272    # (':' is used as a separater between host and port) 
     3273    addport = ':' + str(port) 
     3274 
     3275  # SECOND line of the header request which include the host name with port if the port is not 80    
     3276  main_httpheader += 'Host: ' + host + addport + '\r\n' 
     3277 
     3278  # return the firs two lines of the http request  
     3279  return main_httpheader 
     3280 
     3281 
     3282 
     3283 
     3284def _httpretrieve_parse_clienthttpheader(http_header): 
     3285  # builds a http header from the given http header dictionary 
     3286  if http_header == None: 
     3287    # if the http header isnt given return a empty string 
     3288    return '' 
     3289  elif not type(http_header) == dict: 
     3290    # raise an exception if the http header isnt dictionary 
     3291    raise HttpUserInputError('The given http header is not a dictionary, given: ' + str(type(http_header))) 
     3292  else:  
     3293    # take the given key and val from the http_header dictionary and add them to the http header with 
     3294    # correct http format 
     3295    clienthttpheader = '' 
     3296    for key, val in http_header.items(): 
     3297      # make simple checks on the key and val  
     3298      if not type(key) == str: 
     3299        raise HttpUserInputError('The given http header key isnt str given ' + str(type(key))) 
     3300      if not type(val) == str: 
     3301        raise HttpUserInputError('The given http header value isnt str given ' + str( type(val))) 
     3302      if key == '' or val == '': 
     3303        # raise an exception if the key or value is a empty field 
     3304        raise HttpUserInputError('The given empty http header feild of key or value') 
     3305      if key.capitalize() != key: 
     3306        raise HttpUserInputError('The given http header key is not capitalized, given: ' + key) 
     3307 
     3308      # add the key and value to the http header field 
     3309      clienthttpheader += key + ' : ' + val + '\r\n'   
     3310 
     3311    # return the string of the http header   
     3312    return clienthttpheader 
     3313 
     3314 
     3315 
     3316 
     3317def _httpretrieve_sendhttprequest(sock, httpheader): 
     3318  # send the request, and if there is any error raise an excetion 
     3319  try: 
     3320    sock.send(httpheader) 
     3321  except Exception, e: 
     3322    if 'Socket closed' not in str(e):  
     3323      sock.close() 
     3324    raise HttpConnectionError('Connection error: on sending http request to server ' + str(e)) 
     3325 
     3326 
     3327 
     3328   
     3329def _httpretrieve_receive_httpheader(sock, header_timeout, httpheader_limit): 
     3330  # receives the http header leaving alone rest of the http response and returns in list 
     3331  # of each header as a line. default httpheader limit is set to 8 kb                         
     3332 
     3333  # set a time out if the server fails to send http header  
     3334  sock.settimeout(header_timeout) 
     3335 
     3336  httpheader_received = 0  
     3337  httpheader = ''  
     3338  while True: 
     3339    # receive until a empty line (\n\n or \r\n\r\n ) which separates the 
     3340    # http header from the http content 
     3341    if '\r\n\r\n' in httpheader: 
     3342      # return split to return a list of httpheader lines 
     3343      return httpheader.split('\r\n') 
     3344    if '\n\n' in httpheader: 
     3345      # return split to return a list of httpheader lines 
     3346      return httpheader.split('\n') 
     3347 
     3348    if httpheader_limit == httpheader_received: 
     3349      sock.close()                   
     3350      raise HttpHeaderFormatError('Http header length Error: The http header is too long, exceeded 8 kb') 
     3351                         
     3352    try: 
     3353      # receive one character at a time inorder to check for the empty line 
     3354      content = sock.recv(1) 
     3355      # keep track of the received characters to raise an exception if the limit is exceeded                   
     3356      httpheader_received += 1                   
     3357 
     3358    except SocketTimeoutError: 
     3359      raise HttpHeaderReceivingError('Timeout Error on receiving header: server taking too long to send http header') 
     3360    except Exception, e: 
     3361      sock.close()  
     3362      raise HttpHeaderReceivingError('Error on recieving http header: ' + str(e)) 
     3363    else: 
     3364      # if there was not receiving error add keep on adding the receieved content 
     3365      httpheader += content 
     3366 
     3367 
     3368 
     3369 
     3370def _httpretrieve_get_httpstatuscode(httpHeaderLines):  
     3371  # checks if the http status code is valid and return the status number and msg  
     3372 
     3373  # http response header includes 3 "words": HTTP<version> http_status_number http_status_msg  
     3374  httpstatusheader = httpHeaderLines[0] 
     3375  headersplit = httpstatusheader.split(' ', 2) 
     3376 
     3377  # length of the header has to be 3 or greater because depending up on the http_status_msg might be more than one word 
     3378  if len(headersplit) != 3: 
     3379    raise HttpHeaderFormatError('Invalid Http header status code format: Correct format is HTTP<version> http_status_number http_status_msg: Given '  + httpstatusheader) 
     3380  if not httpstatusheader.startswith('HTTP'): 
     3381    raise HttpHeaderFormatError('Invalid Http header status code format: Http header status code should start of with HTTP<version> but given: '  + httpstatusheader) 
     3382 
     3383  # the first split is the http version 
     3384  http_version = headersplit[0]                       
     3385 
     3386  # check if http_status_number is valid int 
     3387  try:  
     3388    int(headersplit[1]) 
     3389  except ValueError, e: 
     3390    raise HttpHeaderFormatError('Invalid Http header status code format: Status number should be a int, Given: ' + str(headersplit[1]) + str(e)) 
     3391  else: 
     3392    http_status_number = headersplit[1] 
     3393   
     3394  # what ever is left is the http status msg 
     3395  http_status_msg = headersplit[2] 
     3396   
     3397  # return the values 
     3398  return http_status_number, http_status_msg 
     3399 
     3400 
     3401 
     3402 
     3403def _httpretrieve_receive_httperror_content(sock, contentlength): 
     3404  # receives the http error content which is located after the http error header                         
     3405  httperror_content = ''  
     3406  while True: 
     3407    try: 
     3408      content = sock.recv(1024)                   
     3409 
     3410    except SocketTimeoutError: 
     3411      raise HttpContentReceivingError('Timeout Error on receiving http error conent: server taking too long to send http error content') 
     3412    except Exception, e: 
     3413      # socket closed - signal for when the server is done sending error content 
     3414      # if there is any other exceptions close connection and raise an error besides socket closing   
     3415      if 'Socket closed' not in str(e):  
     3416        sock.close()             
     3417        raise HttpContentReceivingError('Error on receiving http error content: ' + str(e)) 
     3418      break 
     3419         
     3420    else: 
     3421      # if there was not a receiving error keep on adding the receieved content 
     3422      httperror_content += content 
     3423 
     3424  # return the received http error content. If the content length is given check 
     3425  # if the content length maches the received content 
     3426  if contentlength != None: 
     3427    if contentlength != len(httperror_content): 
     3428      raise HttpContentLengthError('Error on receiving http error content: received conent length: ' + str(len(httperror_content)) + ' actual content length: ' + str(contentlength))    
     3429  return httperror_content       
     3430 
     3431 
     3432 
     3433     
     3434def _httpretrieve_raise_httpstatuscode_error(http_status_number, http_status_msg, http_errorcontent):  
     3435  # raises an exception using the http_status_number 1xx for Informational, 2xx for Success 3xx for Redirection, 
     3436  # 4xx for Client Error, and 5xx Server Error 
     3437  
     3438  # raise a detailed error message explaining the http_status_number and http_status_msg for popular http errors   
     3439  if http_status_number == '202': 
     3440    raise HttpError202('Http response error: ' + http_status_number + ' ' + http_status_msg +  ' http proccesing not responding. Http error content: ' + http_errorcontent)       
     3441  elif http_status_number == '204': 
     3442    raise HttpError204('Http response error: ' + http_status_number + ' ' + http_status_msg +  ' thier is no http body content. Http error content: ' + http_errorcontent)  
     3443  elif http_status_number == '300': 
     3444    raise HttpError300('Http response error: ' + http_status_number + ' ' + http_status_msg +  ' multiple redirect isnt suported. Http error content: ' + http_errorcontent)                 
     3445  elif http_status_number == '404': 
     3446    raise HttpError404('Http response error: ' + http_status_number + ' ' + http_status_msg +  ' cant find anything matching the given url. Http error content: ' + http_errorcontent)                 
     3447  elif http_status_number == '403': 
     3448    raise HttpError403('Http response error: ' + http_status_number + ' ' + http_status_msg +  ' the request was illegal. Http error content: ' + http_errorcontent)                 
     3449  elif http_status_number == '400': 
     3450    raise HttpError400('Http response error: ' + http_status_number + ' ' + http_status_msg +  ' the request contians bad syntex. Http error content: ' + http_errorcontent)                 
     3451  elif http_status_number == '500': 
     3452    raise HttpError500('Http response error: ' + http_status_number + ' ' + http_status_msg +  ' The server encountered an unexpected condition. Http error content: ' + http_errorcontent)                  
     3453  elif http_status_number == '502': 
     3454    raise HttpError502('Http response error: ' + http_status_number + ' ' + http_status_msg +  ' acting like a gateway received an invalid response. Http error content: ' + http_errorcontent)                 
     3455 
     3456  # if the http number wasnt any of the popular http error msgs, raise an exception using 
     3457  # the defualt http status number with http status msg   
     3458  elif http_status_number >= '100' and http_status_number < '200':  
     3459    raise HttpError1xx('Http response error: Information ' + http_status_number + ' ' + http_status_msg + '.Http error content: ' + http_errorcontent)  
     3460  elif http_status_number > '200' and http_status_number < '300':  
     3461    raise HttpError2xx('Http response error: success error ' + http_status_number + ' ' + http_status_msg + '.Http error content: ' + http_errorcontent) 
     3462  elif http_status_number >= '300' and http_status_number < '400':  
     3463    raise HttpError3xx('Http response error: Redirection error' + http_status_number + ' ' + http_status_msg + '.Http error content: ' + http_errorcontent) 
     3464  elif http_status_number >= '400' and http_status_number < '500':  
     3465    raise HttpError4xx('Http response error: client error ' + http_status_number + ' ' + http_status_msg + '.Http error content: ' + http_errorcontent)   
     3466  elif http_status_number >= '500' and http_status_number < '600':  
     3467    raise HttpError5xx('Http response error: server error: ' + http_status_number + ' ' + http_status_msg + '.Http error content: ' + http_errorcontent) 
     3468  else: 
     3469    raise HttpStatusCodeError('Http response error: invalid http status response, given ' + http_status_number + '.Http error content: ' + http_errorcontent)   
     3470 
     3471 
     3472 
     3473 
     3474def _httpretrieve_httpredirect(httpheaderlines): 
     3475  # given http header retruns the redirect location    
     3476 
     3477  # if there is a redirect location given by the server it will look like 
     3478  # eg.'Location: http://www.google.com' 
     3479  for headerline in httpheaderlines: 
     3480    if headerline.startswith('Location:'): 
     3481      # if found redirect need to strip out 'Location: ' to return the url    
     3482      redirect = headerline[len('Location: '):] 
     3483                         
     3484      # check if the redirect has given a location then return it 
     3485      if len(redirect) == 0: 
     3486        raise HttpHeaderFormatError('Http header redirection format error: http server gave a redierect location with no URL') 
     3487      return redirect 
     3488                         
     3489  # there wasn't a redirect location given 
     3490  raise HttpHeaderFormatError('Http header redirection format error: http redirect header didnt include the location')   
     3491 
     3492 
     3493 
     3494 
     3495def _httpretrieve_get_contentlength(httpheaderlines): 
     3496  # returns the content legth if given or returns None if not given by server 
     3497 
     3498  # if there is a content length given by the server it will look like 
     3499  # eg.'Content-Length: 34' 
     3500  for headerline in httpheaderlines: 
     3501    if headerline.startswith('Content-Length:'): 
     3502      # if found content length need to strip out 'Content-Length: ' to return the length    
     3503       
     3504      try:  
     3505        contentlength = int(headerline[len('Content-Length: '):]) 
     3506      except ValueError, e: 
     3507        raise HttpHeaderFormatError('Http header Content-Length format error: http server provided content length that isnt a int ' + str(e))                 
     3508 
     3509       
     3510      # check if the content length is valid and retrun it  
     3511      if contentlength <= 0:                 
     3512        raise HttpHeaderFormatError('Http header Content-Length format error: provided content length with invalid number ' + str(contentlength)) 
     3513      else: 
     3514        return contentlength 
     3515                         
     3516  # there wasn't a content-length line or the content length was given but didnt give a int  
     3517  return None 
     3518 
     3519 
     3520 
     3521 
     3522   
     3523def urllib_quote(string, safe="/"): 
     3524  """ 
     3525  <Purpose> 
     3526    Encode a string such that it can be used safely in a URL or XML 
     3527    document. 
     3528 
     3529  <Arguments> 
     3530    string: 
     3531           The string to urlencode. 
     3532 
     3533    safe (optional): 
     3534           Specifies additional characters that should not be quoted -- 
     3535           defaults to "/". 
     3536 
     3537  <Exceptions> 
     3538    TypeError if the safe parameter isn't an enumerable. 
     3539 
     3540  <Side Effects> 
     3541    None. 
     3542 
     3543  <Returns> 
     3544    Urlencoded version of the passed string. 
     3545  """ 
     3546 
     3547  resultstr = "" 
     3548 
     3549  # We go through each character in the string; if it's not in [0-9a-zA-Z] 
     3550  # we wrap it. 
     3551 
     3552  safeset = set(safe) 
     3553 
     3554  for char in string: 
     3555    asciicode = ord(char) 
     3556    if (asciicode >= ord("0") and asciicode <= ord("9")) or \ 
     3557        (asciicode >= ord("A") and asciicode <= ord("Z")) or \ 
     3558        (asciicode >= ord("a") and asciicode <= ord("z")) or \ 
     3559        asciicode == ord("_") or asciicode == ord(".") or \ 
     3560        asciicode == ord("-") or char in safeset: 
     3561      resultstr += char 
     3562    else: 
     3563      resultstr += "%%%02X" % asciicode 
     3564 
     3565  return resultstr 
     3566 
     3567 
     3568 
     3569 
     3570def urllib_quote_plus(string, safe=""): 
     3571  """ 
     3572  <Purpose> 
     3573    Encode a string to go in the query fragment of a URL. 
     3574 
     3575  <Arguments> 
     3576    string: 
     3577           The string to urlencode. 
     3578 
     3579    safe (optional): 
     3580           Specifies additional characters that should not be quoted -- 
     3581           defaults to the empty string. 
     3582 
     3583  <Exceptions> 
     3584    TypeError if the safe parameter isn't a string. 
     3585 
     3586  <Side Effects> 
     3587    None. 
     3588 
     3589  <Returns> 
     3590    Urlencoded version of the passed string. 
     3591  """ 
     3592 
     3593  return urllib_quote(string, safe + " ").replace(" ", "+") 
     3594 
     3595 
     3596 
     3597 
     3598def urllib_unquote(string): 
     3599  """ 
     3600  <Purpose> 
     3601    Unquote a urlencoded string. 
     3602 
     3603  <Arguments> 
     3604    string: 
     3605           The string to unquote. 
     3606 
     3607  <Exceptions> 
     3608    ValueError thrown if the last wrapped octet isn't a valid wrapped octet 
     3609    (i.e. if the string ends in "%" or "%x" rather than "%xx". Also throws 
     3610    ValueError if the nibbles aren't valid hex digits. 
     3611 
     3612  <Side Effects> 
     3613    None. 
     3614 
     3615  <Returns> 
     3616    The decoded string. 
     3617  """ 
     3618 
     3619  resultstr = "" 
     3620 
     3621  # We go through the string from end to beginning, looking for wrapped 
     3622  # octets. When one is found we add it (unwrapped) and the following 
     3623  # string to the resultant string, and shorten the original string. 
     3624 
     3625  while True: 
     3626    lastpercentlocation = string.rfind("%") 
     3627    if lastpercentlocation < 0: 
     3628      break 
     3629 
     3630    wrappedoctetstr = string[lastpercentlocation+1:lastpercentlocation+3] 
     3631    if len(wrappedoctetstr) != 2: 
     3632      raise ValueError("Quoted string is poorly formed") 
     3633 
     3634    resultstr = \ 
     3635        chr(int(wrappedoctetstr, 16)) + \ 
     3636        string[lastpercentlocation+3:] + \ 
     3637        resultstr 
     3638    string = string[:lastpercentlocation] 
     3639 
     3640  resultstr = string + resultstr 
     3641  return resultstr 
     3642 
     3643 
     3644 
     3645 
     3646def urllib_unquote_plus(string): 
     3647  """ 
     3648  <Purpose> 
     3649    Unquote the urlencoded query fragment of a URL. 
     3650 
     3651  <Arguments> 
     3652    string: 
     3653           The string to unquote. 
     3654 
     3655  <Exceptions> 
     3656    ValueError thrown if the last wrapped octet isn't a valid wrapped octet 
     3657    (i.e. if the string ends in "%" or "%x" rather than "%xx". Also throws 
     3658    ValueError if the nibbles aren't valid hex digits. 
     3659 
     3660  <Side Effects> 
     3661    None. 
     3662 
     3663  <Returns> 
     3664    The decoded string. 
     3665  """ 
     3666 
     3667  return urllib_unquote(string.replace("+", " ")) 
     3668 
     3669 
     3670 
     3671 
     3672def urllib_quote_parameters(dictionary): 
     3673  """ 
     3674  <Purpose> 
     3675    Encode a dictionary of (key, value) pairs into an HTTP query string or 
     3676    POST body (same form). 
     3677 
     3678  <Arguments> 
     3679    dictionary: 
     3680           The dictionary to quote. 
     3681 
     3682  <Exceptions> 
     3683    None. 
     3684 
     3685  <Side Effects> 
     3686    None. 
     3687 
     3688  <Returns> 
     3689    The quoted dictionary. 
     3690  """ 
     3691 
     3692  quoted_keyvals = [] 
     3693  for key, val in dictionary.items(): 
     3694    quoted_keyvals.append("%s=%s" % (urllib_quote(key), urllib_quote(val))) 
     3695 
     3696  return "&".join(quoted_keyvals) 
     3697 
     3698 
     3699 
     3700 
     3701def urllib_unquote_parameters(string): 
     3702  """ 
     3703  <Purpose> 
     3704    Decode a urlencoded query string or POST body. 
     3705 
     3706  <Arguments> 
     3707    string: 
     3708           The string to decode. 
     3709 
     3710  <Exceptions> 
     3711    ValueError if the string is poorly formed. 
     3712 
     3713  <Side Effects> 
     3714    None. 
     3715 
     3716  <Returns> 
     3717    A dictionary mapping keys to values. 
     3718  """ 
     3719 
     3720  keyvalpairs = string.split("&") 
     3721  res = {} 
     3722 
     3723  for quotedkeyval in keyvalpairs: 
     3724    # Throw ValueError if there is more or less than one '='. 
     3725    quotedkey, quotedval = quotedkeyval.split("=") 
     3726    key = urllib_unquote(quotedkey) 
     3727    val = urllib_unquote(quotedval) 
     3728    res[key] = val 
     3729 
     3730  return res 
     3731 
     3732 
     3733 
     3734 
     3735def server_test_content(httprequest_dictionary, http_query, http_post): 
    153736  # temp server that sends the clients posted data as the http content    
    163737       
    173738  # return the content to check if the httpretrieve gets the same content  
    18   return httprequest_dictionary['posted_data'] 
     3739  return [httprequest_dictionary['posted_data'], None] 
    193740   
    203741   
     
    293750    server_handle = registerhttpcallback('http://127.0.0.1:12345', server_test_content) 
    303751  except Exception, e: 
    31     stop_registerhttpcallback(server_handle) 
    323752    raise Exception('Server failed internally ' + str(e))   
    333753 
  • seattle/trunk/seattlelib/tests/httpretrieve_test/ut_httpretrieve_read_done.py

    r3253 r3314  
    1212 
    1313 
    14 def server_test_read(httprequest_dictionary): 
     14def server_test_read(httprequest_dictionary, http_query, http_post): 
    1515  # for this test the server should just act normal and send a http content 
    16   return 'normal server'       
     16  return ['normal server', None]       
    1717 
    1818 
  • seattle/trunk/seattlelib/tests/httpretrieve_test/ut_httpretrieve_read_limit.py

    r3253 r3314  
    1212 
    1313 
    14 def server_test_read(httprequest_dictionary): 
     14def server_test_read(httprequest_dictionary, http_query, http_post): 
    1515  # for this test the server should just act normal and send a http content 
    16   return 'normal server' 
     16  return ['normal server', None] 
    1717 
    1818 
  • seattle/trunk/seattlelib/tests/httpretrieve_test/ut_httpretrieve_statuscode_errors.py

    r3253 r3314  
    1414 
    1515 
    16 def server_test_errormsg(httprequest_dictionary): 
     16def server_test_errormsg(httprequest_dictionary, http_query, http_post): 
    1717  # raises an exception depending up on the change_error_numb(which is used as a signal 
    1818  # for the count on the httpretrieve request)  
     
    4040  # make multiple requests to server using httpretrieve to see if it raises an exception for different 
    4141  # http error responses messages 
     42  else: 
     43    # used to change the error msg after each exception 
     44    mycontext['change_error_numb'] = 0 
     45     
     46    while True: 
     47      # change the error number   
     48      mycontext['change_error_numb'] += 1 
     49      try: 
     50        # should raise an exception depending up on the change_error_numb 
     51        recv_msg = httpretrieve_get_string('http://127.0.0.1:12345/') 
    4252 
    43   # used to change the error msg after each exception 
    44   mycontext['change_error_numb'] = 0 
    45    
    46   while True: 
    47     # change the error number   
    48     mycontext['change_error_numb'] += 1 
    49     try: 
    50       # should raise an exception depending up on the change_error_numb 
    51       recv_msg = httpretrieve_get_string('http://127.0.0.1:12345/') 
     53      # catch the right exception depending up on the change_error_numb 
     54      except HttpError202: 
     55        if mycontext['change_error_numb'] != 1: 
     56          print 'Failed: raised a HttpError202 when the server didnt send a HttpError202'  
     57          break 
     58        pass 
     59      except HttpError500: 
     60        if mycontext['change_error_numb'] != 2: 
     61          print 'Failed: raised a HttpError500 when the server didnt send a HttpError500' 
     62          break 
     63        pass 
     64      except HttpError400: 
     65        if mycontext['change_error_numb'] != 3: 
     66          print 'Failed: raised a HttpError400 when the server didnt send a HttpError400' 
     67          break 
     68        pass 
     69      except HttpError404: 
     70        if mycontext['change_error_numb'] != 4: 
     71          print 'Failed: raised a HttpError404 when the server didnt send a HttpError404' 
     72        # last exception break the loop  
     73        break 
     74      except Exception, e: 
     75        # print a filed msg if the server raises a different exception 
     76        print 'Failed: didnt raise right ecxeption ' + str(e)    
     77        break 
    5278 
    53     # catch the right exception depending up on the change_error_numb 
    54     except HttpError202: 
    55       if mycontext['change_error_numb'] != 1: 
    56         print 'Failed: raised a HttpError202 when the server didnt send a HttpError202'  
    57         break 
    58       pass 
    59     except HttpError500: 
    60       if mycontext['change_error_numb'] != 2: 
    61         print 'Failed: raised a HttpError500 when the server didnt send a HttpError500' 
    62         break 
    63       pass 
    64     except HttpError400: 
    65       if mycontext['change_error_numb'] != 3: 
    66         print 'Failed: raised a HttpError400 when the server didnt send a HttpError400' 
    67         break 
    68       pass 
    69     except HttpError404: 
    70       if mycontext['change_error_numb'] != 4: 
    71         print 'Failed: raised a HttpError404 when the server didnt send a HttpError404' 
    72       # last exception break the loop  
    73       break 
    74     except Exception, e: 
    75       # print a filed msg if the server raises a different exception 
    76       print 'Failed: didnt raise right ecxeption'    
    77       break 
     79      # if any one of the errors excute without raising an exception, print failed msg      
     80      else: 
     81        print 'Failed: should have raised an exception because the server sent a http error' 
    7882 
    79     # if any one of the errors excute without raising an exception, print failed msg      
    80     else: 
    81       print 'Failed: should have raised an exception because the server sent a http error' 
     83    # stop the server    
     84    stop_registerhttpcallback(handle)    
    8285 
    83   # stop the server    
    84   stop_registerhttpcallback(handle)    
    85  
    86        
     86