Changeset 3331

Show
Ignore:
Timestamp:
01/05/10 15:49:05 (10 years ago)
Author:
yemuru
Message:

Update test

Files:
1 modified

Legend:

Unmodified
Added
Removed
  • seattle/trunk/seattlelib/tests/httpretrieve_test/ut_httpretrieve_post.py

    r3314 r3331  
    66 
    77 
    8 """ 
    9 <Program Name> 
    10   registerhttpcallback.repy 
    11  
    12 <Started> 
    13   July 29, 2009 
    14  
    15 <Author> 
    16   Yafete Yemuru 
    17  
    18 <Purpose> 
    19   This program is Web server that a user can use without understanding http protocol. 
    20   Given a URL this program acts like a web server that places a call back callbackfunc when a 
    21   client(web browser) makes a connection to a given URL. The call back function is meant to 
    22   modify a dynamic page using query and posted data. Once the call back return the modified 
    23   version of the website the registerhttpcallback will complete the process by sending the 
    24   web site content to the client(web browser). 
    25  
    26 """ 
    27  
    28  
    29 #begin include urllib.repy 
    30 def urllib_quote(string, safe="/"): 
    31   """ 
    32   <Purpose> 
    33     Encode a string such that it can be used safely in a URL or XML 
    34     document. 
    35  
    36   <Arguments> 
    37     string: 
    38            The string to urlencode. 
    39  
    40     safe (optional): 
    41            Specifies additional characters that should not be quoted -- 
    42            defaults to "/". 
    43  
    44   <Exceptions> 
    45     TypeError if the safe parameter isn't an enumerable. 
    46  
    47   <Side Effects> 
    48     None. 
    49  
    50   <Returns> 
    51     Urlencoded version of the passed string. 
    52   """ 
    53  
    54   resultstr = "" 
    55  
    56   # We go through each character in the string; if it's not in [0-9a-zA-Z] 
    57   # we wrap it. 
    58  
    59   safeset = set(safe) 
    60  
    61   for char in string: 
    62     asciicode = ord(char) 
    63     if (asciicode >= ord("0") and asciicode <= ord("9")) or \ 
    64         (asciicode >= ord("A") and asciicode <= ord("Z")) or \ 
    65         (asciicode >= ord("a") and asciicode <= ord("z")) or \ 
    66         asciicode == ord("_") or asciicode == ord(".") or \ 
    67         asciicode == ord("-") or char in safeset: 
    68       resultstr += char 
    69     else: 
    70       resultstr += "%%%02X" % asciicode 
    71  
    72   return resultstr 
    73  
    74  
    75  
    76  
    77 def urllib_quote_plus(string, safe=""): 
    78   """ 
    79   <Purpose> 
    80     Encode a string to go in the query fragment of a URL. 
    81  
    82   <Arguments> 
    83     string: 
    84            The string to urlencode. 
    85  
    86     safe (optional): 
    87            Specifies additional characters that should not be quoted -- 
    88            defaults to the empty string. 
    89  
    90   <Exceptions> 
    91     TypeError if the safe parameter isn't a string. 
    92  
    93   <Side Effects> 
    94     None. 
    95  
    96   <Returns> 
    97     Urlencoded version of the passed string. 
    98   """ 
    99  
    100   return urllib_quote(string, safe + " ").replace(" ", "+") 
    101  
    102  
    103  
    104  
    105 def urllib_unquote(string): 
    106   """ 
    107   <Purpose> 
    108     Unquote a urlencoded string. 
    109  
    110   <Arguments> 
    111     string: 
    112            The string to unquote. 
    113  
    114   <Exceptions> 
    115     ValueError thrown if the last wrapped octet isn't a valid wrapped octet 
    116     (i.e. if the string ends in "%" or "%x" rather than "%xx". Also throws 
    117     ValueError if the nibbles aren't valid hex digits. 
    118  
    119   <Side Effects> 
    120     None. 
    121  
    122   <Returns> 
    123     The decoded string. 
    124   """ 
    125  
    126   resultstr = "" 
    127  
    128   # We go through the string from end to beginning, looking for wrapped 
    129   # octets. When one is found we add it (unwrapped) and the following 
    130   # string to the resultant string, and shorten the original string. 
    131  
    132   while True: 
    133     lastpercentlocation = string.rfind("%") 
    134     if lastpercentlocation < 0: 
    135       break 
    136  
    137     wrappedoctetstr = string[lastpercentlocation+1:lastpercentlocation+3] 
    138     if len(wrappedoctetstr) != 2: 
    139       raise ValueError("Quoted string is poorly formed") 
    140  
    141     resultstr = \ 
    142         chr(int(wrappedoctetstr, 16)) + \ 
    143         string[lastpercentlocation+3:] + \ 
    144         resultstr 
    145     string = string[:lastpercentlocation] 
    146  
    147   resultstr = string + resultstr 
    148   return resultstr 
    149  
    150  
    151  
    152  
    153 def urllib_unquote_plus(string): 
    154   """ 
    155   <Purpose> 
    156     Unquote the urlencoded query fragment of a URL. 
    157  
    158   <Arguments> 
    159     string: 
    160            The string to unquote. 
    161  
    162   <Exceptions> 
    163     ValueError thrown if the last wrapped octet isn't a valid wrapped octet 
    164     (i.e. if the string ends in "%" or "%x" rather than "%xx". Also throws 
    165     ValueError if the nibbles aren't valid hex digits. 
    166  
    167   <Side Effects> 
    168     None. 
    169  
    170   <Returns> 
    171     The decoded string. 
    172   """ 
    173  
    174   return urllib_unquote(string.replace("+", " ")) 
    175  
    176  
    177  
    178  
    179 def urllib_quote_parameters(dictionary): 
    180   """ 
    181   <Purpose> 
    182     Encode a dictionary of (key, value) pairs into an HTTP query string or 
    183     POST body (same form). 
    184  
    185   <Arguments> 
    186     dictionary: 
    187            The dictionary to quote. 
    188  
    189   <Exceptions> 
    190     None. 
    191  
    192   <Side Effects> 
    193     None. 
    194  
    195   <Returns> 
    196     The quoted dictionary. 
    197   """ 
    198  
    199   quoted_keyvals = [] 
    200   for key, val in dictionary.items(): 
    201     quoted_keyvals.append("%s=%s" % (urllib_quote(key), urllib_quote(val))) 
    202  
    203   return "&".join(quoted_keyvals) 
    204  
    205  
    206  
    207  
    208 def urllib_unquote_parameters(string): 
    209   """ 
    210   <Purpose> 
    211     Decode a urlencoded query string or POST body. 
    212  
    213   <Arguments> 
    214     string: 
    215            The string to decode. 
    216  
    217   <Exceptions> 
    218     ValueError if the string is poorly formed. 
    219  
    220   <Side Effects> 
    221     None. 
    222  
    223   <Returns> 
    224     A dictionary mapping keys to values. 
    225   """ 
    226  
    227   keyvalpairs = string.split("&") 
    228   res = {} 
    229  
    230   for quotedkeyval in keyvalpairs: 
    231     # Throw ValueError if there is more or less than one '='. 
    232     quotedkey, quotedval = quotedkeyval.split("=") 
    233     key = urllib_unquote(quotedkey) 
    234     val = urllib_unquote(quotedval) 
    235     res[key] = val 
    236  
    237   return res 
    238  
    239 #end include urllib.repy 
    240 #begin include urlparse.repy   
    241 """ 
    242 <Program Name> 
    243   urlparse.repy 
    244  
    245 <Started> 
    246   May 15, 2009 
    247  
    248 <Author> 
    249   Michael Phan-Ba 
    250  
    251 <Purpose> 
    252   Provides utilities for parsing URLs, based on the Python 2.6.1 module urlparse. 
    253  
    254 """ 
    255  
    256  
    257 def urlparse_urlsplit(urlstring, default_scheme="", allow_fragments=True): 
    258   """ 
    259   <Purpose> 
    260     Parse a URL into five components, returning a dictionary.  This corresponds 
    261     to the general structure of a URL: 
    262     scheme://netloc/path;parameters?query#fragment.  The parameters are not 
    263     split from the URL and individual componenets are not separated. 
    264  
    265     Only absolute server-based URIs are currently supported (all URLs will be 
    266     parsed into the components listed, regardless of the scheme). 
    267  
    268   <Arguments> 
    269     default_scheme: 
    270       Optional: defaults to the empty string.  If specified, gives the default 
    271       addressing scheme, to be used only if the URL does not specify one. 
    272  
    273     allow_fragments: 
    274       Optional: defaults to True.  If False, fragment identifiers are not 
    275       allowed, even if the URL's addressing scheme normally does support them. 
    276  
    277   <Exceptions> 
    278     ValueError on parsing a non-numeric port value. 
    279  
    280   <Side Effects> 
    281     None. 
    282  
    283   <Returns> 
    284     A dictionary containing: 
    285  
    286     Key         Value                               Value if not present 
    287     ============================================================================ 
    288     scheme      URL scheme specifier                empty string 
    289     netloc      Network location part               empty string 
    290     path        Hierarchical path                   empty string 
    291     query       Query component                     empty string 
    292     fragment    Fragment identifier                 empty string 
    293     username    User name                           None 
    294     password    Password                            None 
    295     hostname    Host name (lower case)              None 
    296     port        Port number as integer, if present  None 
    297  
    298   """ 
    299  
    300   components = {"scheme": default_scheme, "netloc": "", "path": "", "query": "", 
    301     "fragment": "", "username": None, "password": None, "hostname": None, 
    302     "port": None } 
    303  
    304   # Extract the scheme, if present. 
    305   (lpart, rpart) = _urlparse_splitscheme(urlstring) 
    306   if lpart: 
    307     components["scheme"] = lpart 
    308  
    309   # Extract the server information, if present. 
    310   if rpart.startswith("//"): 
    311     (lpart, rpart) = _urlparse_splitnetloc(rpart, 2) 
    312     components["netloc"] = lpart 
    313  
    314     (components["username"], components["password"], components["hostname"], 
    315       components["port"]) = _urlparse_splitauthority(lpart) 
    316  
    317   # Extract the fragment. 
    318   if allow_fragments: 
    319     (rpart, components["fragment"]) = _urlparse_splitfragment(rpart) 
    320  
    321  
    322   # Extract the query. 
    323   (components["path"], components["query"]) = _urlparse_splitquery(rpart) 
    324  
    325   return components 
    326  
    327  
    328 def _urlparse_splitscheme(url): 
    329   """Parse the scheme portion of the URL""" 
    330   # The scheme is valid only if it contains these characters. 
    331   scheme_chars = \ 
    332     "abcdefghijklmnopqrstuvwxyz0123456789+-." 
    333  
    334   scheme = "" 
    335   rest = url 
    336  
    337   spart = url.split(":", 1) 
    338   if len(spart) == 2: 
    339  
    340     # Normalize the scheme. 
    341     spart[0] = spart[0].lower() 
    342  
    343     # A scheme is valid only if it starts with an alpha character. 
    344     if spart[0] and spart[0][0].isalpha(): 
    345       for char in spart[0]: 
    346         if char not in scheme_chars: 
    347           break 
    348       (scheme, rest) = spart 
    349  
    350   return scheme, rest 
    351  
    352  
    353 def _urlparse_splitnetloc(url, start=0): 
    354   """Parse the netloc portion of the URL""" 
    355  
    356   # By default, the netloc is delimited by the end of the URL. 
    357   delim = len(url) 
    358  
    359   # Find the left-most delimiter. 
    360   for char in "/?#": 
    361     xdelim = url.find(char, start) 
    362     if xdelim >= 0: 
    363       delim = min(delim, xdelim) 
    364  
    365   # Return the netloc and the rest of the URL. 
    366   return url[start:delim], url[delim:] 
    367  
    368  
    369 def _urlparse_splitauthority(netloc): 
    370   """Parse the authority portion of the netloc""" 
    371  
    372   # The authority can have a userinfo portion delimited by "@". 
    373   authority = netloc.split("@", 1) 
    374  
    375   # Default values. 
    376   username = None 
    377   password = None 
    378   hostname = None 
    379   port = None 
    380  
    381   # Is there a userinfo portion? 
    382   if len(authority) == 2: 
    383  
    384     # userinfo can be split into username:password 
    385     userinfo = authority[0].split(":", 1) 
    386  
    387     # hostport can be split into hostname:port 
    388     hostport = authority[1].split(":", 1) 
    389  
    390     if userinfo[0]: 
    391       username = userinfo[0] 
    392     if len(userinfo) == 2: 
    393       password = userinfo[1] 
    394  
    395   # No userinfo portion found. 
    396   else: 
    397  
    398     # hostport can be split into hostname:port 
    399     hostport = netloc.split(":", 1) 
    400  
    401   # Is there a port value? 
    402   if hostport[0]: 
    403     hostname = hostport[0] 
    404   if len(hostport) == 2: 
    405     port = int(hostport[1], 10) 
    406  
    407   # Return the values. 
    408   return username, password, hostname, port 
    409  
    410  
    411 def _urlparse_splitquery(url): 
    412   """Parse the query portion of the url""" 
    413  
    414   qpart = url.split("?", 1) 
    415   if len(qpart) == 2: 
    416     query = qpart[1] 
    417   else: 
    418     query = "" 
    419  
    420   return qpart[0], query 
    421  
    422  
    423 def _urlparse_splitfragment(url): 
    424   """Parse the query portion of the url""" 
    425  
    426   fpart = url.split("#", 1) 
    427   if len(fpart) == 2: 
    428     fragment = fpart[1] 
    429   else: 
    430     fragment = "" 
    431  
    432   return fpart[0], fragment 
    433  
    434 #end include urlparse.repy   
    435 #begin include sockettimeout.repy 
    436 """ 
    437 <Description> 
    438   Puts back in Python's non-blocking functionality. 
    439  
    440   send(): 
    441     Raises SocketTimeout Error if the send call lasts 
    442     longer than the set timeout. 
    443  
    444   recv(): 
    445     Guarentees the receipt of a message.   Raises SocketTimeoutError if it does not 
    446     receive any message before a given timeout. 
    447     If actually receives the message, returns the message and continues. 
    448  
    449 <Usage> 
    450   Text-replacable for Repy Sockets: 
    451     timeout_openconn(desthost, destport, localip=None, localport=None, timeout = 5) 
    452     timeout_waitforconn(localip, localport, function) 
    453  
    454   Object: 
    455     sockobj.settimeout(seconds) 
    456     sockobj.send(data) 
    457     sockobj.recv(bytes) 
    458     sockobj.close() 
    459  
    460 <Date> 
    461   Sun Mar  1 10:27:35 PST 2009 
    462  
    463 <Example> 
    464   # hello world 
    465   include sockettimer.repy 
    466  
    467   def callback(ip, port, timeout_sockobj, commhandle, listenhandle): 
    468     hw_message = timeout_sockobj.recv(1047) 
    469  
    470     # cleanup 
    471     stopcomm(commhandle) 
    472     stopcomm(listenhandle) 
    473     timeout_sockobj.close() 
    474  
    475     print hw_message # => "hello world!" 
    476    
    477   def server(): 
    478     sockobj = timeout_waitforconn(getmyip(), 12345, callback) 
    479  
    480   def client(): 
    481     sockobj = timeout_openconn(getmyip(), 12345) 
    482     sockobj.send("hello world!") 
    483  
    484   def main(): 
    485     server() 
    486     client() 
    487     exitall() 
    488  
    489   if callfunc == 'initialize': 
    490     main()  
    491 """ 
    492  
    493 class SocketTimeoutError(Exception): 
    494   """The socket timed out before receiving a response""" 
    495  
    496 def timeout_openconn(desthost, destport, localip=None, localport=None, timeout = 5): 
    497   """ 
    498   <Purpose>  
    499     Wrapper for Repy like socket interface 
    500  
    501   <Args> 
    502     Same as Repy openconn 
    503  
    504   <Exception> 
    505     Timeout exception if the dest address doesnt respond. 
    506  
    507   <Returns> 
    508     socket obj on success 
    509   """ 
    510  
    511   tsock = TimeoutSocket() 
    512   tsock.settimeout(timeout) 
    513   if localip and localport: 
    514     tsock.bind((localip, localport)) 
    515   tsock.connect((desthost, destport)) 
    516   return tsock 
    517  
    518 def timeout_waitforconn(localip, localport, function): 
    519   """ 
    520   <Purpose>  
    521     Wrapper for Repy like socket interface 
    522  
    523   <Args> 
    524     Same as Repy waitforconn 
    525  
    526   <Side Effects> 
    527     Sets up event listener which calls function on messages. 
    528  
    529   <Returns> 
    530     Handle to listener. 
    531   """ 
    532  
    533   tsock = TimeoutSocket() 
    534   tsock.bind((localip, localport)) 
    535   tsock.setcallback(function) 
    536   return tsock.listen() 
    537  
    538 class TimeoutSocket: 
    539   """ 
    540   <Purpose> 
    541     Provide an socket object like the Repy usual one. 
    542  
    543   <Side Effects> 
    544     Uses a getlock() to watch for a timeout 
    545     Uses waitforconn and openconn to simulate socket 
    546   """ 
    547  
    548   ################ 
    549   # Constructors 
    550   ################ 
    551  
    552   def __init__(self): 
    553     """ Constructor for socket """ 
    554 #    self.lock = getlock() # general lock BUG: Do we need to lock everything? 
    555     self.timeout_lock = getlock() # special lock for Timeout condition 
    556     self.timeout = 5 # seconds to wait 
    557     self.bytes_sent = None # used to check if send() timed out 
    558  
    559     # user vars    
    560     self.local_address = None # ip, port 
    561     self.remote_address = None # ip, port 
    562     self.callback = None # the user's function to call 
    563  
    564     # repy socket vars 
    565     self.sockobj = None #  the Repy socket 
    566     self.commhandle = None # the current comm 
    567     self.listencommhandle = None # the listener comm 
    568  
    569     #error tracking vars 
    570      
    571     #if any exceptions are thrown in the separate thread executing _send_and_release,  
    572     #they are caught and stored in this variable, then raised in _send_or_close 
    573     self.TCPSendError = None 
    574  
    575   ################ 
    576   # Mutator methods 
    577   ################# 
    578  
    579   def settimeout(self, value): 
    580     """ Setter for timeout""" 
    581     self.timeout = value 
    582  
    583   def setcallback(self, function): 
    584     """ Setter for callback function""" 
    585     self.callback = function 
    586  
    587   #################### 
    588   # Public Methods 
    589   #################### 
    590  
    591   def bind(self, local_address = None): 
    592     """ 
    593     <Purpose> 
    594       Set local address 
    595  
    596     <Args> 
    597       Tuple of (ip, port) local. 
    598     """ 
    599     self.local_address = local_address 
    600  
    601   def listen(self): 
    602     """ 
    603     <Purpose> 
    604       Listen for peer 
    605      
    606     <Side Effects> 
    607       Calls Repy waitforconn() 
    608     """ 
    609     return self._waitforconn() 
    610  
    611   def connect(self, remote_address): 
    612     """ 
    613     <Purpose> 
    614       Connect to peer. 
    615  
    616     <Args> 
    617       Tuple of (ip, port) remote. 
    618     
    619     <Side Effects> 
    620       Calls Repy openconn. 
    621     """ 
    622     self.remote_address = remote_address 
    623     self._openconn() 
    624  
    625   def recv(self, maxLen): # timeout as optional arg ??? 
    626     """ 
    627     <Purpose> 
    628       If it fails to finish within the timeout, I close the socket and raise a 
    629       TimeoutError exception. I.e. if there's no message, we call it an error 
    630       and raise it. 
    631        
    632     <Arguments> 
    633       maxLen - bytes to recv 
    634  
    635     <Exception> 
    636       Raises TimeoutError exception if the recv times out 
    637       without receiving a message. 
    638  
    639     <Side Effects> 
    640       Closes the connection if times out. 
    641  
    642     <Returns> 
    643       The message. 
    644     """ 
    645     return self._recv_or_close(maxLen) 
    646  
    647   def send(self, data): 
    648     """ 
    649     <Purpose> 
    650       Just like normal Repy socket.  Sends messages. 
    651        
    652     <Arguments> 
    653       data - the string message 
    654  
    655     <Exception> 
    656       Same as Repy socket. 
    657   
    658     <Returns> 
    659       The bytes sent. 
    660     """ 
    661     return self._send_or_close(data) 
    662  
    663   def close(self): 
    664     self.local_address = None # ip, port 
    665     self.remote_address = None # ip, port 
    666     self.callback = None # the user's function to call 
    667  
    668     if self.sockobj: 
    669       self.sockobj.close() 
    670     self.sockobj = None #  the Repy socket 
    671      
    672     # Armon: As part of the semantics, stopcomm will raise an  
    673     # exception given an invalid handle, e.g. None. Thus, 
    674     # we need to check for this. 
    675     if self.commhandle:  
    676       stopcomm(self.commhandle) 
    677       self.commhandle = None # the current comm 
    678      
    679     # Armon: Same as above. 
    680     if self.listencommhandle: 
    681       stopcomm(self.listencommhandle) 
    682       self.listencommhandle = None # the listener comm 
    683  
    684  
    685   ######################## 
    686   # Private 
    687   ######################### 
    688  
    689   def _openconn(self): 
    690     """Handle current state variables and call Repy openconn.""" 
    691  
    692     destip, destport = self.remote_address 
    693     if self.local_address: 
    694       srcip, srcport = self.local_address 
    695       self.sockobj = openconn(destip, destport, srcip, srcport, self.timeout) 
    696     else: 
    697       self.sockobj = openconn(destip, destport) 
    698  
    699   def _waitforconn(self): 
    700     """Setup way between Repy waitforconn event""" 
    701     localip, localport = self.local_address 
    702     self.listencommhandle = waitforconn(localip, localport, self._callback) 
    703     return self.listencommhandle 
    704  
    705   def _callback(self, ip, port, sockobj, ch, lh): 
    706     """Pass on through to user callback""" 
    707     self.sockobj = sockobj 
    708     self.listencommhandle = lh # same as the 1st from wait for comm, right? 
    709     self.commhandle = ch # should we care? 
    710      
    711     if not self.remote_address: 
    712       self.remote_address = (ip, port) 
    713     else:  
    714       raise Exception("what! peer does not match?") 
    715  
    716     self.callback(ip, port, self, ch, lh) 
    717  
    718   def _send(self, data): 
    719     """Send data""" 
    720     return self.sockobj.send(data) 
    721  
    722   def _recv(self, maxLen): 
    723     """Recv data of length maxLen""" 
    724     return self.sockobj.recv(maxLen) 
    725  
    726   def _send_and_release(self, data): 
    727     """Send data then release the timeout lock""" 
    728      
    729     #Bug Fix (Cosmin): exceptions thrown in separate thread _send_and_release could not be caught 
    730     #now we store the exception if it is thrown and raise it back in send_or_close 
    731     try: 
    732       self.bytes_sent = self._send(data) 
    733     except Exception, e: 
    734       self.TCPSendError = e 
    735      
    736     self._quietly_release() # release the lock 
    737   
    738   def _quietly_release(self): 
    739     """Release the timeout lock and ignore if already released""" 
    740     try: 
    741       self.timeout_lock.release() 
    742     except: 
    743       pass 
    744     
    745   def _send_or_close(self, data): 
    746     """Raise the Timeout Error if no receipt.  Keep track by timeout_lock.""" 
    747  
    748     # acquire the lock, when it's release we'll carry on 
    749     self.timeout_lock.acquire() 
    750  
    751     # fork off a lock that'll release the lock at the timeout 
    752     timerhandle = settimer(self.timeout, self._quietly_release, ()) 
    753  
    754     # fork off a send call so we can raise the exception in the main thread 
    755     # the send call will also release our lock 
    756     settimer(0, self._send_and_release, (data,)) 
    757  
    758     # block until either the timeout or _send finishes 
    759     self.timeout_lock.acquire() 
    760     self.timeout_lock.release() 
    761  
    762     if self.bytes_sent: # send finished 
    763       canceltimer(timerhandle) 
    764       retdata = self.bytes_sent 
    765       self.bytes_sent = None 
    766       return retdata 
    767     elif self.TCPSendError != None: 
    768       #Bug Fix (Cosmin): exceptions thrown in separate thread _send_and_release could not be caught 
    769        
    770       #we got an error within the separate thread that performed the send operation 
    771       exception_to_throw = self.TCPSendError 
    772       self.TCPSendError = None 
    773       raise exception_to_throw 
    774     else: # it timed out 
    775       self.close() 
    776       raise SocketTimeoutError 
    777  
    778   def _recv_or_close(self, amount): 
    779     """Raise the Timeout Error if no receipt.  Keep track by timeout_lock.""" 
    780     timerhandle = settimer(self.timeout, self._clobbersocket, ()) 
    781     try: 
    782       retdata = self._recv(amount) 
    783     except Exception, e: 
    784       # if it's not the timeout, reraise... 
    785       if self.timeout_lock.acquire(False): 
    786         raise 
    787       raise SocketTimeoutError 
    788      
    789     # I acquired the lock, I should stop the timer because I succeeded... 
    790     if self.timeout_lock.acquire(False): 
    791       # even if this isn't in time, the lock prevents a race condition  
    792       # this is merely an optimization to prevent the timer from ever firing... 
    793       canceltimer(timerhandle) 
    794       self.timeout_lock.release() # Alper's bug 3/10/09 
    795       return retdata 
    796     else: 
    797       raise SocketTimeoutError 
    798  
    799   def _clobbersocket(self): 
    800     """If I can acquire the lock without blocking, then close the socket to abort""" 
    801     if self.timeout_lock.acquire(False): 
    802       self.close() 
    803  
    804  
    805 ############################ 
    806 # Deprecated functions 
    807 ############################## 
    808  
    809 # private function... 
    810 def sockettimeout_clobbersocket(sockobj,mylock): 
    811   # if I can acquire the lock without blocking, then close the socket to abort 
    812   if mylock.acquire(False): 
    813     sockobj.close() 
    814  
    815 # if it fails to finish within the timeout, I close the socket and raise a 
    816 # SocketTimeout exception... 
    817 def sockettimeout_recv_or_close(sockobj, amount, timeout): 
    818   # A lock I'll use for this attempt 
    819   mylock = getlock() 
    820   timerhandle = settimer(timeout,clobbersocket, (sockobj, mylock)) 
    821   try: 
    822     retdata = sockobj.recv(amount) 
    823   except Exception, e: 
    824     # if it's not the timeout, reraise... 
    825     if mylock.acquire(False): 
    826       raise 
    827     raise SocketTimeout 
    828      
    829   # I acquired the lock, I should stop the timer because I succeeded... 
    830   if mylock.acquire(False): 
    831     # even if this isn't in time, the lock prevents a race condition  
    832     # this is merely an optimization to prevent the timer from ever firing... 
    833     canceltimer(timerhandle) 
    834     return retdata 
    835   else: 
    836     raise SocketTimeout 
    837  
    838  
    839 #end include sockettimeout.repy 
    840 # used for hierarchy exception  
    841 #begin include http_hierarchy_error.repy 
    842 """ 
    843 <Program Name> 
    844   http_hierarchy_error.repy 
    845  
    846 <Started> 
    847   Oct 05, 2009 
    848  
    849 <Author> 
    850   Yafete Yemuru 
    851  
    852 <Purpose> 
    853   provides a hierachy http error using status code including client and server errors 
    854   classes.   
    855 """ 
    856  
    857  
    858 ''' 
    859 http hierarchy error exception classes 
    860  
    861 -> HttpError 
    862    -> HttpRetrieveClientError 
    863       -> HttpUserInputError 
    864       -> HttpConnectionError 
    865  
    866    -> HttpServerError 
    867       -> HttpResponseError 
    868          -> HttpHeaderError 
    869             -> HttpHeaderReceivingError 
    870             -> HttpHeaderFormatError 
    871          -> HttpContentError 
    872             -> HttpContentReceivingError 
    873             -> HttpContentLengthError 
    874         
    875    -> HttpStatuscodeError 
    876      -> HttpError1xx 
    877         -> followed by all http status code error number HttpError(number) 
    878          
    879      -> HttpError2xx 
    880         -> followed by all http status code error number HttpError(number) 
    881          
    882      -> HttpError3xx 
    883         -> followed by all http status code error number HttpError(number) 
    884  
    885      -> HttpError4xx 
    886         -> followed by all http status code error number HttpError(number) 
    887          
    888      -> HttpError5xx 
    889         -> followed by all http status code error number HttpError(number) 
    890  
    891 ''' 
    892  
    893 class HttpError(Exception): 
    894   pass 
    895  
    896 # raises an exception for http client error  
    897 class HttpRetrieveClientError(HttpError):# extend HttpError  
    898   pass 
    899 class HttpUserInputError(HttpRetrieveClientError): 
    900   pass 
    901 class HttpConnectionError(HttpRetrieveClientError): 
    902   pass 
    903  
    904  
    905 # raises an exception for any http server failure   
    906 class HttpServerError(HttpError):# extend HttpError  
    907   pass 
    908 class HttpResponseError(HttpServerError): 
    909   pass 
    910 class HttpHeaderError(HttpResponseError): 
    911   pass 
    912 class HttpHeaderReceivingError(HttpHeaderError): 
    913   pass 
    914 class HttpHeaderFormatError(HttpHeaderError): 
    915   pass 
    916 class HttpContentError(HttpResponseError): 
    917   pass 
    918 class HttpContentReceivingError(HttpContentError): 
    919   pass 
    920 class HttpContentLengthError(HttpContentError): 
    921   pass 
    922  
    923  
    924 class HttpStatusCodeError(HttpError):# extend HttpError 
    925   pass 
    926 class HttpError1xx(HttpStatusCodeError): 
    927   pass 
    928 class HttpError100(HttpError1xx): 
    929   pass 
    930 class HttpError101(HttpError1xx):  
    931   pass 
    932 class HttpError102(HttpError1xx):  
    933   pass 
    934  
    935  
    936 class HttpError2xx(HttpStatusCodeError): 
    937   pass 
    938 class HttpError201(HttpError2xx):  
    939   pass 
    940 class HttpError202(HttpError2xx):    
    941   pass 
    942 class HttpError203(HttpError2xx):  
    943   pass 
    944 class HttpError204(HttpError2xx):  
    945   pass 
    946 class HttpError205(HttpError2xx):  
    947   pass 
    948 class HttpError206(HttpError2xx):  
    949   pass 
    950 class HttpError207(HttpError2xx):  
    951   pass 
    952 class HttpError226(HttpError2xx):  
    953   pass                  
    954  
    955  
    956 class HttpError3xx(HttpStatusCodeError): 
    957   pass 
    958 class HttpError300(HttpError3xx):  
    959   pass 
    960 class HttpError301(HttpError3xx):  
    961   pass 
    962 class HttpError302(HttpError3xx):  
    963   pass 
    964 class HttpError303(HttpError3xx):  
    965   pass 
    966 class HttpError304(HttpError3xx): 
    967   pass 
    968 class HttpError305(HttpError3xx):  
    969   pass 
    970 class HttpError306(HttpError3xx):  
    971   pass 
    972 class HttpError307(HttpError3xx):  
    973   pass 
    974                      
    975  
    976 class HttpError4xx(HttpStatusCodeError): 
    977   pass 
    978 class HttpError400(HttpError4xx): 
    979   pass   
    980 class HttpError401(HttpError4xx):  
    981   pass 
    982 class HttpError402(HttpError4xx):  
    983   pass 
    984 class HttpError403(HttpError4xx): 
    985   pass   
    986 class HttpError404(HttpError4xx):  
    987   pass 
    988 class HttpError405(HttpError4xx):  
    989   pass 
    990 class HttpError406(HttpError4xx):  
    991   pass 
    992 class HttpError407(HttpError4xx):  
    993   pass 
    994 class HttpError408(HttpError4xx):  
    995   pass 
    996 class HttpError409(HttpError4xx):  
    997   pass 
    998 class HttpError410(HttpError4xx):  
    999   pass 
    1000 class HttpError411(HttpError4xx):  
    1001   pass 
    1002 class HttpError412(HttpError4xx):  
    1003   pass 
    1004 class HttpError413(HttpError4xx):  
    1005   pass 
    1006 class HttpError414(HttpError4xx):  
    1007   pass 
    1008 class HttpError415(HttpError4xx):  
    1009   pass 
    1010 class HttpError416(HttpError4xx):  
    1011   pass 
    1012 class HttpError417(HttpError4xx):  
    1013   pass 
    1014 class HttpError418(HttpError4xx):  
    1015   pass 
    1016 class HttpError422(HttpError4xx):  
    1017   pass 
    1018 class HttpError423(HttpError4xx):  
    1019   pass 
    1020 class HttpError424(HttpError4xx):  
    1021   pass 
    1022 class HttpError425(HttpError4xx):  
    1023   pass 
    1024 class HttpError426(HttpError4xx):  
    1025   pass 
    1026  
    1027  
    1028 class HttpError5xx(HttpStatusCodeError): 
    1029   pass 
    1030 class HttpError500(HttpError5xx):  
    1031   pass 
    1032 class HttpError501(HttpError5xx):  
    1033   pass 
    1034 class HttpError502(HttpError5xx):  
    1035   pass 
    1036 class HttpError503(HttpError5xx):  
    1037   pass 
    1038 class HttpError504(HttpError5xx):  
    1039   pass 
    1040 class HttpError505(HttpError5xx):  
    1041   pass 
    1042 class HttpError506(HttpError5xx): 
    1043   pass   
    1044 class HttpError507(HttpError5xx): 
    1045   pass   
    1046 class HttpError510(HttpError5xx): 
    1047   pass 
    1048  
    1049 #end include http_hierarchy_error.repy 
    1050  
    1051  
    1052  
    1053  
    1054  
    1055 def registerhttpcallback(url, callbackfunc, httprequest_limit = 131072, httppost_limit = 2048): 
    1056   """ 
    1057   <Purpose> 
    1058      Waits for a connection to the given url(host). Calls callbackfunc with a argument of http request 
    1059      dictionary up on success.  
    1060    
    1061       
    1062   <Arguments> 
    1063      url: 
    1064            String of a http web server-URL  
    1065  
    1066      callbackfunc: 
    1067             The callbackfunc to be called. It can take up to three argument . 
    1068             First argument- The http request dictionary keys are as follows: 
    1069               http_command - for the http request method GET or POST 
    1070               http_version - which http version the client  used HTTP\1.1 or HTTP\1.0   
    1071               path - the exact request path parsed  
    1072               query - the exact request query parsed  
    1073               posted_data - if there is any posted data returns None if there isnt one 
    1074  
    1075               And all the http requests headers the client provided. eg. Content-Length: 22   
    1076               will incude Content-Length as a key and 22 as a value 
    1077  
    1078             Second argument - httprequest query; this returns a unencoded dictionary of the query 
    1079             Third argument - httprequest posted data; this returns a unencoded dictionary of the posted data 
    1080  
    1081             RESTRICTIONS: 
    1082               -> Follow the http_hierarchy_error(HttpStatusCodeError)to raise an exception. eg. raise 
    1083                   HttpError404 for file not found 
    1084               -> To redirect raise HttpError301(url) or HttpError302(url). The url has to be valid.   
    1085               -> server only excepts tuple type of [httpcontent, httpheader] where the httpcontent is a 
    1086                   string of the content intended to display. httpheader is a dictionary used only if you have a 
    1087                   extra header you want to add on the http response.  
    1088  
    1089      httprequest_limit: 
    1090             -> used to to limit the http request a client can make(default value set to 128kb or 131072 charactors) 
    1091  
    1092      httppost_limit: 
    1093             -> used to to limit the http post a client can make(default value set to 2mb or 2048 charactors)      
    1094  
    1095  
    1096   <Exceptions> 
    1097           HttpConnectionError-> if the server fails on waiting for a connection 
    1098           HttpUserInputError -> if server fails on making a connection to client(web browser) 
    1099           HttpServerError -> If server fails internally 
    1100  
    1101           HttpError408 -> If client(web browser) takes to long to send request   
    1102           HttpError413 -> If the http posted data length is too long 
    1103                        -> If the http request length is too long 
    1104           HttpError417 -> If there is the http request format problem 
    1105                        -> If the content length doesnt match the actual posted data length 
    1106           HttpError501 -> If the given http command is not supported or valid (supported 'GET' and 'POST') 
    1107           HttpError411 -> If the http command is POST and content length is not given 
    1108           HttpError500 -> If server fails internally on sending the http content OR HTTP header,sending error and 
    1109                        -> If the callback fucntion doesnt follow restrictions   
    1110                        -> if the users input for url is not in correct format or unsuported 
    1111                           protocol(this program only supports Http) 
    1112  
    1113  
    1114   <Side Effects> 
    1115      None  
    1116  
    1117   <Returns> 
    1118      A handle to the listener. This can be used to stop the server from waiting for a connection.   
    1119   """   
    1120   def run_webserver(ip, port, sock, thiscommhandle, listencommhandle): 
    1121   # this function is defined inside the registerhttpcallback fuction so callback function name that is 
    1122   # given by a user is placed a call on when there is a connection.  
    1123     try: 
    1124       # receive the client(web browser) request and return a list of the request 
    1125       client_request_lines = _registerhttpcallback_receive_client_request(sock, httprequest_limit) 
    1126  
    1127       # check if the received request meets http requsest format standards and return a 
    1128       # dictionary of the http request headers.  
    1129       httprequest_dictionary = _registerhttpcallback_make_httprequsest_dictionary(client_request_lines)     
    1130  
    1131       # check if there is posted data and add to the httprequest_dictionary with a key posted_data. 
    1132       # returns None if there isnt a posted data 
    1133       _registerhttpcallback_receive_httpposted_data(sock, httprequest_dictionary, httppost_limit) 
    1134        
    1135       # get dictionary decode from the given query  
    1136       httprequest_query = _registerhttpcallback_get_decode_dict(httprequest_dictionary, 'query') 
    1137  
    1138       # get dictionary decode from the given post  
    1139       httprequest_posted_data = _registerhttpcallback_get_decode_dict(httprequest_dictionary, 'posted_data') 
    1140  
    1141       # place the call back callbackfunc with dictionary that includes http_command, 
    1142       # http_version, path, query, posted_data, and all the http requests headers 
    1143       webpage_content = callbackfunc(httprequest_dictionary, httprequest_query, httprequest_posted_data) 
    1144          
    1145       # callback callbackfunc excecuted, send the processed dynamic web page data to client(web browser) 
    1146       _registerhttpcallback_send_httpresponse(sock, webpage_content) 
    1147  
    1148        
    1149     except HttpStatusCodeError, e: 
    1150       # send any error that occur during processing server to the client(web browser) 
    1151       _registerhttpcallback_send_httpformat_error(sock, e) 
    1152        
    1153        
    1154     except Exception, e: 
    1155       # if the program failed to catch an internal error raise an exception and send it to client(web browser) 
    1156       try: 
    1157         raise HttpError500('Server failed internally: ' + str(e)) 
    1158       except Exception, e: 
    1159         _registerhttpcallback_send_httpformat_error(sock, e) 
    1160  
    1161  
    1162  
    1163    
    1164   # get the host and port from the given url to use for connection to listen on 
    1165   (host, port) = _registerhttpcallback_get_host_port(url) 
    1166  
    1167   try: 
    1168     # waits for a client(web browser) to make a connetion and run the web server  
    1169     listencommhandle = waitforconn(host, port, run_webserver) 
    1170  
    1171   except Exception, e: 
    1172     # the waiting for a connection failed, stop waiting and raise an exception  
    1173     stopcomm(listencommhandle) 
    1174     raise HttpConnectionError('Web server failed on waiting for connection ' + str(e)) 
    1175  
    1176   else: 
    1177     # store the listencommhandle, to stop server if needed 
    1178     return listencommhandle 
    1179  
    1180  
    1181  
    1182  
    1183  
    1184  
    1185  
    1186  
    1187 def stop_registerhttpcallback(handle):    
    1188   """ 
    1189     <Purpose> 
    1190           Deregister a callback for a commhandle.  
    1191  
    1192     <Arguments> 
    1193           commhandle: 
    1194               A commhandle as returned by registerhttpcallback. 
    1195  
    1196     <Exceptions> 
    1197           None. 
    1198  
    1199     <Side Effects> 
    1200           This has an undefined effect on a socket-like object if it is currently in use. 
    1201  
    1202     <Returns> 
    1203           Returns True if commhandle was successfully closed, False if the handle cannot be closed  
    1204   """ 
    1205   #close the connection 
    1206   return stopcomm(handle) 
    1207  
    1208     
    1209  
    1210  
    1211    
    1212  
    1213 def _registerhttpcallback_get_host_port(url): 
    1214   # get host and path from the given url 
    1215   try: 
    1216     # returns a dictionary of {scheme, netloc, path, quer, fragment, username, password, hostname and port} form the url 
    1217     urlparse = urlparse_urlsplit(url)   
    1218   except Exception, e: 
    1219     raise HttpUserInputError('Server URL format error:' + str(e)) 
    1220   else: 
    1221     # check if the given url is valid using the url parse  
    1222     if urlparse['scheme'] != 'http': 
    1223       raise HttpUserInputError('The given protocol type isnt suported: Given ' + urlparse['scheme'])        
    1224     if urlparse['hostname'] == None: 
    1225       raise HttpUserInputError('Server URL format error: host name is not given')  
    1226  
    1227     host = urlparse['hostname'] 
    1228      
    1229     # use default port 80 if the port isnt given 
    1230     if urlparse['port'] == None: 
    1231       port = 80 
    1232     else: 
    1233       # if given use the given port 
    1234       port = urlparse['port'] 
    1235  
    1236     return host, port 
    1237  
    1238  
    1239  
    1240        
    1241 def _registerhttpcallback_receive_client_request(sock, httprequest_limit): 
    1242   # receive request from the client using the socket connection 
    1243   if not type(httprequest_limit) == int: 
    1244     # check if the given httprequest limit is valid 
    1245     raise HttpServerError('The given http request limit isnt int ' + str(e)) 
    1246    
    1247    
    1248   # empty line is used as a signal for when the http request is done, and we set a 
    1249   # default request length limit to be 131072 character(128kb) 
    1250   client_request = _registerhttpcallback_receive_untilemptyline(sock, httprequest_limit) 
    1251          
    1252    
    1253   # http request format requires the request to be line by line 
    1254   # build a list of the received request split line by line to check the formating of the request 
    1255   client_request_lines = '' 
    1256   try:  
    1257     # split the entire message line by line  
    1258     client_request_lines = client_request.splitlines() 
    1259   except Exception, e: 
    1260     # raise an exception if the request doenst follow the http protocol format  
    1261     raise HttpError417('Error on the http request format ' + str(e)) 
    1262    
    1263   # the http request has to be at least one line including the http header request 
    1264   if len(client_request_lines) == 0: 
    1265     raise HttpError417('The received request doesnt follow http protocol requirments: Given ' + client_request) 
    1266  
    1267   # returns a list of client request 
    1268   return client_request_lines 
    1269  
    1270  
    1271  
    1272  
    1273  
    1274 def _registerhttpcallback_make_httprequsest_dictionary(client_request_lines): 
    1275   # builds up a dictionary from the received request or raises an exception if the 
    1276   # request format isnt http protocol. The dictionary also includes the http header request 
    1277   # parsed with custome keys (http_command - for http comand methed, path - parsed form 
    1278   # the request url, query - query string parsed form the request url, http_version - 
    1279   # for the http protocol version) 
    1280    
    1281   httpheader_request = True 
    1282   httprequest_dictionary = {} 
    1283    
    1284   for request_line in client_request_lines:   
    1285     if httpheader_request: 
    1286       # acording to the http protocol format the first line is different because it is includes comand url and version 
    1287  
    1288       # check if the http request is valid and parse the http request method, URL and http version 
    1289       (http_command, path, query, http_version) = _registerhttpcallback_httprequest_header_parse(request_line) 
    1290  
    1291       # use custom names for the parsed data of the first request line and add to the dictionary  
    1292       httprequest_dictionary['http_command'] = http_command 
    1293       httprequest_dictionary['http_version'] = http_version 
    1294       httprequest_dictionary['path'] = path 
    1295       httprequest_dictionary['query'] = query   
    1296  
    1297       # used to flag the for the upcoming lines, because they have stationary keys 
    1298       httpheader_request = False  
    1299        
    1300     elif request_line == '': 
    1301       # last line is empty line, return the http dictionary built so far 
    1302       return httprequest_dictionary   
    1303  
    1304     else: 
    1305       # the rest of the lines should be formated 'httpheader_key: httpheader_value'  eg.'Content-Length: 34' 
    1306       try: 
    1307         modified_request = request_line.split(': ') 
    1308       except Exception, e: 
    1309         raise HttpError417('Error on the http request format; Given: ' + request_line + 'in the request line ' + str(e)) 
    1310  
    1311       # raise an exception if the request line doesnt contain 2 contents for 'httpheader_key: httpheader_value'  
    1312       if len(modified_request) != 2: 
    1313         raise HttpError417('Error on the http request format; Given: ' + request_line + 'in the request line') 
    1314  
    1315       httprequest_key = modified_request[0] 
    1316       httprequest_value = modified_request[1] 
    1317        
    1318       httprequest_dictionary[httprequest_key] = httprequest_value 
    1319  
    1320   return httprequest_dictionary 
    1321    
    1322  
    1323  
    1324  
    1325  
    1326 def _registerhttpcallback_get_decode_dict(httprequest_dictionary , decode_type): 
    1327   # returns a decode dictionary of post or query depending up on the encoded style    
    1328  
    1329   # get the data from the request dictionary  
    1330   data_to_decode = httprequest_dictionary[decode_type] 
    1331        
    1332   if decode_type == 'posted_data': 
    1333     # inorder to check if the post is empty use None    
    1334     empty_check = None     
    1335   else: 
    1336     # inorder to check if the query is empty use empty string    
    1337      empty_check = '' 
    1338          
    1339   if data_to_decode != empty_check: 
    1340     try: 
    1341       # decode the given data depending up on the style it was encoded   
    1342       return urllib_unquote_parameters(data_to_decode) 
    1343  
    1344     except Exception, e: 
    1345       # raise an exception if the fails decoding the given data   
    1346       raise HttpUserInputError('Invalid ' + decode_type + ' Raised ' + str(e) + ' on decoding') 
    1347  
    1348   # if the data is empty return None  
    1349   return None     
    1350  
    1351  
    1352  
    1353  
    1354    
    1355 def _registerhttpcallback_httprequest_header_parse(http_header_request): 
    1356   # varifiy's if the http request header format is valid and returns the http command, path, query, 
    1357   # and http version parsed from the http header 
    1358  
    1359   # http request header should include RequestMethod <url> HTTP<version> or RequestMethod HTTP<version> 
    1360   # and is located at the top of the http request 
    1361   try: 
    1362     http_command_url_version = http_header_request.split() 
    1363   except Exception, e: 
    1364     raise HttpError417('Http header request needs spacing in between: Given: ' + http_header_request + str(e)) 
    1365  
    1366   # Check that the first line at least contains 3  words: RequestMethod <url> HTTP<version> 
    1367   if len(http_command_url_version) >= 3:  
    1368     url = http_command_url_version[1] 
    1369     http_version = http_command_url_version[2] 
    1370    
    1371   else: 
    1372     # http request header cant have any more data than RequestMethod <url> HTTP<version> or RequestMethod HTTP<version> 
    1373     raise HttpError417('The request header should contain  RequestMethod <url> HTTP<version>, Given: ' + http_header_request) 
    1374  
    1375   # check the http comand is valid or if its even suported(suported comands include GET and POST) 
    1376   if http_command_url_version[0] == 'GET' or http_command_url_version[0].lower() == 'post': 
    1377      http_command = http_command_url_version[0] 
    1378   else: 
    1379     raise HttpError501('The given http comand is not suported or valid. Given: ' + str(http_command_url_version[0])) 
    1380    
    1381  
    1382   # check the if the http version is valid 
    1383   if not http_version.startswith('HTTP'): 
    1384     raise HttpError417('Http header request version should start of with HTTP then <version>, Given: ' +  httpversion + ' as a http version')  
    1385  
    1386   # (query used to modify the dynamic page) http header includes the path and query, pasrse the given url to path and query  
    1387   (path, query) = _registerhttpcallback_parse_httpheader_url(url)   
    1388  
    1389   return http_command, path, query, http_version  
    1390  
    1391  
    1392  
    1393  
    1394 def _registerhttpcallback_parse_httpheader_url(url): 
    1395   # parse out the query and path from the url 
    1396   path = '' 
    1397   query = '' 
    1398  
    1399   # if url isnt given return empty strings   
    1400   if url != '': 
    1401     # if url is given parse the query and path using url parse 
    1402     try: 
    1403      # returns a dictionary of {scheme, netloc, path, query, fragment, username, 
    1404      # password, hostname and port} parsing the url                       
    1405       urlparse = urlparse_urlsplit(url)   
    1406     except Exception, e: 
    1407       raise HttpError417('Http request given url doesnt meet the http protocol standards ' + str(e) + 'Given url: ' + url) 
    1408  
    1409     # retrieve only the path and query  
    1410     try: 
    1411       path = urlparse['path'] 
    1412       query = urlparse['query'] 
    1413     except Exception, e: 
    1414       raise HttpError417('Http request given url doesnt meet the http protocol standards ' + str(e) + 'Given url: ' + url) 
    1415          
    1416     # if there is a url there has to be a path so raise an exception because the given url format is wrong  
    1417     if path == '': 
    1418       raise HttpError417('Error on parsing the http request header url: Given ' + url) 
    1419      
    1420   return path, query 
    1421  
    1422  
    1423  
    1424  
    1425 def _registerhttpcallback_receive_httpposted_data(sock, httprequest_dictionary, httppoost_limit): 
    1426   # receive the posted data which sent right after the http request with a empty line 
    1427   # indicating the end of the posted data(this is if the http comand is only a POST) 
    1428  
    1429   if not type(httppoost_limit) == int: 
    1430     # check if the given http post limit is valid 
    1431     raise HttpServerError('The given http post limit is not a int, given: ' + str(type(httppoost_limit))) 
    1432  
    1433   # if the http comand method isnt post theres is no posted data 
    1434   posted_data = None 
    1435  
    1436   # Bug pointed out by Albert Rafetseder: not all browsers send post with caps  
    1437   if httprequest_dictionary['http_command'].lower() == 'post': 
    1438     # get the posted data length or raise an exception if not given  
    1439     try: 
    1440       posted_data_length = int(httprequest_dictionary['Content-Length']) 
    1441     except Exception, e:    
    1442       raise HttpError411('content length is required on a http POST comand') 
    1443     else: 
    1444       # Bug pointed out by Albert Rafetseder: post doesnt send a empty line after the posted data 
    1445       # recieve the posted data using the posted data length 
    1446       posted_data = _registerhttpcallback_receive_httppost(sock, posted_data_length, httppoost_limit) 
    1447  
    1448     # check if there is a posted data and return it 
    1449     if len(posted_data) == 0: 
    1450       raise HttpError417('The request included a http POST comand with no data posted') 
    1451      
    1452   # adds the posted data or None to the httprequest dictionary  
    1453   httprequest_dictionary['posted_data'] = posted_data 
    1454  
    1455  
    1456  
    1457  
    1458  
    1459 def _registerhttpcallback_receive_httppost(sock, http_post_length, length_limit): 
    1460   # receives posted data from the given connection untill the given content length field matchs the received amount  
    1461   total_recvd_post = '' 
    1462    
    1463   while True: 
    1464     if len(total_recvd_post) == http_post_length: 
    1465       # if the content length matchs the received posted data return it  
    1466       return total_recvd_post 
    1467  
    1468     # raise an exception if the received posted data length is greater than the given http header content length    
    1469     if len(total_recvd_post) > http_post_length: 
    1470       raise HttpError417('The http posted data didnt match the http content length header, given content length: ' + str(http_post_length) + ' while posted data length is ' + str(len(total_recvd_post))) 
    1471  
    1472     # raise an exception if the total received length has exceeded the given length limit 
    1473     if len(total_recvd_post) > length_limit:                   
    1474       raise HttpError413('The http posted data length exceeded length limit of ' + str(length_limit)) 
    1475  
    1476                         
    1477     try: 
    1478       # receive one character at a time inorder to check for the empty line 
    1479       content = sock.recv(512) 
    1480  
    1481     # catch any error that happens while receiving content              
    1482     except SocketTimeoutError, e: 
    1483       raise HttpError408('The server timed out while waiting to receive the post data ' + str(e)) 
    1484     except Exception, e:  
    1485       raise HttpError500('Error while receiving request posted data ' + str(e)) 
    1486  
    1487     else: 
    1488       # if there was not receiving error, keep on adding the receieved content  
    1489       total_recvd_post += content 
    1490  
    1491  
    1492  
    1493    
    1494  
    1495 def _registerhttpcallback_receive_untilemptyline(sock, length_limit): 
    1496   # receives data from socket connection until it a empty line or until the given 
    1497   # length limit is exceeded                       
    1498   total_recvd_content = '' 
    1499    
    1500   while True: 
    1501     # receive until a empty line (\n\n or \r\n\r\n because new line is different by computer)  
    1502     if '\r\n\r\n' in total_recvd_content or '\n\n' in total_recvd_content: 
    1503       # found a empty line return the total received content 
    1504       return total_recvd_content.strip() 
    1505  
    1506     # raise an exception if the total received length has exceeded the given length limit 
    1507     if len(total_recvd_content) > length_limit:                   
    1508       raise HttpError413('The http request length exceeded the given length limit ' + str(length_limit)) 
    1509                          
    1510     try: 
    1511       # receive one character at a time inorder to check for the empty line 
    1512       content = sock.recv(1) 
    1513  
    1514     # catch any error that happens while receiving content              
    1515     except SocketTimeoutError, e: 
    1516       raise HttpError408('The server timed out while waiting to receive the request ' + str(e)) 
    1517     except Exception, e:  
    1518       raise HttpError500('Error while receiving http request ' + str(e)) 
    1519  
    1520     else: 
    1521       # if there was not receiving error, keep on adding the receieved content  
    1522       total_recvd_content += content 
    1523  
    1524  
    1525  
    1526  
    1527  
    1528 def _registerhttpcallback_send_httpresponse(sock, callbackfunc_val): 
    1529   # sends a response to the client(web browser) with a ok http header and the http web page content 
    1530   if not type(callbackfunc_val) == list: 
    1531     raise HttpUserInputError('Callback func didnt return list, returned ' + str(type(callbackfunc_val))) 
    1532  
    1533   try: 
    1534     webpage_content = callbackfunc_val[0] 
    1535     callbckfunc_httpheader = callbackfunc_val[1] 
    1536   except Exception, e: 
    1537     raise HttpUserInputError('Callback func returned data failed ' + str(e)) 
    1538  
    1539   # check the given web page content  
    1540   if not type(webpage_content) == str: 
    1541     raise HttpUserInputError('Callback func didnt return str for the content, returned ' + str(type(webpage_content))) 
    1542   if len(webpage_content) == 0: 
    1543     raise HttpUserInputError('Callback func didnt return any content') 
    1544    
    1545    
    1546   # build the http ok response header  
    1547   httpheader = 'HTTP/1.0 200 OK\n' 
    1548   httpheader += 'Content-Length: ' + str(len(webpage_content)) + '\n' 
    1549   #check if there is a given http header and add it to the response 
    1550   httpheader += _registerhttpcallback_parse_callbckfunc_httpheader(callbckfunc_httpheader) 
    1551      
    1552   httpheader += 'Server: Seattle Testbed\n\n' 
    1553  
    1554   # http header followed by http content and close the connection 
    1555   try: 
    1556     sock.send(httpheader)  
    1557     sock.send(webpage_content) 
    1558     sock.close()  
    1559   except Exception, e: 
    1560     raise HttpConnectionError('server failed to send the http content ' + str(e))   
    1561  
    1562  
    1563  
    1564  
    1565  
    1566 def _registerhttpcallback_parse_callbckfunc_httpheader(callbckfunc_httpheader): 
    1567   # builds a http header from the given http callback func list 
    1568   if callbckfunc_httpheader == None: 
    1569     # if the http header isnt given return a empty string 
    1570     return '' 
    1571   elif not type(callbckfunc_httpheader) == dict: 
    1572     # raise an exception if the http header isnt dictionary 
    1573     raise HttpUserInputError('The given http header is not a dictionary, given: ' + str(type(callbckfunc_httpheader))) 
    1574   else:  
    1575     # take the given key and val from the callbckfunc_httpheader dictionary and add them to the http header with 
    1576     # correct http format 
    1577     httpheaders = '' 
    1578     for key, val in callbckfunc_httpheader.items(): 
    1579       # make simple checks on the key and val  
    1580       if not type(key) == str: 
    1581         raise HttpUserInputError('The callback func given http header key isnt str given ' + str(type(key))) 
    1582       if not type(val) == str: 
    1583         raise HttpUserInputError('The callback func given http header value isnt str given ' + str( type(val))) 
    1584       if key == '' or val == '': 
    1585         # raise an exception if the key or value is a empty field 
    1586         raise HttpUserInputError('The callback func given empty http header feild of key or value') 
    1587       if key.capitalize() != key: 
    1588         raise HttpUserInputError('The callback func given http header key is not capitalized, given: ' + key) 
    1589  
    1590       # add the key and value to the http header field 
    1591       httpheaders += key + ' : ' + val + '\n'   
    1592  
    1593     # return the string of the http header   
    1594     return httpheaders 
    1595  
    1596    
    1597  
    1598  
    1599  
    1600 def _registerhttpcallback_send_httpformat_error(sock, e): 
    1601   # send  correct format http header with a  http content that displays detailed error msg and close connection  
    1602  
    1603   # using the httpstatuscode dictionary get the statuscode number and statuscode constant from the given httperror 
    1604   (statuscode_numb, client_error_msg, statuscode_constant) = _registerhttpcallback_get_http_statuscode(e) 
    1605    
    1606   # build http body error msg to client(web browser) 
    1607   error_msg = client_error_msg 
    1608  
    1609   # error content body 
    1610   httpcontent = '<html>' 
    1611   httpcontent += '<head><title>' + str(statuscode_numb) + ' ' + statuscode_constant + '</title></head>' 
    1612   httpcontent += '<body><h1>' + str(statuscode_numb) + ' ' + statuscode_constant + '</h1>' 
    1613   httpcontent += '<p>' + error_msg + '</p></body>' 
    1614   httpcontent += '</html>' 
    1615   # to end the error content 
    1616   httpcontent += '\n\n' 
    1617    
    1618   # build the http header to send     
    1619   httpheader = 'HTTP/1.0 ' + str(statuscode_numb)  + ' ' + statuscode_constant + '\n' 
    1620  
    1621   # for redirect add the location of the redirection to the http header     
    1622   if statuscode_numb == 301 or statuscode_numb == 302: 
    1623     if client_error_msg == '': 
    1624       raise HttpUserInputError('Internal server error: callback func client should put the location on raising redirect') 
    1625     elif not client_error_msg.startswith('http://'): 
    1626       raise HttpUserInputError('Internal server error: calback func client redirect is invalid, Given: ' + client_error_msg) 
    1627     else: 
    1628       httpheader += 'Location: ' + str(client_error_msg) + '\n' 
    1629    
    1630   # finish up the http header 
    1631   httpheader += 'Content-Length: ' + str(len(httpcontent)) + '\n' 
    1632   httpheader += 'Server: Seattle Testbed\n\n' 
    1633    
    1634   # send the http response header and body to the client(web browser) and close connection 
    1635   try: 
    1636     sock.send(httpheader) 
    1637     sock.send(httpcontent) 
    1638     sock.close() 
    1639   except Exception, e: 
    1640     raise HttpConnectionError('server failed internally on send http error ' + str(statuscode_numb) + ' ' + statuscode_constant + ' ' + error_msg + ' Raised' + str(e))  
    1641  
    1642  
    1643  
    1644  
    1645 def _registerhttpcallback_get_http_statuscode(e): 
    1646   # retrieves the status code number and constant given a exception class  
    1647    
    1648   # httpstatus code dictionary with the statuscode constant 
    1649   httpstatuscode_dict = { 
    1650       HttpError100: (100, 'Continue'), 
    1651       HttpError101: (101, 'Switching Protocols'), 
    1652       HttpError102: (102, 'Processing'), 
    1653       HttpError201: (201 ,'Created'), 
    1654       HttpError202: (202, 'Accepted'),   
    1655       HttpError203: (203, 'Non-Authoritative Information'), 
    1656       HttpError204: (204, 'No Content'), 
    1657       HttpError205: (205, 'Reset Content'), 
    1658       HttpError206: (206, 'Partial Content'), 
    1659       HttpError207: (207, 'Multi-Status'), 
    1660       HttpError226: (226, 'IM Used'), 
    1661       HttpError300: (300, 'Multiple Choices'), 
    1662       HttpError301: (301, 'Moved Permanently'), 
    1663       HttpError302: (302, 'Found'), 
    1664       HttpError303: (303, 'See Other'), 
    1665       HttpError304: (304, 'Not Modified'), 
    1666       HttpError305: (305, 'Use Proxy'), 
    1667       HttpError306: (306, 'Unused'), 
    1668       HttpError307: (307, 'Temporary Redirect'), 
    1669       HttpError400: (400, 'Bad Request'), 
    1670       HttpError401: (401, 'Unauthorized'), 
    1671       HttpError402: (402, 'Payment Required'), 
    1672       HttpError403: (403, 'Forbidden'), 
    1673       HttpError404: (404, 'Not Found'), 
    1674       HttpError405: (405, 'Method Not Allowed'), 
    1675       HttpError406: (406, 'Not Acceptable'), 
    1676       HttpError407: (407, 'Proxy Authentication Required'), 
    1677       HttpError408: (408, 'Request Timeout'), 
    1678       HttpError409: (409, 'Conflict'), 
    1679       HttpError410: (410, 'Gone'), 
    1680       HttpError411: (411, 'Length Required'), 
    1681       HttpError412: (412, 'Precondition Failed'), 
    1682       HttpError413: (413, 'Request Entity Too Large'), 
    1683       HttpError414: (414, 'Request-URI Too Long'), 
    1684       HttpError415: (415, 'Unsupported Media Type'), 
    1685       HttpError416: (416, 'Requested Range Not Satisfiable'), 
    1686       HttpError417: (417, 'Expectation Failed'), 
    1687       HttpError418: (418, 'Im a teapot'), 
    1688       HttpError422: (422, 'Unprocessable Entity'), 
    1689       HttpError423: (423, 'Locked'), 
    1690       HttpError424: (424, 'Failed Dependency'), 
    1691       HttpError425: (425, 'Unordered Collection'), 
    1692       HttpError426: (426, 'Upgrade Required'), 
    1693       HttpError500: (500, 'Internal Server Error'), 
    1694       HttpError501: (501, 'Not Implemented'), 
    1695       HttpError502: (502, 'Bad Gateway'), 
    1696       HttpError503: (503, 'Service Unavailable'), 
    1697       HttpError504: (504, 'Gateway Timeout'), 
    1698       HttpError505: (505, 'HTTP Version Not Supported'), 
    1699       HttpError506: (506, 'Variant Also Negotiates'), 
    1700       HttpError507: (507, 'Insufficient Storage'), 
    1701       HttpError510: (510, 'Not Extended')} 
    1702    
    1703   # retrieves the status number and constant from the given exception class using the dictionary  
    1704   try: 
    1705     (statuscode_numb, statuscode_constant) = httpstatuscode_dict[type(e)] 
    1706   except Exception, e: 
    1707     raise HttpServerError('Internal error on generating error msg: ' + str(e)) 
    1708  
    1709   # get any extra error msg that the callback fucntion raised  
    1710   client_error_msg = str(e) 
    1711  
    1712   # return what is retrieved 
    1713   return statuscode_numb, client_error_msg, statuscode_constant 
    1714  
    1715  
    1716  
    1717  
    1718  
    1719  
    1720  
    1721  
    1722  
    1723  
    1724  
    1725  
    1726  
    1727  
    1728  
    1729 """ 
    1730 <Program Name> 
    1731   httpretrieve.repy 
    1732  
    1733 <Started> 
    1734   August 19, 2009 
    1735  
    1736 <Author> 
    1737   Yafete Yemuru 
    1738  
    1739 <Purpose> 
    1740   provides a http content from a web server using http protocol. It sends a http request to 
    1741   any http server through socket connection to get the http content. Then once the http server 
    1742   replies with http header and content, the http header is checked for any error message. 
    1743   Then provides http content in a format of string, saved in a file or as a file like object.  
    1744 """ 
    1745  
    1746  
    1747  
    1748  
    1749 #begin include urlparse.repy   
    1750 """ 
    1751 <Program Name> 
    1752   urlparse.repy 
    1753  
    1754 <Started> 
    1755   May 15, 2009 
    1756  
    1757 <Author> 
    1758   Michael Phan-Ba 
    1759  
    1760 <Purpose> 
    1761   Provides utilities for parsing URLs, based on the Python 2.6.1 module urlparse. 
    1762  
    1763 """ 
    1764  
    1765  
    1766 def urlparse_urlsplit(urlstring, default_scheme="", allow_fragments=True): 
    1767   """ 
    1768   <Purpose> 
    1769     Parse a URL into five components, returning a dictionary.  This corresponds 
    1770     to the general structure of a URL: 
    1771     scheme://netloc/path;parameters?query#fragment.  The parameters are not 
    1772     split from the URL and individual componenets are not separated. 
    1773  
    1774     Only absolute server-based URIs are currently supported (all URLs will be 
    1775     parsed into the components listed, regardless of the scheme). 
    1776  
    1777   <Arguments> 
    1778     default_scheme: 
    1779       Optional: defaults to the empty string.  If specified, gives the default 
    1780       addressing scheme, to be used only if the URL does not specify one. 
    1781  
    1782     allow_fragments: 
    1783       Optional: defaults to True.  If False, fragment identifiers are not 
    1784       allowed, even if the URL's addressing scheme normally does support them. 
    1785  
    1786   <Exceptions> 
    1787     ValueError on parsing a non-numeric port value. 
    1788  
    1789   <Side Effects> 
    1790     None. 
    1791  
    1792   <Returns> 
    1793     A dictionary containing: 
    1794  
    1795     Key         Value                               Value if not present 
    1796     ============================================================================ 
    1797     scheme      URL scheme specifier                empty string 
    1798     netloc      Network location part               empty string 
    1799     path        Hierarchical path                   empty string 
    1800     query       Query component                     empty string 
    1801     fragment    Fragment identifier                 empty string 
    1802     username    User name                           None 
    1803     password    Password                            None 
    1804     hostname    Host name (lower case)              None 
    1805     port        Port number as integer, if present  None 
    1806  
    1807   """ 
    1808  
    1809   components = {"scheme": default_scheme, "netloc": "", "path": "", "query": "", 
    1810     "fragment": "", "username": None, "password": None, "hostname": None, 
    1811     "port": None } 
    1812  
    1813   # Extract the scheme, if present. 
    1814   (lpart, rpart) = _urlparse_splitscheme(urlstring) 
    1815   if lpart: 
    1816     components["scheme"] = lpart 
    1817  
    1818   # Extract the server information, if present. 
    1819   if rpart.startswith("//"): 
    1820     (lpart, rpart) = _urlparse_splitnetloc(rpart, 2) 
    1821     components["netloc"] = lpart 
    1822  
    1823     (components["username"], components["password"], components["hostname"], 
    1824       components["port"]) = _urlparse_splitauthority(lpart) 
    1825  
    1826   # Extract the fragment. 
    1827   if allow_fragments: 
    1828     (rpart, components["fragment"]) = _urlparse_splitfragment(rpart) 
    1829  
    1830  
    1831   # Extract the query. 
    1832   (components["path"], components["query"]) = _urlparse_splitquery(rpart) 
    1833  
    1834   return components 
    1835  
    1836  
    1837 def _urlparse_splitscheme(url): 
    1838   """Parse the scheme portion of the URL""" 
    1839   # The scheme is valid only if it contains these characters. 
    1840   scheme_chars = \ 
    1841     "abcdefghijklmnopqrstuvwxyz0123456789+-." 
    1842  
    1843   scheme = "" 
    1844   rest = url 
    1845  
    1846   spart = url.split(":", 1) 
    1847   if len(spart) == 2: 
    1848  
    1849     # Normalize the scheme. 
    1850     spart[0] = spart[0].lower() 
    1851  
    1852     # A scheme is valid only if it starts with an alpha character. 
    1853     if spart[0] and spart[0][0].isalpha(): 
    1854       for char in spart[0]: 
    1855         if char not in scheme_chars: 
    1856           break 
    1857       (scheme, rest) = spart 
    1858  
    1859   return scheme, rest 
    1860  
    1861  
    1862 def _urlparse_splitnetloc(url, start=0): 
    1863   """Parse the netloc portion of the URL""" 
    1864  
    1865   # By default, the netloc is delimited by the end of the URL. 
    1866   delim = len(url) 
    1867  
    1868   # Find the left-most delimiter. 
    1869   for char in "/?#": 
    1870     xdelim = url.find(char, start) 
    1871     if xdelim >= 0: 
    1872       delim = min(delim, xdelim) 
    1873  
    1874   # Return the netloc and the rest of the URL. 
    1875   return url[start:delim], url[delim:] 
    1876  
    1877  
    1878 def _urlparse_splitauthority(netloc): 
    1879   """Parse the authority portion of the netloc""" 
    1880  
    1881   # The authority can have a userinfo portion delimited by "@". 
    1882   authority = netloc.split("@", 1) 
    1883  
    1884   # Default values. 
    1885   username = None 
    1886   password = None 
    1887   hostname = None 
    1888   port = None 
    1889  
    1890   # Is there a userinfo portion? 
    1891   if len(authority) == 2: 
    1892  
    1893     # userinfo can be split into username:password 
    1894     userinfo = authority[0].split(":", 1) 
    1895  
    1896     # hostport can be split into hostname:port 
    1897     hostport = authority[1].split(":", 1) 
    1898  
    1899     if userinfo[0]: 
    1900       username = userinfo[0] 
    1901     if len(userinfo) == 2: 
    1902       password = userinfo[1] 
    1903  
    1904   # No userinfo portion found. 
    1905   else: 
    1906  
    1907     # hostport can be split into hostname:port 
    1908     hostport = netloc.split(":", 1) 
    1909  
    1910   # Is there a port value? 
    1911   if hostport[0]: 
    1912     hostname = hostport[0] 
    1913   if len(hostport) == 2: 
    1914     port = int(hostport[1], 10) 
    1915  
    1916   # Return the values. 
    1917   return username, password, hostname, port 
    1918  
    1919  
    1920 def _urlparse_splitquery(url): 
    1921   """Parse the query portion of the url""" 
    1922  
    1923   qpart = url.split("?", 1) 
    1924   if len(qpart) == 2: 
    1925     query = qpart[1] 
    1926   else: 
    1927     query = "" 
    1928  
    1929   return qpart[0], query 
    1930  
    1931  
    1932 def _urlparse_splitfragment(url): 
    1933   """Parse the query portion of the url""" 
    1934  
    1935   fpart = url.split("#", 1) 
    1936   if len(fpart) == 2: 
    1937     fragment = fpart[1] 
    1938   else: 
    1939     fragment = "" 
    1940  
    1941   return fpart[0], fragment 
    1942  
    1943 #end include urlparse.repy   
    1944 #begin include sockettimeout.repy 
    1945 """ 
    1946 <Description> 
    1947   Puts back in Python's non-blocking functionality. 
    1948  
    1949   send(): 
    1950     Raises SocketTimeout Error if the send call lasts 
    1951     longer than the set timeout. 
    1952  
    1953   recv(): 
    1954     Guarentees the receipt of a message.   Raises SocketTimeoutError if it does not 
    1955     receive any message before a given timeout. 
    1956     If actually receives the message, returns the message and continues. 
    1957  
    1958 <Usage> 
    1959   Text-replacable for Repy Sockets: 
    1960     timeout_openconn(desthost, destport, localip=None, localport=None, timeout = 5) 
    1961     timeout_waitforconn(localip, localport, function) 
    1962  
    1963   Object: 
    1964     sockobj.settimeout(seconds) 
    1965     sockobj.send(data) 
    1966     sockobj.recv(bytes) 
    1967     sockobj.close() 
    1968  
    1969 <Date> 
    1970   Sun Mar  1 10:27:35 PST 2009 
    1971  
    1972 <Example> 
    1973   # hello world 
    1974   include sockettimer.repy 
    1975  
    1976   def callback(ip, port, timeout_sockobj, commhandle, listenhandle): 
    1977     hw_message = timeout_sockobj.recv(1047) 
    1978  
    1979     # cleanup 
    1980     stopcomm(commhandle) 
    1981     stopcomm(listenhandle) 
    1982     timeout_sockobj.close() 
    1983  
    1984     print hw_message # => "hello world!" 
    1985    
    1986   def server(): 
    1987     sockobj = timeout_waitforconn(getmyip(), 12345, callback) 
    1988  
    1989   def client(): 
    1990     sockobj = timeout_openconn(getmyip(), 12345) 
    1991     sockobj.send("hello world!") 
    1992  
    1993   def main(): 
    1994     server() 
    1995     client() 
    1996     exitall() 
    1997  
    1998   if callfunc == 'initialize': 
    1999     main()  
    2000 """ 
    2001  
    2002 class SocketTimeoutError(Exception): 
    2003   """The socket timed out before receiving a response""" 
    2004  
    2005 def timeout_openconn(desthost, destport, localip=None, localport=None, timeout = 5): 
    2006   """ 
    2007   <Purpose>  
    2008     Wrapper for Repy like socket interface 
    2009  
    2010   <Args> 
    2011     Same as Repy openconn 
    2012  
    2013   <Exception> 
    2014     Timeout exception if the dest address doesnt respond. 
    2015  
    2016   <Returns> 
    2017     socket obj on success 
    2018   """ 
    2019  
    2020   tsock = TimeoutSocket() 
    2021   tsock.settimeout(timeout) 
    2022   if localip and localport: 
    2023     tsock.bind((localip, localport)) 
    2024   tsock.connect((desthost, destport)) 
    2025   return tsock 
    2026  
    2027 def timeout_waitforconn(localip, localport, function): 
    2028   """ 
    2029   <Purpose>  
    2030     Wrapper for Repy like socket interface 
    2031  
    2032   <Args> 
    2033     Same as Repy waitforconn 
    2034  
    2035   <Side Effects> 
    2036     Sets up event listener which calls function on messages. 
    2037  
    2038   <Returns> 
    2039     Handle to listener. 
    2040   """ 
    2041  
    2042   tsock = TimeoutSocket() 
    2043   tsock.bind((localip, localport)) 
    2044   tsock.setcallback(function) 
    2045   return tsock.listen() 
    2046  
    2047 class TimeoutSocket: 
    2048   """ 
    2049   <Purpose> 
    2050     Provide an socket object like the Repy usual one. 
    2051  
    2052   <Side Effects> 
    2053     Uses a getlock() to watch for a timeout 
    2054     Uses waitforconn and openconn to simulate socket 
    2055   """ 
    2056  
    2057   ################ 
    2058   # Constructors 
    2059   ################ 
    2060  
    2061   def __init__(self): 
    2062     """ Constructor for socket """ 
    2063 #    self.lock = getlock() # general lock BUG: Do we need to lock everything? 
    2064     self.timeout_lock = getlock() # special lock for Timeout condition 
    2065     self.timeout = 5 # seconds to wait 
    2066     self.bytes_sent = None # used to check if send() timed out 
    2067  
    2068     # user vars    
    2069     self.local_address = None # ip, port 
    2070     self.remote_address = None # ip, port 
    2071     self.callback = None # the user's function to call 
    2072  
    2073     # repy socket vars 
    2074     self.sockobj = None #  the Repy socket 
    2075     self.commhandle = None # the current comm 
    2076     self.listencommhandle = None # the listener comm 
    2077  
    2078     #error tracking vars 
    2079      
    2080     #if any exceptions are thrown in the separate thread executing _send_and_release,  
    2081     #they are caught and stored in this variable, then raised in _send_or_close 
    2082     self.TCPSendError = None 
    2083  
    2084   ################ 
    2085   # Mutator methods 
    2086   ################# 
    2087  
    2088   def settimeout(self, value): 
    2089     """ Setter for timeout""" 
    2090     self.timeout = value 
    2091  
    2092   def setcallback(self, function): 
    2093     """ Setter for callback function""" 
    2094     self.callback = function 
    2095  
    2096   #################### 
    2097   # Public Methods 
    2098   #################### 
    2099  
    2100   def bind(self, local_address = None): 
    2101     """ 
    2102     <Purpose> 
    2103       Set local address 
    2104  
    2105     <Args> 
    2106       Tuple of (ip, port) local. 
    2107     """ 
    2108     self.local_address = local_address 
    2109  
    2110   def listen(self): 
    2111     """ 
    2112     <Purpose> 
    2113       Listen for peer 
    2114      
    2115     <Side Effects> 
    2116       Calls Repy waitforconn() 
    2117     """ 
    2118     return self._waitforconn() 
    2119  
    2120   def connect(self, remote_address): 
    2121     """ 
    2122     <Purpose> 
    2123       Connect to peer. 
    2124  
    2125     <Args> 
    2126       Tuple of (ip, port) remote. 
    2127     
    2128     <Side Effects> 
    2129       Calls Repy openconn. 
    2130     """ 
    2131     self.remote_address = remote_address 
    2132     self._openconn() 
    2133  
    2134   def recv(self, maxLen): # timeout as optional arg ??? 
    2135     """ 
    2136     <Purpose> 
    2137       If it fails to finish within the timeout, I close the socket and raise a 
    2138       TimeoutError exception. I.e. if there's no message, we call it an error 
    2139       and raise it. 
    2140        
    2141     <Arguments> 
    2142       maxLen - bytes to recv 
    2143  
    2144     <Exception> 
    2145       Raises TimeoutError exception if the recv times out 
    2146       without receiving a message. 
    2147  
    2148     <Side Effects> 
    2149       Closes the connection if times out. 
    2150  
    2151     <Returns> 
    2152       The message. 
    2153     """ 
    2154     return self._recv_or_close(maxLen) 
    2155  
    2156   def send(self, data): 
    2157     """ 
    2158     <Purpose> 
    2159       Just like normal Repy socket.  Sends messages. 
    2160        
    2161     <Arguments> 
    2162       data - the string message 
    2163  
    2164     <Exception> 
    2165       Same as Repy socket. 
    2166   
    2167     <Returns> 
    2168       The bytes sent. 
    2169     """ 
    2170     return self._send_or_close(data) 
    2171  
    2172   def close(self): 
    2173     self.local_address = None # ip, port 
    2174     self.remote_address = None # ip, port 
    2175     self.callback = None # the user's function to call 
    2176  
    2177     if self.sockobj: 
    2178       self.sockobj.close() 
    2179     self.sockobj = None #  the Repy socket 
    2180      
    2181     # Armon: As part of the semantics, stopcomm will raise an  
    2182     # exception given an invalid handle, e.g. None. Thus, 
    2183     # we need to check for this. 
    2184     if self.commhandle:  
    2185       stopcomm(self.commhandle) 
    2186       self.commhandle = None # the current comm 
    2187      
    2188     # Armon: Same as above. 
    2189     if self.listencommhandle: 
    2190       stopcomm(self.listencommhandle) 
    2191       self.listencommhandle = None # the listener comm 
    2192  
    2193  
    2194   ######################## 
    2195   # Private 
    2196   ######################### 
    2197  
    2198   def _openconn(self): 
    2199     """Handle current state variables and call Repy openconn.""" 
    2200  
    2201     destip, destport = self.remote_address 
    2202     if self.local_address: 
    2203       srcip, srcport = self.local_address 
    2204       self.sockobj = openconn(destip, destport, srcip, srcport, self.timeout) 
    2205     else: 
    2206       self.sockobj = openconn(destip, destport) 
    2207  
    2208   def _waitforconn(self): 
    2209     """Setup way between Repy waitforconn event""" 
    2210     localip, localport = self.local_address 
    2211     self.listencommhandle = waitforconn(localip, localport, self._callback) 
    2212     return self.listencommhandle 
    2213  
    2214   def _callback(self, ip, port, sockobj, ch, lh): 
    2215     """Pass on through to user callback""" 
    2216     self.sockobj = sockobj 
    2217     self.listencommhandle = lh # same as the 1st from wait for comm, right? 
    2218     self.commhandle = ch # should we care? 
    2219      
    2220     if not self.remote_address: 
    2221       self.remote_address = (ip, port) 
    2222     else:  
    2223       raise Exception("what! peer does not match?") 
    2224  
    2225     self.callback(ip, port, self, ch, lh) 
    2226  
    2227   def _send(self, data): 
    2228     """Send data""" 
    2229     return self.sockobj.send(data) 
    2230  
    2231   def _recv(self, maxLen): 
    2232     """Recv data of length maxLen""" 
    2233     return self.sockobj.recv(maxLen) 
    2234  
    2235   def _send_and_release(self, data): 
    2236     """Send data then release the timeout lock""" 
    2237      
    2238     #Bug Fix (Cosmin): exceptions thrown in separate thread _send_and_release could not be caught 
    2239     #now we store the exception if it is thrown and raise it back in send_or_close 
    2240     try: 
    2241       self.bytes_sent = self._send(data) 
    2242     except Exception, e: 
    2243       self.TCPSendError = e 
    2244      
    2245     self._quietly_release() # release the lock 
    2246   
    2247   def _quietly_release(self): 
    2248     """Release the timeout lock and ignore if already released""" 
    2249     try: 
    2250       self.timeout_lock.release() 
    2251     except: 
    2252       pass 
    2253     
    2254   def _send_or_close(self, data): 
    2255     """Raise the Timeout Error if no receipt.  Keep track by timeout_lock.""" 
    2256  
    2257     # acquire the lock, when it's release we'll carry on 
    2258     self.timeout_lock.acquire() 
    2259  
    2260     # fork off a lock that'll release the lock at the timeout 
    2261     timerhandle = settimer(self.timeout, self._quietly_release, ()) 
    2262  
    2263     # fork off a send call so we can raise the exception in the main thread 
    2264     # the send call will also release our lock 
    2265     settimer(0, self._send_and_release, (data,)) 
    2266  
    2267     # block until either the timeout or _send finishes 
    2268     self.timeout_lock.acquire() 
    2269     self.timeout_lock.release() 
    2270  
    2271     if self.bytes_sent: # send finished 
    2272       canceltimer(timerhandle) 
    2273       retdata = self.bytes_sent 
    2274       self.bytes_sent = None 
    2275       return retdata 
    2276     elif self.TCPSendError != None: 
    2277       #Bug Fix (Cosmin): exceptions thrown in separate thread _send_and_release could not be caught 
    2278        
    2279       #we got an error within the separate thread that performed the send operation 
    2280       exception_to_throw = self.TCPSendError 
    2281       self.TCPSendError = None 
    2282       raise exception_to_throw 
    2283     else: # it timed out 
    2284       self.close() 
    2285       raise SocketTimeoutError 
    2286  
    2287   def _recv_or_close(self, amount): 
    2288     """Raise the Timeout Error if no receipt.  Keep track by timeout_lock.""" 
    2289     timerhandle = settimer(self.timeout, self._clobbersocket, ()) 
    2290     try: 
    2291       retdata = self._recv(amount) 
    2292     except Exception, e: 
    2293       # if it's not the timeout, reraise... 
    2294       if self.timeout_lock.acquire(False): 
    2295         raise 
    2296       raise SocketTimeoutError 
    2297      
    2298     # I acquired the lock, I should stop the timer because I succeeded... 
    2299     if self.timeout_lock.acquire(False): 
    2300       # even if this isn't in time, the lock prevents a race condition  
    2301       # this is merely an optimization to prevent the timer from ever firing... 
    2302       canceltimer(timerhandle) 
    2303       self.timeout_lock.release() # Alper's bug 3/10/09 
    2304       return retdata 
    2305     else: 
    2306       raise SocketTimeoutError 
    2307  
    2308   def _clobbersocket(self): 
    2309     """If I can acquire the lock without blocking, then close the socket to abort""" 
    2310     if self.timeout_lock.acquire(False): 
    2311       self.close() 
    2312  
    2313  
    2314 ############################ 
    2315 # Deprecated functions 
    2316 ############################## 
    2317  
    2318 # private function... 
    2319 def sockettimeout_clobbersocket(sockobj,mylock): 
    2320   # if I can acquire the lock without blocking, then close the socket to abort 
    2321   if mylock.acquire(False): 
    2322     sockobj.close() 
    2323  
    2324 # if it fails to finish within the timeout, I close the socket and raise a 
    2325 # SocketTimeout exception... 
    2326 def sockettimeout_recv_or_close(sockobj, amount, timeout): 
    2327   # A lock I'll use for this attempt 
    2328   mylock = getlock() 
    2329   timerhandle = settimer(timeout,clobbersocket, (sockobj, mylock)) 
    2330   try: 
    2331     retdata = sockobj.recv(amount) 
    2332   except Exception, e: 
    2333     # if it's not the timeout, reraise... 
    2334     if mylock.acquire(False): 
    2335       raise 
    2336     raise SocketTimeout 
    2337      
    2338   # I acquired the lock, I should stop the timer because I succeeded... 
    2339   if mylock.acquire(False): 
    2340     # even if this isn't in time, the lock prevents a race condition  
    2341     # this is merely an optimization to prevent the timer from ever firing... 
    2342     canceltimer(timerhandle) 
    2343     return retdata 
    2344   else: 
    2345     raise SocketTimeout 
    2346  
    2347  
    2348 #end include sockettimeout.repy 
    2349 #begin include http_hierarchy_error.repy 
    2350 """ 
    2351 <Program Name> 
    2352   http_hierarchy_error.repy 
    2353  
    2354 <Started> 
    2355   Oct 05, 2009 
    2356  
    2357 <Author> 
    2358   Yafete Yemuru 
    2359  
    2360 <Purpose> 
    2361   provides a hierachy http error using status code including client and server errors 
    2362   classes.   
    2363 """ 
    2364  
    2365  
    2366 ''' 
    2367 http hierarchy error exception classes 
    2368  
    2369 -> HttpError 
    2370    -> HttpRetrieveClientError 
    2371       -> HttpUserInputError 
    2372       -> HttpConnectionError 
    2373  
    2374    -> HttpServerError 
    2375       -> HttpResponseError 
    2376          -> HttpHeaderError 
    2377             -> HttpHeaderReceivingError 
    2378             -> HttpHeaderFormatError 
    2379          -> HttpContentError 
    2380             -> HttpContentReceivingError 
    2381             -> HttpContentLengthError 
    2382         
    2383    -> HttpStatuscodeError 
    2384      -> HttpError1xx 
    2385         -> followed by all http status code error number HttpError(number) 
    2386          
    2387      -> HttpError2xx 
    2388         -> followed by all http status code error number HttpError(number) 
    2389          
    2390      -> HttpError3xx 
    2391         -> followed by all http status code error number HttpError(number) 
    2392  
    2393      -> HttpError4xx 
    2394         -> followed by all http status code error number HttpError(number) 
    2395          
    2396      -> HttpError5xx 
    2397         -> followed by all http status code error number HttpError(number) 
    2398  
    2399 ''' 
    2400  
    2401 class HttpError(Exception): 
    2402   pass 
    2403  
    2404 # raises an exception for http client error  
    2405 class HttpRetrieveClientError(HttpError):# extend HttpError  
    2406   pass 
    2407 class HttpUserInputError(HttpRetrieveClientError): 
    2408   pass 
    2409 class HttpConnectionError(HttpRetrieveClientError): 
    2410   pass 
    2411  
    2412  
    2413 # raises an exception for any http server failure   
    2414 class HttpServerError(HttpError):# extend HttpError  
    2415   pass 
    2416 class HttpResponseError(HttpServerError): 
    2417   pass 
    2418 class HttpHeaderError(HttpResponseError): 
    2419   pass 
    2420 class HttpHeaderReceivingError(HttpHeaderError): 
    2421   pass 
    2422 class HttpHeaderFormatError(HttpHeaderError): 
    2423   pass 
    2424 class HttpContentError(HttpResponseError): 
    2425   pass 
    2426 class HttpContentReceivingError(HttpContentError): 
    2427   pass 
    2428 class HttpContentLengthError(HttpContentError): 
    2429   pass 
    2430  
    2431  
    2432 class HttpStatusCodeError(HttpError):# extend HttpError 
    2433   pass 
    2434 class HttpError1xx(HttpStatusCodeError): 
    2435   pass 
    2436 class HttpError100(HttpError1xx): 
    2437   pass 
    2438 class HttpError101(HttpError1xx):  
    2439   pass 
    2440 class HttpError102(HttpError1xx):  
    2441   pass 
    2442  
    2443  
    2444 class HttpError2xx(HttpStatusCodeError): 
    2445   pass 
    2446 class HttpError201(HttpError2xx):  
    2447   pass 
    2448 class HttpError202(HttpError2xx):    
    2449   pass 
    2450 class HttpError203(HttpError2xx):  
    2451   pass 
    2452 class HttpError204(HttpError2xx):  
    2453   pass 
    2454 class HttpError205(HttpError2xx):  
    2455   pass 
    2456 class HttpError206(HttpError2xx):  
    2457   pass 
    2458 class HttpError207(HttpError2xx):  
    2459   pass 
    2460 class HttpError226(HttpError2xx):  
    2461   pass                  
    2462  
    2463  
    2464 class HttpError3xx(HttpStatusCodeError): 
    2465   pass 
    2466 class HttpError300(HttpError3xx):  
    2467   pass 
    2468 class HttpError301(HttpError3xx):  
    2469   pass 
    2470 class HttpError302(HttpError3xx):  
    2471   pass 
    2472 class HttpError303(HttpError3xx):  
    2473   pass 
    2474 class HttpError304(HttpError3xx): 
    2475   pass 
    2476 class HttpError305(HttpError3xx):  
    2477   pass 
    2478 class HttpError306(HttpError3xx):  
    2479   pass 
    2480 class HttpError307(HttpError3xx):  
    2481   pass 
    2482                      
    2483  
    2484 class HttpError4xx(HttpStatusCodeError): 
    2485   pass 
    2486 class HttpError400(HttpError4xx): 
    2487   pass   
    2488 class HttpError401(HttpError4xx):  
    2489   pass 
    2490 class HttpError402(HttpError4xx):  
    2491   pass 
    2492 class HttpError403(HttpError4xx): 
    2493   pass   
    2494 class HttpError404(HttpError4xx):  
    2495   pass 
    2496 class HttpError405(HttpError4xx):  
    2497   pass 
    2498 class HttpError406(HttpError4xx):  
    2499   pass 
    2500 class HttpError407(HttpError4xx):  
    2501   pass 
    2502 class HttpError408(HttpError4xx):  
    2503   pass 
    2504 class HttpError409(HttpError4xx):  
    2505   pass 
    2506 class HttpError410(HttpError4xx):  
    2507   pass 
    2508 class HttpError411(HttpError4xx):  
    2509   pass 
    2510 class HttpError412(HttpError4xx):  
    2511   pass 
    2512 class HttpError413(HttpError4xx):  
    2513   pass 
    2514 class HttpError414(HttpError4xx):  
    2515   pass 
    2516 class HttpError415(HttpError4xx):  
    2517   pass 
    2518 class HttpError416(HttpError4xx):  
    2519   pass 
    2520 class HttpError417(HttpError4xx):  
    2521   pass 
    2522 class HttpError418(HttpError4xx):  
    2523   pass 
    2524 class HttpError422(HttpError4xx):  
    2525   pass 
    2526 class HttpError423(HttpError4xx):  
    2527   pass 
    2528 class HttpError424(HttpError4xx):  
    2529   pass 
    2530 class HttpError425(HttpError4xx):  
    2531   pass 
    2532 class HttpError426(HttpError4xx):  
    2533   pass 
    2534  
    2535  
    2536 class HttpError5xx(HttpStatusCodeError): 
    2537   pass 
    2538 class HttpError500(HttpError5xx):  
    2539   pass 
    2540 class HttpError501(HttpError5xx):  
    2541   pass 
    2542 class HttpError502(HttpError5xx):  
    2543   pass 
    2544 class HttpError503(HttpError5xx):  
    2545   pass 
    2546 class HttpError504(HttpError5xx):  
    2547   pass 
    2548 class HttpError505(HttpError5xx):  
    2549   pass 
    2550 class HttpError506(HttpError5xx): 
    2551   pass   
    2552 class HttpError507(HttpError5xx): 
    2553   pass   
    2554 class HttpError510(HttpError5xx): 
    2555   pass 
    2556  
    2557 #end include http_hierarchy_error.repy 
    2558 #begin include urllib.repy 
    2559 def urllib_quote(string, safe="/"): 
    2560   """ 
    2561   <Purpose> 
    2562     Encode a string such that it can be used safely in a URL or XML 
    2563     document. 
    2564  
    2565   <Arguments> 
    2566     string: 
    2567            The string to urlencode. 
    2568  
    2569     safe (optional): 
    2570            Specifies additional characters that should not be quoted -- 
    2571            defaults to "/". 
    2572  
    2573   <Exceptions> 
    2574     TypeError if the safe parameter isn't an enumerable. 
    2575  
    2576   <Side Effects> 
    2577     None. 
    2578  
    2579   <Returns> 
    2580     Urlencoded version of the passed string. 
    2581   """ 
    2582  
    2583   resultstr = "" 
    2584  
    2585   # We go through each character in the string; if it's not in [0-9a-zA-Z] 
    2586   # we wrap it. 
    2587  
    2588   safeset = set(safe) 
    2589  
    2590   for char in string: 
    2591     asciicode = ord(char) 
    2592     if (asciicode >= ord("0") and asciicode <= ord("9")) or \ 
    2593         (asciicode >= ord("A") and asciicode <= ord("Z")) or \ 
    2594         (asciicode >= ord("a") and asciicode <= ord("z")) or \ 
    2595         asciicode == ord("_") or asciicode == ord(".") or \ 
    2596         asciicode == ord("-") or char in safeset: 
    2597       resultstr += char 
    2598     else: 
    2599       resultstr += "%%%02X" % asciicode 
    2600  
    2601   return resultstr 
    2602  
    2603  
    2604  
    2605  
    2606 def urllib_quote_plus(string, safe=""): 
    2607   """ 
    2608   <Purpose> 
    2609     Encode a string to go in the query fragment of a URL. 
    2610  
    2611   <Arguments> 
    2612     string: 
    2613            The string to urlencode. 
    2614  
    2615     safe (optional): 
    2616            Specifies additional characters that should not be quoted -- 
    2617            defaults to the empty string. 
    2618  
    2619   <Exceptions> 
    2620     TypeError if the safe parameter isn't a string. 
    2621  
    2622   <Side Effects> 
    2623     None. 
    2624  
    2625   <Returns> 
    2626     Urlencoded version of the passed string. 
    2627   """ 
    2628  
    2629   return urllib_quote(string, safe + " ").replace(" ", "+") 
    2630  
    2631  
    2632  
    2633  
    2634 def urllib_unquote(string): 
    2635   """ 
    2636   <Purpose> 
    2637     Unquote a urlencoded string. 
    2638  
    2639   <Arguments> 
    2640     string: 
    2641            The string to unquote. 
    2642  
    2643   <Exceptions> 
    2644     ValueError thrown if the last wrapped octet isn't a valid wrapped octet 
    2645     (i.e. if the string ends in "%" or "%x" rather than "%xx". Also throws 
    2646     ValueError if the nibbles aren't valid hex digits. 
    2647  
    2648   <Side Effects> 
    2649     None. 
    2650  
    2651   <Returns> 
    2652     The decoded string. 
    2653   """ 
    2654  
    2655   resultstr = "" 
    2656  
    2657   # We go through the string from end to beginning, looking for wrapped 
    2658   # octets. When one is found we add it (unwrapped) and the following 
    2659   # string to the resultant string, and shorten the original string. 
    2660  
    2661   while True: 
    2662     lastpercentlocation = string.rfind("%") 
    2663     if lastpercentlocation < 0: 
    2664       break 
    2665  
    2666     wrappedoctetstr = string[lastpercentlocation+1:lastpercentlocation+3] 
    2667     if len(wrappedoctetstr) != 2: 
    2668       raise ValueError("Quoted string is poorly formed") 
    2669  
    2670     resultstr = \ 
    2671         chr(int(wrappedoctetstr, 16)) + \ 
    2672         string[lastpercentlocation+3:] + \ 
    2673         resultstr 
    2674     string = string[:lastpercentlocation] 
    2675  
    2676   resultstr = string + resultstr 
    2677   return resultstr 
    2678  
    2679  
    2680  
    2681  
    2682 def urllib_unquote_plus(string): 
    2683   """ 
    2684   <Purpose> 
    2685     Unquote the urlencoded query fragment of a URL. 
    2686  
    2687   <Arguments> 
    2688     string: 
    2689            The string to unquote. 
    2690  
    2691   <Exceptions> 
    2692     ValueError thrown if the last wrapped octet isn't a valid wrapped octet 
    2693     (i.e. if the string ends in "%" or "%x" rather than "%xx". Also throws 
    2694     ValueError if the nibbles aren't valid hex digits. 
    2695  
    2696   <Side Effects> 
    2697     None. 
    2698  
    2699   <Returns> 
    2700     The decoded string. 
    2701   """ 
    2702  
    2703   return urllib_unquote(string.replace("+", " ")) 
    2704  
    2705  
    2706  
    2707  
    2708 def urllib_quote_parameters(dictionary): 
    2709   """ 
    2710   <Purpose> 
    2711     Encode a dictionary of (key, value) pairs into an HTTP query string or 
    2712     POST body (same form). 
    2713  
    2714   <Arguments> 
    2715     dictionary: 
    2716            The dictionary to quote. 
    2717  
    2718   <Exceptions> 
    2719     None. 
    2720  
    2721   <Side Effects> 
    2722     None. 
    2723  
    2724   <Returns> 
    2725     The quoted dictionary. 
    2726   """ 
    2727  
    2728   quoted_keyvals = [] 
    2729   for key, val in dictionary.items(): 
    2730     quoted_keyvals.append("%s=%s" % (urllib_quote(key), urllib_quote(val))) 
    2731  
    2732   return "&".join(quoted_keyvals) 
    2733  
    2734  
    2735  
    2736  
    2737 def urllib_unquote_parameters(string): 
    2738   """ 
    2739   <Purpose> 
    2740     Decode a urlencoded query string or POST body. 
    2741  
    2742   <Arguments> 
    2743     string: 
    2744            The string to decode. 
    2745  
    2746   <Exceptions> 
    2747     ValueError if the string is poorly formed. 
    2748  
    2749   <Side Effects> 
    2750     None. 
    2751  
    2752   <Returns> 
    2753     A dictionary mapping keys to values. 
    2754   """ 
    2755  
    2756   keyvalpairs = string.split("&") 
    2757   res = {} 
    2758  
    2759   for quotedkeyval in keyvalpairs: 
    2760     # Throw ValueError if there is more or less than one '='. 
    2761     quotedkey, quotedval = quotedkeyval.split("=") 
    2762     key = urllib_unquote(quotedkey) 
    2763     val = urllib_unquote(quotedval) 
    2764     res[key] = val 
    2765  
    2766   return res 
    2767  
    2768 #end include urllib.repy 
    2769  
    2770  
    2771  
    2772  
    2773  
    2774 def httpretrieve_open(url, http_query=None, http_post=None, http_header=None, header_timeout=30, content_timeout=30, httpheader_limit=8192, httpcontent_limit=4194304): 
    2775   """ 
    2776   <Purpose> 
    2777      Returns file like object that that can read the http content form a http server. The file like 
    2778      object gets string from http server using read method.   
    2779  
    2780   <Arguments> 
    2781     url: 
    2782            String of a http web server-URL 
    2783     http_post: 
    2784            dictionary of data to post to server(unencoded string, the library encodes the post it self) 
    2785     http_query: 
    2786            dictionary of query to send to server(unencoded string, the library encodes the query it self) 
    2787     http_header: 
    2788            dictionary of http header to add the the http header request 
    2789     header_timeout: 
    2790            socket timeout for receiving header from server(default value set to 30 seconds) 
    2791     content_timeout: 
    2792            socket timeout for receiving content from server(default value set to 30 seconds) 
    2793     httpheader_limit: 
    2794            length limit for when a server sends http header(default value set to 8kb(8192 charactors))  
    2795     httpcontent_limit: 
    2796             limits the the amount of content a server can send to the retrieval  
    2797   
    2798   <Exceptions> 
    2799         HttpUserInputError 
    2800             ->  If given a invalid URL  
    2801             ->  If given a none http protocol server 
    2802             ->  if file like object read is given a negative number or a none int as a limit 
    2803             ->  if the file like object is called after it is closed 
    2804  
    2805         HttpConnectionError 
    2806             ->  if opening connection with server fails       
    2807             ->  if sending http request to http server fails   
    2808  
    2809         HttpHeaderReceivingError 
    2810             ->  If the timeout(default timeout set to 5 seconds) for receiving exceeds    
    2811             ->  If the http header length exceeds (default  
    2812              
    2813         HttpHeaderFormatError 
    2814             ->  If The http header is too long(default set to 8 kb(8192 charactors)) 
    2815             ->  If the http header statuscode format is invalid. The right http header 
    2816                     status code includes HTTP<version> http_status_number http_status_msg                  
    2817             ->  If the http server gives a http header with a invalid content length 
    2818                     or invalid redirect location 
    2819              
    2820         HttpContentReceivingError: 
    2821             ->  if the http server fails to send http content  
    2822             ->  if the timeout(default timeout set to 5 seconds) for receiving exceeds 
    2823             ->  if the server socket connection fails for any reason besides connection closing 
    2824                     during receiving http content 
    2825                          
    2826         HttpContentLengthError: 
    2827             ->  if content length exceeds(default set to 4096 kb(4194304 charactors))  
    2828                     (ONLY WORKS CONTENT LENGTH IS GIVEN BY THE HTTP SERVER) 
    2829             ->  If the total received length is not the same as the content length(this check will fail 
    2830                      if the content length isnt given) 
    2831             ->  If read is called with limits and it returns a empty string but the total read is                                       
    2832                      not equal to the content length(this check will fail if the content length isnt given)                 
    2833                                              
    2834         HttpStatuscodeError: 
    2835             -> if the http response status code isnt ok or redirect, it will raise an exception 
    2836                    depending up on the http protocol status number   
    2837          
    2838  
    2839   <Side Effects> 
    2840     None  
    2841  
    2842   <Returns> 
    2843     Returns file like obj which can read the http content from http web server.  
    2844   """ 
    2845    
    2846   # check if url is valid and get host, path, port and query from the given url 
    2847   (host, port, path, url_query) = _httpretrieve_parse_given_url(url) 
    2848  
    2849   # get connection to the http web server 
    2850   try: 
    2851     sock = timeout_openconn(host, port) 
    2852      
    2853   except Exception, e: 
    2854     raise HttpConnectionError('Error: opening a connection failed with given http server, Given: ' + str(url) + ' ' + str(e)) 
    2855      
    2856   # build a http format request using the given port, host, path and query 
    2857   httpheader = _httpretrieve_buildhttprequest(http_header, port, host, path, url_query, http_query, http_post) 
    2858    
    2859   # send http format request to http web server   
    2860   _httpretrieve_sendhttprequest(sock, httpheader)  
    2861    
    2862   # receive the http header lines in a form of list from the http web server 
    2863   httpheaderlines = _httpretrieve_receive_httpheader(sock, header_timeout, httpheader_limit) 
    2864    
    2865   # get the http status number and http status msg from http header response 
    2866   (http_status_number, http_status_msg) = _httpretrieve_get_httpstatuscode(httpheaderlines) 
    2867  
    2868   if http_status_number == '200':# ok message 
    2869     # gets the content length if given. 
    2870     contentlength = _httpretrieve_get_contentlength(httpheaderlines) 
    2871     # return a filelikeobj to read the http content from the http server  
    2872     return _httpretrieve_filelikeobject(sock, contentlength, httpcontent_limit, content_timeout)  
    2873    
    2874   elif http_status_number == '301' or http_status_number == '302': # redirect 
    2875     # redirect to the new location via recursion   
    2876     sock.close() 
    2877     # get the redirection location 
    2878     redirect_location = _httpretrieve_httpredirect(httpheaderlines)  
    2879     # redirect to the new location using recursion 
    2880     return httpretrieve_open(redirect_location)      
    2881  
    2882   else: 
    2883     # if given receive content length inorder to check http content error is received fully 
    2884     contentlength = _httpretrieve_get_contentlength(httpheaderlines) 
    2885     # receive the http content error  
    2886     http_errorcontent = _httpretrieve_receive_httperror_content(sock, contentlength) 
    2887     # raise exception depending up on the http status number and add on the http error content after a 
    2888     # discription that says 'Http error content: ' 
    2889     _httpretrieve_raise_httpstatuscode_error(http_status_number, http_status_msg, http_errorcontent)    
    2890  
    2891  
    2892    
    2893  
    2894  
    2895 def httpretrieve_save_file(url, filename, http_query=None, http_post=None, http_header=None, header_timeout=30, content_timeout=30, httpheader_limit=8192, httpcontent_limit=4194304): 
    2896   """ 
    2897   <Purpose> 
    2898      Saves http content of the given URL to current directory   
    2899  
    2900   <Arguments> 
    2901     url: 
    2902            String of a http web server URL 
    2903     filename: 
    2904            The file name for the http content to be saved in 
    2905     http_post: 
    2906            dictionary of data to post to server(unencoded string, the library encodes the post it self) 
    2907     http_query: 
    2908            dictionary of query to send to server(unencoded string, the library encodes the query it self) 
    2909     http_header: 
    2910            dictionary of http header to add the the http header request 
    2911     header_timeout: 
    2912            socket timeout for receiving header from server(default value set to 30 seconds) 
    2913     content_timeout: 
    2914            socket timeout for receiving content from server(default value set to 30 seconds) 
    2915     httpheader_limit: 
    2916            length limit for when a server sends http header(default value set to 8kb(8192 charactors))  
    2917     httpcontent_limit: 
    2918             limits the the amount of content a server can send to the retrieval  
    2919             
    2920   <Exceptions> 
    2921    
    2922     HttpRetrieveClientError: cant create a file to save the http content too 
    2923     Also includes:  all the exception from httpretrieve_open  
    2924  
    2925   <Side Effects> 
    2926     same as httpretrieve_open   
    2927  
    2928   <Returns> 
    2929     None  
    2930   """ 
    2931    
    2932   httpcontent = '' 
    2933   try: 
    2934     # create a new file with the given filename 
    2935     newfile = open(filename, 'w') 
    2936   except Exception, e: 
    2937     raise HttpRetrieveClientError('Error on creating a file to saving http content' + str(e)) 
    2938  
    2939   http_obj = httpretrieve_open(url, http_query, http_post, http_header, header_timeout, content_timeout, httpheader_limit, httpcontent_limit) 
    2940  
    2941   # keep on reading 1024 and writing it into a file, until it receives an empty string 
    2942   # which means the http server content is completely read  
    2943   while True: 
    2944     httpcontent = http_obj.read(1024) 
    2945     if httpcontent == '': 
    2946       # done reading close file and file like obj and exit loop  
    2947       newfile.close()   
    2948       http_obj.close() 
    2949       break 
    2950     newfile.write(httpcontent) 
    2951  
    2952  
    2953    
    2954 def httpretrieve_get_string(url, http_query=None, http_post=None, http_header=None, header_timeout=30, content_timeout=30, httpheader_limit=8192, httpcontent_limit=4194304): 
    2955   """ 
    2956   <Purpose> 
    2957      retruns string of the http content from a given URL 
    2958  
    2959   <Arguments> 
    2960     url: 
    2961            String of a http web server-URL 
    2962     http_post: 
    2963            dictionary of data to post to server(unencoded string, the library encodes the post it self) 
    2964     http_query: 
    2965            dictionary of query to send to server(unencoded string, the library encodes the query it self) 
    2966     http_header: 
    2967            dictionary of http header to add the the http header request 
    2968     header_timeout: 
    2969            socket timeout for receiving header from server(default value set to 30 seconds) 
    2970     content_timeout: 
    2971            socket timeout for receiving content from server(default value set to 30 seconds) 
    2972     httpheader_limit: 
    2973            length limit for when a server sends http header(default value set to 8kb(8192 charactors))  
    2974     httpcontent_limit: 
    2975             limits the the amount of content a server can send to the retrieval  
    2976   
    2977   <Exceptions> 
    2978      same as httpretrieve_open 
    2979  
    2980   <Side Effects> 
    2981      same as httpretrieve_open  
    2982  
    2983   <Returns> 
    2984      returns a string of the http content.  
    2985   """        
    2986    
    2987   http_obj = httpretrieve_open(url, http_query, http_post, http_header, header_timeout, content_timeout, httpheader_limit, httpcontent_limit) 
    2988   # retrieve the http content from server using httpretrieve file like object 
    2989   httpcontent = http_obj.read() 
    2990   http_obj.close() 
    2991   # return the http content in a form of string 
    2992   return httpcontent 
    2993  
    2994  
    2995  
    2996 class _httpretrieve_filelikeobject: 
    2997   # file like object used to receive the http content with a length limit    
    2998   def __init__(self, sock, contentlength, httpcontent_limit, content_timeout): 
    2999     self.sock = sock 
    3000     if contentlength == None: 
    3001       self.contentlengthisknown = False 
    3002     else: 
    3003       self.contentlengthisknown = True 
    3004       self.contentlength = contentlength 
    3005     self.httpcontent_limit = httpcontent_limit   
    3006     self.content_timeout = content_timeout   
    3007     self.fileobjclosed = False 
    3008     self.totalcontentisreceived = False  
    3009     self.totalread = 0 
    3010      
    3011  
    3012   def read(self, limit = None): 
    3013     """ 
    3014     <Purpose> 
    3015         reads the http content from http server using the file like object    
    3016  
    3017     <Arguments> 
    3018         limit(optional): 
    3019              maximum number of bytes to read. If not specified the whole file is read. 
    3020              Can be 0 or any positive int 
    3021     
    3022     <Exceptions> 
    3023  
    3024         HttpContentReceivingError: 
    3025             ->  if the http server fails to send http content  
    3026             ->  if the timeout(default timeout set to 5 seconds) for receiving exceeds 
    3027             ->  if the server socket connection fails for any reason besides connection closing 
    3028                     during receiving http content 
    3029         HttpContentLengthError: 
    3030             ->  if content length exceeds(default set to 4096 kb(4194304 charactors)) 
    3031                     (ONLY WORKS CONTENT LENGTH IS GIVEN BY THE HTTP SERVER) 
    3032             ->  If the total received length is not the same as the content length(this check will fail 
    3033                      if the content length isnt given) 
    3034             ->  If read is called with limits and it returns a empty string but the total read is                                       
    3035                      not equal to the content length(this check will fail if the content length isnt given) 
    3036                       
    3037         HttpStatuscodeError: 
    3038             if the http response status code isnt ok or redirect, it will raise an exception 
    3039             depending up on the http protocol status number   
    3040  
    3041  
    3042     <Side Effects> 
    3043        None 
    3044  
    3045  
    3046     <Returns> 
    3047       returns the content of http server in a form of string or an empty string if the content is completely read. 
    3048     """ 
    3049      
    3050     # raises an exception, if read is called after the filelikeobj is closed  
    3051     if self.fileobjclosed == True: 
    3052       raise HttpUserInputError('Http Error: filelikeobj is closed') 
    3053  
    3054     # if read is called after all http content received return an empty string 
    3055     if self.totalcontentisreceived: 
    3056       return '' 
    3057     
    3058     # check if limit is given 
    3059     if limit == None:  
    3060       readhaslimit = False  
    3061       left_to_read = 1024 
    3062     else:  
    3063     # check if limit is a valid number 
    3064       if not type(limit) == int: 
    3065         raise HttpUserInputError('User input Error: given a none int to receive' + str(e))   
    3066       elif limit < 0: 
    3067         # raise an exception if limit is a negative number 
    3068         raise HttpUserInputError('User input Error: given a negative number to receive, given: ' + str(limit)) 
    3069       readhaslimit = True 
    3070       left_to_read = limit   
    3071  
    3072     # set a timeout for receiveing content from server  
    3073     self.sock.settimeout(self.content_timeout) 
    3074  
    3075     # if limit is given it will receiveby subtracting what is left until the limit is reached  
    3076     # if limit isnt given it will receive1024 until the server closes connection        
    3077     httpcontent = '' 
    3078     while True: 
    3079       try: 
    3080         content = self.sock.recv(left_to_read) 
    3081  
    3082       except SocketTimeoutError: 
    3083         # raise an exception if receiveis taking too long to respond 
    3084         self.sock.close()   
    3085         raise HttpContentReceivingError('Timeout Error on receiving content: server taking too long to send content')   
    3086  
    3087       except Exception , e: 
    3088         # socket closed - signal for when the server is done sending content 
    3089         # if there is any other exceptions close connection and raise an error   
    3090         if 'Socket closed' not in str(e):  
    3091           self.sock.close()             
    3092           raise HttpContentReceivingError('Error on receiving content:' + str(e)) 
    3093         self.totalcontentisreceived = True 
    3094         break 
    3095  
    3096       else: 
    3097         # By default, httpretrieve permits content length to be less than 4,096 kilobytes(4194304 charactors) 
    3098         if len(content) >= self.httpcontent_limit: 
    3099           raise HttpContentLengthError('content length exceeded ' + self.httpcontent_limit) 
    3100  
    3101         # add what is received 
    3102         httpcontent += content 
    3103         if readhaslimit: 
    3104           # keep subtracting what is left to receieve until it reachs the given limit amount 
    3105           self.totalread += len(content) 
    3106           if len(content) == left_to_read: 
    3107             break 
    3108           else: 
    3109             left_to_read -= len(content) 
    3110  
    3111     # check if there was an error during reciving http content 
    3112     self._check_recieving_error(readhaslimit, httpcontent) 
    3113  
    3114     return httpcontent 
    3115    
    3116  
    3117   def close(self): 
    3118     """ 
    3119     <Purpose> 
    3120       close the file like object  
    3121  
    3122     <Arguments> 
    3123       None 
    3124     
    3125     <Exceptions> 
    3126       None  
    3127  
    3128     <Side Effects> 
    3129       closes socket connection for the http client to http server 
    3130  
    3131     <Returns> 
    3132       Nothing  
    3133     """    
    3134     self.fileobjclosed = True# flag used to raise an exception if the file like object is called after closed 
    3135     self.sock.close() 
    3136  
    3137  
    3138   def _check_recieving_error(self, readhaslimit, httpcontent): 
    3139     if len(httpcontent) == 0: 
    3140       self.sock.close() 
    3141       raise HttpContentLengthError('Error on recieving content: received a http header but didnt receive any http content') 
    3142      
    3143     if self.contentlengthisknown:                     
    3144       # if limit is given and content length is given and total content is received, check the total read equals the content length     
    3145       if readhaslimit and self.totalcontentisreceived: 
    3146         if self.totalread != self.contentlength: 
    3147           self.sock.close()                 
    3148           raise HttpContentLengthError('Total length read with limit did not match the content length: total read: ' + str(self.totalread) + ' content length: ' + str(self.contentlength)) 
    3149  
    3150       # if called read without limit and content length is given; check if the received length is the same as the content length        
    3151       if readhaslimit == False: 
    3152         if len(httpcontent) != self.contentlength: 
    3153           self.sock.close() 
    3154           raise HttpContentLengthError('Total received length did not match the content length: received: ' + str(len(httpcontent)) + ' content length : ' + str(self.contentlength))      
    3155  
    3156  
    3157  
    3158  
    3159  
    3160 def _httpretrieve_parse_given_url(url): 
    3161   # checks if the URL is in the right format and returns a string of host, port, path and query by parsing the URL   
    3162   try: 
    3163    # returns a dictionary of {scheme, netloc, path, quer, fragment, username, password, hostname and port} form the url                      
    3164     urlparse = urlparse_urlsplit(url)   
    3165   except Exception, e: 
    3166     raise HttpUserInputError('Given URL error: ' + str(e)) 
    3167   else: 
    3168     # check if the protocol is http  
    3169     if urlparse['scheme'] != 'http': 
    3170       raise HttpUserInputError('Given URL error: the given protocol ' + urlparse['scheme'] + ' isnt supported')        
    3171     if urlparse['hostname'] == None: 
    3172       raise HttpUserInputError('Given URL error: host name is not given')  
    3173  
    3174     # get only the host path, port, query from the urlparse dictionary 
    3175     host = urlparse['hostname'] 
    3176     path = urlparse['path'] 
    3177     query = urlparse['query'] 
    3178      
    3179     # use default port 80 if the port isnt given                     
    3180     if urlparse['port'] == None: 
    3181       port = 80  
    3182     else: 
    3183       port = urlparse['port'] 
    3184  
    3185     return host, port, path, query 
    3186  
    3187  
    3188  
    3189  
    3190  
    3191 def _httpretrieve_buildhttprequest(http_header, port, host, path, url_query, dict_query, http_post): 
    3192   # send http request to the http web server using socket connection   
    3193    
    3194   if http_post != None: 
    3195     # there is a posted data, thus use http POST command 
    3196     
    3197     # check if the given post data is valid 
    3198     if not type(http_post) == dict: 
    3199       raise HttpUserInputError('The given http_post is not a dictionary, given: ' + str(type(http_post))) 
    3200  
    3201     # change the given http post dictionary into a encoded post data with a key and val   
    3202     try:  
    3203       http_post = urllib_quote_parameters(http_post) 
    3204     except Exception, e: 
    3205       raise HttpUserInputError('Error encoding the given http post dictionary ' +  str(http_post) + str(e)) 
    3206  
    3207  
    3208     # build the main http request header which includes the GET/POST and the Host name field 
    3209     httpheader = _httpretrieve_httprequestmain_header('POST', url_query, dict_query, path, host, port) 
    3210  
    3211     # if given add a client http header to the request 
    3212     httpheader += _httpretrieve_parse_clienthttpheader(http_header) 
    3213  
    3214     # indicate the http post content length 
    3215     httpheader += 'Content-Length: ' + str(len(http_post)) + '\r\n'   
    3216     # add a new line to indicate that the http header is done and the http post is followed. 
    3217     httpheader += '\r\n' 
    3218     # include the posted data after the http header empty line 
    3219     httpheader += http_post 
    3220          
    3221  
    3222   else: 
    3223     # there is no posted data, use http GET method    
    3224     httpheader = _httpretrieve_httprequestmain_header('GET', url_query, dict_query, path, host, port) 
    3225     # if http header is given, add client headers to the request  
    3226     httpheader += _httpretrieve_parse_clienthttpheader(http_header) 
    3227     # add a new line to indicate that the header request is complete 
    3228     httpheader += '\r\n' 
    3229  
    3230   # return header with a new line which is signal for http header is done  
    3231   return httpheader  
    3232  
    3233  
    3234  
    3235  
    3236 def _httpretrieve_httprequestmain_header(http_command, url_query, dict_query, path, host, port): 
    3237   # builds the first two main http request headers which include the GET/POST and the HOST name 
    3238    
    3239   # before building the httprequest make sure there isnt two fields of query given by the client 
    3240   if url_query != '' and dict_query != None: 
    3241     # cant have two different fields with query 
    3242     raise HttpUserInputError('Cant input a http query with the url and an extra parameter dictionary with a http query') 
    3243  
    3244   elif dict_query != None: 
    3245     # user has given a http query  
    3246     try:  
    3247       encoded_query = '?' + urllib_quote_parameters(dict_query) 
    3248     except Exception, e: 
    3249       raise HttpUserInputError('Error encoding the given http query dictionary ' +  str(dict_query) + str(e)) 
    3250  
    3251   elif url_query != '': 
    3252     # if there is a query include the query on the main header('?' is used as a seperater between path) 
    3253     encoded_query = '?' + url_query 
    3254   else: 
    3255     # there is no query 
    3256     encoded_query = '' 
    3257      
    3258  
    3259   # if there is no path add a '/' on the request and if there is a path use the given path 
    3260   addpath = '/' 
    3261   if path != '': 
    3262     addpath = path 
    3263  
    3264   # FIRST header which includes the POST/GET request   
    3265   main_httpheader = http_command + ' ' + addpath + encoded_query + ' HTTP/1.0\r\n' 
    3266  
    3267  
    3268   # if port is 80, dont need to include the port upon request 
    3269   addport = '' 
    3270   if port != 80: 
    3271     # if the port is not 80 the host needs to include the port number on Host header 
    3272     # (':' is used as a separater between host and port) 
    3273     addport = ':' + str(port) 
    3274  
    3275   # SECOND line of the header request which include the host name with port if the port is not 80    
    3276   main_httpheader += 'Host: ' + host + addport + '\r\n' 
    3277  
    3278   # return the firs two lines of the http request  
    3279   return main_httpheader 
    3280  
    3281  
    3282  
    3283  
    3284 def _httpretrieve_parse_clienthttpheader(http_header): 
    3285   # builds a http header from the given http header dictionary 
    3286   if http_header == None: 
    3287     # if the http header isnt given return a empty string 
    3288     return '' 
    3289   elif not type(http_header) == dict: 
    3290     # raise an exception if the http header isnt dictionary 
    3291     raise HttpUserInputError('The given http header is not a dictionary, given: ' + str(type(http_header))) 
    3292   else:  
    3293     # take the given key and val from the http_header dictionary and add them to the http header with 
    3294     # correct http format 
    3295     clienthttpheader = '' 
    3296     for key, val in http_header.items(): 
    3297       # make simple checks on the key and val  
    3298       if not type(key) == str: 
    3299         raise HttpUserInputError('The given http header key isnt str given ' + str(type(key))) 
    3300       if not type(val) == str: 
    3301         raise HttpUserInputError('The given http header value isnt str given ' + str( type(val))) 
    3302       if key == '' or val == '': 
    3303         # raise an exception if the key or value is a empty field 
    3304         raise HttpUserInputError('The given empty http header feild of key or value') 
    3305       if key.capitalize() != key: 
    3306         raise HttpUserInputError('The given http header key is not capitalized, given: ' + key) 
    3307  
    3308       # add the key and value to the http header field 
    3309       clienthttpheader += key + ' : ' + val + '\r\n'   
    3310  
    3311     # return the string of the http header   
    3312     return clienthttpheader 
    3313  
    3314  
    3315  
    3316  
    3317 def _httpretrieve_sendhttprequest(sock, httpheader): 
    3318   # send the request, and if there is any error raise an excetion 
    3319   try: 
    3320     sock.send(httpheader) 
    3321   except Exception, e: 
    3322     if 'Socket closed' not in str(e):  
    3323       sock.close() 
    3324     raise HttpConnectionError('Connection error: on sending http request to server ' + str(e)) 
    3325  
    3326  
    3327  
    3328    
    3329 def _httpretrieve_receive_httpheader(sock, header_timeout, httpheader_limit): 
    3330   # receives the http header leaving alone rest of the http response and returns in list 
    3331   # of each header as a line. default httpheader limit is set to 8 kb                         
    3332  
    3333   # set a time out if the server fails to send http header  
    3334   sock.settimeout(header_timeout) 
    3335  
    3336   httpheader_received = 0  
    3337   httpheader = ''  
    3338   while True: 
    3339     # receive until a empty line (\n\n or \r\n\r\n ) which separates the 
    3340     # http header from the http content 
    3341     if '\r\n\r\n' in httpheader: 
    3342       # return split to return a list of httpheader lines 
    3343       return httpheader.split('\r\n') 
    3344     if '\n\n' in httpheader: 
    3345       # return split to return a list of httpheader lines 
    3346       return httpheader.split('\n') 
    3347  
    3348     if httpheader_limit == httpheader_received: 
    3349       sock.close()                   
    3350       raise HttpHeaderFormatError('Http header length Error: The http header is too long, exceeded 8 kb') 
    3351                          
    3352     try: 
    3353       # receive one character at a time inorder to check for the empty line 
    3354       content = sock.recv(1) 
    3355       # keep track of the received characters to raise an exception if the limit is exceeded                   
    3356       httpheader_received += 1                   
    3357  
    3358     except SocketTimeoutError: 
    3359       raise HttpHeaderReceivingError('Timeout Error on receiving header: server taking too long to send http header') 
    3360     except Exception, e: 
    3361       sock.close()  
    3362       raise HttpHeaderReceivingError('Error on recieving http header: ' + str(e)) 
    3363     else: 
    3364       # if there was not receiving error add keep on adding the receieved content 
    3365       httpheader += content 
    3366  
    3367  
    3368  
    3369  
    3370 def _httpretrieve_get_httpstatuscode(httpHeaderLines):  
    3371   # checks if the http status code is valid and return the status number and msg  
    3372  
    3373   # http response header includes 3 "words": HTTP<version> http_status_number http_status_msg  
    3374   httpstatusheader = httpHeaderLines[0] 
    3375   headersplit = httpstatusheader.split(' ', 2) 
    3376  
    3377   # length of the header has to be 3 or greater because depending up on the http_status_msg might be more than one word 
    3378   if len(headersplit) != 3: 
    3379     raise HttpHeaderFormatError('Invalid Http header status code format: Correct format is HTTP<version> http_status_number http_status_msg: Given '  + httpstatusheader) 
    3380   if not httpstatusheader.startswith('HTTP'): 
    3381     raise HttpHeaderFormatError('Invalid Http header status code format: Http header status code should start of with HTTP<version> but given: '  + httpstatusheader) 
    3382  
    3383   # the first split is the http version 
    3384   http_version = headersplit[0]                       
    3385  
    3386   # check if http_status_number is valid int 
    3387   try:  
    3388     int(headersplit[1]) 
    3389   except ValueError, e: 
    3390     raise HttpHeaderFormatError('Invalid Http header status code format: Status number should be a int, Given: ' + str(headersplit[1]) + str(e)) 
    3391   else: 
    3392     http_status_number = headersplit[1] 
    3393    
    3394   # what ever is left is the http status msg 
    3395   http_status_msg = headersplit[2] 
    3396    
    3397   # return the values 
    3398   return http_status_number, http_status_msg 
    3399  
    3400  
    3401  
    3402  
    3403 def _httpretrieve_receive_httperror_content(sock, contentlength): 
    3404   # receives the http error content which is located after the http error header                         
    3405   httperror_content = ''  
    3406   while True: 
    3407     try: 
    3408       content = sock.recv(1024)                   
    3409  
    3410     except SocketTimeoutError: 
    3411       raise HttpContentReceivingError('Timeout Error on receiving http error conent: server taking too long to send http error content') 
    3412     except Exception, e: 
    3413       # socket closed - signal for when the server is done sending error content 
    3414       # if there is any other exceptions close connection and raise an error besides socket closing   
    3415       if 'Socket closed' not in str(e):  
    3416         sock.close()             
    3417         raise HttpContentReceivingError('Error on receiving http error content: ' + str(e)) 
    3418       break 
    3419          
    3420     else: 
    3421       # if there was not a receiving error keep on adding the receieved content 
    3422       httperror_content += content 
    3423  
    3424   # return the received http error content. If the content length is given check 
    3425   # if the content length maches the received content 
    3426   if contentlength != None: 
    3427     if contentlength != len(httperror_content): 
    3428       raise HttpContentLengthError('Error on receiving http error content: received conent length: ' + str(len(httperror_content)) + ' actual content length: ' + str(contentlength))    
    3429   return httperror_content       
    3430  
    3431  
    3432  
    3433      
    3434 def _httpretrieve_raise_httpstatuscode_error(http_status_number, http_status_msg, http_errorcontent):  
    3435   # raises an exception using the http_status_number 1xx for Informational, 2xx for Success 3xx for Redirection, 
    3436   # 4xx for Client Error, and 5xx Server Error 
    3437   
    3438   # raise a detailed error message explaining the http_status_number and http_status_msg for popular http errors   
    3439   if http_status_number == '202': 
    3440     raise HttpError202('Http response error: ' + http_status_number + ' ' + http_status_msg +  ' http proccesing not responding. Http error content: ' + http_errorcontent)       
    3441   elif http_status_number == '204': 
    3442     raise HttpError204('Http response error: ' + http_status_number + ' ' + http_status_msg +  ' thier is no http body content. Http error content: ' + http_errorcontent)  
    3443   elif http_status_number == '300': 
    3444     raise HttpError300('Http response error: ' + http_status_number + ' ' + http_status_msg +  ' multiple redirect isnt suported. Http error content: ' + http_errorcontent)                 
    3445   elif http_status_number == '404': 
    3446     raise HttpError404('Http response error: ' + http_status_number + ' ' + http_status_msg +  ' cant find anything matching the given url. Http error content: ' + http_errorcontent)                 
    3447   elif http_status_number == '403': 
    3448     raise HttpError403('Http response error: ' + http_status_number + ' ' + http_status_msg +  ' the request was illegal. Http error content: ' + http_errorcontent)                 
    3449   elif http_status_number == '400': 
    3450     raise HttpError400('Http response error: ' + http_status_number + ' ' + http_status_msg +  ' the request contians bad syntex. Http error content: ' + http_errorcontent)                 
    3451   elif http_status_number == '500': 
    3452     raise HttpError500('Http response error: ' + http_status_number + ' ' + http_status_msg +  ' The server encountered an unexpected condition. Http error content: ' + http_errorcontent)                  
    3453   elif http_status_number == '502': 
    3454     raise HttpError502('Http response error: ' + http_status_number + ' ' + http_status_msg +  ' acting like a gateway received an invalid response. Http error content: ' + http_errorcontent)                 
    3455  
    3456   # if the http number wasnt any of the popular http error msgs, raise an exception using 
    3457   # the defualt http status number with http status msg   
    3458   elif http_status_number >= '100' and http_status_number < '200':  
    3459     raise HttpError1xx('Http response error: Information ' + http_status_number + ' ' + http_status_msg + '.Http error content: ' + http_errorcontent)  
    3460   elif http_status_number > '200' and http_status_number < '300':  
    3461     raise HttpError2xx('Http response error: success error ' + http_status_number + ' ' + http_status_msg + '.Http error content: ' + http_errorcontent) 
    3462   elif http_status_number >= '300' and http_status_number < '400':  
    3463     raise HttpError3xx('Http response error: Redirection error' + http_status_number + ' ' + http_status_msg + '.Http error content: ' + http_errorcontent) 
    3464   elif http_status_number >= '400' and http_status_number < '500':  
    3465     raise HttpError4xx('Http response error: client error ' + http_status_number + ' ' + http_status_msg + '.Http error content: ' + http_errorcontent)   
    3466   elif http_status_number >= '500' and http_status_number < '600':  
    3467     raise HttpError5xx('Http response error: server error: ' + http_status_number + ' ' + http_status_msg + '.Http error content: ' + http_errorcontent) 
    3468   else: 
    3469     raise HttpStatusCodeError('Http response error: invalid http status response, given ' + http_status_number + '.Http error content: ' + http_errorcontent)   
    3470  
    3471  
    3472  
    3473  
    3474 def _httpretrieve_httpredirect(httpheaderlines): 
    3475   # given http header retruns the redirect location    
    3476  
    3477   # if there is a redirect location given by the server it will look like 
    3478   # eg.'Location: http://www.google.com' 
    3479   for headerline in httpheaderlines: 
    3480     if headerline.startswith('Location:'): 
    3481       # if found redirect need to strip out 'Location: ' to return the url    
    3482       redirect = headerline[len('Location: '):] 
    3483                          
    3484       # check if the redirect has given a location then return it 
    3485       if len(redirect) == 0: 
    3486         raise HttpHeaderFormatError('Http header redirection format error: http server gave a redierect location with no URL') 
    3487       return redirect 
    3488                          
    3489   # there wasn't a redirect location given 
    3490   raise HttpHeaderFormatError('Http header redirection format error: http redirect header didnt include the location')   
    3491  
    3492  
    3493  
    3494  
    3495 def _httpretrieve_get_contentlength(httpheaderlines): 
    3496   # returns the content legth if given or returns None if not given by server 
    3497  
    3498   # if there is a content length given by the server it will look like 
    3499   # eg.'Content-Length: 34' 
    3500   for headerline in httpheaderlines: 
    3501     if headerline.startswith('Content-Length:'): 
    3502       # if found content length need to strip out 'Content-Length: ' to return the length    
    3503        
    3504       try:  
    3505         contentlength = int(headerline[len('Content-Length: '):]) 
    3506       except ValueError, e: 
    3507         raise HttpHeaderFormatError('Http header Content-Length format error: http server provided content length that isnt a int ' + str(e))                 
    3508  
    3509        
    3510       # check if the content length is valid and retrun it  
    3511       if contentlength <= 0:                 
    3512         raise HttpHeaderFormatError('Http header Content-Length format error: provided content length with invalid number ' + str(contentlength)) 
    3513       else: 
    3514         return contentlength 
    3515                          
    3516   # there wasn't a content-length line or the content length was given but didnt give a int  
    3517   return None 
    3518  
    3519  
    3520  
    3521  
    3522    
    3523 def urllib_quote(string, safe="/"): 
    3524   """ 
    3525   <Purpose> 
    3526     Encode a string such that it can be used safely in a URL or XML 
    3527     document. 
    3528  
    3529   <Arguments> 
    3530     string: 
    3531            The string to urlencode. 
    3532  
    3533     safe (optional): 
    3534            Specifies additional characters that should not be quoted -- 
    3535            defaults to "/". 
    3536  
    3537   <Exceptions> 
    3538     TypeError if the safe parameter isn't an enumerable. 
    3539  
    3540   <Side Effects> 
    3541     None. 
    3542  
    3543   <Returns> 
    3544     Urlencoded version of the passed string. 
    3545   """ 
    3546  
    3547   resultstr = "" 
    3548  
    3549   # We go through each character in the string; if it's not in [0-9a-zA-Z] 
    3550   # we wrap it. 
    3551  
    3552   safeset = set(safe) 
    3553  
    3554   for char in string: 
    3555     asciicode = ord(char) 
    3556     if (asciicode >= ord("0") and asciicode <= ord("9")) or \ 
    3557         (asciicode >= ord("A") and asciicode <= ord("Z")) or \ 
    3558         (asciicode >= ord("a") and asciicode <= ord("z")) or \ 
    3559         asciicode == ord("_") or asciicode == ord(".") or \ 
    3560         asciicode == ord("-") or char in safeset: 
    3561       resultstr += char 
    3562     else: 
    3563       resultstr += "%%%02X" % asciicode 
    3564  
    3565   return resultstr 
    3566  
    3567  
    3568  
    3569  
    3570 def urllib_quote_plus(string, safe=""): 
    3571   """ 
    3572   <Purpose> 
    3573     Encode a string to go in the query fragment of a URL. 
    3574  
    3575   <Arguments> 
    3576     string: 
    3577            The string to urlencode. 
    3578  
    3579     safe (optional): 
    3580            Specifies additional characters that should not be quoted -- 
    3581            defaults to the empty string. 
    3582  
    3583   <Exceptions> 
    3584     TypeError if the safe parameter isn't a string. 
    3585  
    3586   <Side Effects> 
    3587     None. 
    3588  
    3589   <Returns> 
    3590     Urlencoded version of the passed string. 
    3591   """ 
    3592  
    3593   return urllib_quote(string, safe + " ").replace(" ", "+") 
    3594  
    3595  
    3596  
    3597  
    3598 def urllib_unquote(string): 
    3599   """ 
    3600   <Purpose> 
    3601     Unquote a urlencoded string. 
    3602  
    3603   <Arguments> 
    3604     string: 
    3605            The string to unquote. 
    3606  
    3607   <Exceptions> 
    3608     ValueError thrown if the last wrapped octet isn't a valid wrapped octet 
    3609     (i.e. if the string ends in "%" or "%x" rather than "%xx". Also throws 
    3610     ValueError if the nibbles aren't valid hex digits. 
    3611  
    3612   <Side Effects> 
    3613     None. 
    3614  
    3615   <Returns> 
    3616     The decoded string. 
    3617   """ 
    3618  
    3619   resultstr = "" 
    3620  
    3621   # We go through the string from end to beginning, looking for wrapped 
    3622   # octets. When one is found we add it (unwrapped) and the following 
    3623   # string to the resultant string, and shorten the original string. 
    3624  
    3625   while True: 
    3626     lastpercentlocation = string.rfind("%") 
    3627     if lastpercentlocation < 0: 
    3628       break 
    3629  
    3630     wrappedoctetstr = string[lastpercentlocation+1:lastpercentlocation+3] 
    3631     if len(wrappedoctetstr) != 2: 
    3632       raise ValueError("Quoted string is poorly formed") 
    3633  
    3634     resultstr = \ 
    3635         chr(int(wrappedoctetstr, 16)) + \ 
    3636         string[lastpercentlocation+3:] + \ 
    3637         resultstr 
    3638     string = string[:lastpercentlocation] 
    3639  
    3640   resultstr = string + resultstr 
    3641   return resultstr 
    3642  
    3643  
    3644  
    3645  
    3646 def urllib_unquote_plus(string): 
    3647   """ 
    3648   <Purpose> 
    3649     Unquote the urlencoded query fragment of a URL. 
    3650  
    3651   <Arguments> 
    3652     string: 
    3653            The string to unquote. 
    3654  
    3655   <Exceptions> 
    3656     ValueError thrown if the last wrapped octet isn't a valid wrapped octet 
    3657     (i.e. if the string ends in "%" or "%x" rather than "%xx". Also throws 
    3658     ValueError if the nibbles aren't valid hex digits. 
    3659  
    3660   <Side Effects> 
    3661     None. 
    3662  
    3663   <Returns> 
    3664     The decoded string. 
    3665   """ 
    3666  
    3667   return urllib_unquote(string.replace("+", " ")) 
    3668  
    3669  
    3670  
    3671  
    3672 def urllib_quote_parameters(dictionary): 
    3673   """ 
    3674   <Purpose> 
    3675     Encode a dictionary of (key, value) pairs into an HTTP query string or 
    3676     POST body (same form). 
    3677  
    3678   <Arguments> 
    3679     dictionary: 
    3680            The dictionary to quote. 
    3681  
    3682   <Exceptions> 
    3683     None. 
    3684  
    3685   <Side Effects> 
    3686     None. 
    3687  
    3688   <Returns> 
    3689     The quoted dictionary. 
    3690   """ 
    3691  
    3692   quoted_keyvals = [] 
    3693   for key, val in dictionary.items(): 
    3694     quoted_keyvals.append("%s=%s" % (urllib_quote(key), urllib_quote(val))) 
    3695  
    3696   return "&".join(quoted_keyvals) 
    3697  
    3698  
    3699  
    3700  
    3701 def urllib_unquote_parameters(string): 
    3702   """ 
    3703   <Purpose> 
    3704     Decode a urlencoded query string or POST body. 
    3705  
    3706   <Arguments> 
    3707     string: 
    3708            The string to decode. 
    3709  
    3710   <Exceptions> 
    3711     ValueError if the string is poorly formed. 
    3712  
    3713   <Side Effects> 
    3714     None. 
    3715  
    3716   <Returns> 
    3717     A dictionary mapping keys to values. 
    3718   """ 
    3719  
    3720   keyvalpairs = string.split("&") 
    3721   res = {} 
    3722  
    3723   for quotedkeyval in keyvalpairs: 
    3724     # Throw ValueError if there is more or less than one '='. 
    3725     quotedkey, quotedval = quotedkeyval.split("=") 
    3726     key = urllib_unquote(quotedkey) 
    3727     val = urllib_unquote(quotedval) 
    3728     res[key] = val 
    3729  
    3730   return res 
    3731  
     8include httpretrieve.repy 
     9include registerhttpcallback.repy 
     10include urllib.repy 
    373211 
    373312