Please note that the CVS and issue trackers have moved to GitHub. These Trac pages are no longer kept up-to-date.

root/seattle/trunk/squeeze.py@5637

Revision 1978, 12.7 KB (checked in by richard, 10 years ago)

updated setup.py: try commands 'test', 'demokit', 'build', and 'clean'

Line 
1#!/usr/bin/env python
2
3"""
4<Modifier>
5  richard@jordan.ham-radio-op.net
6
7<Purpose>
8  With this script, we can generate one file, portable, compressed
9  versions of our popular scripts like Repy, Seash, and Repypp.
10
11<Change log>
12  I had to fix a few bugs in this unmaintained, 1998 library:
13
14   * Dos and Unix line endings.  To use Python's compile() all lines must
15     end with '\n'.  According the the Python docs,
16     {{{
17       Note
18       When compiling a string with multi-line statements, line endings
19       must be represented by a single newline character ('\n'), and the
20       input must be terminated by at least one newline character. If line
21       endings are represented by '\r\n', use str.replace() to change them
22       into '\n' (Source: http://docs.python.org/library/functions.html).
23     }}}
24     I replace all '\r\n' and add a '\n' to the end of each file before
25     calling compile().
26
27   * Namespace collision between Repy's misc module and Python's misc module
28     (from the compiler package).  I use a horrible hack for now.
29
30   * Use of __name__ = '__main__'.  We want the main module to execute as if
31     it's been called as a script.  I set the main module name to '__main__'.
32
33<Original Readme>
34  For more information on this library, see:
35    http://effbot.org/zone/squeeze.htm
36
37  This utility works for Python 1.4 and later.
38
39  Enjoy /F
40  fredrik@pythonware.com
41  http://www.pythonware.com
42"""
43#
44# SQUEEZE
45# $Id$
46#
47# squeeze a python program
48#
49# installation:
50# - use this script as is, or squeeze it using the following command:
51#
52# python squeezeTool.py -1su -o squeeze -b squeezeTool squeezeTool.py
53#
54# notes:
55# - this is pretty messy.  make sure to test everything carefully
56#   if you change anything
57#
58# - the name "squeeze" is taken from an ABC800 utility which did
59#   about the same thing with Basic II bytecodes.
60#
61# history:
62# 1.0 1997-04-22 fl   Created
63# 1.1 1997-05-25 fl   Added base64 embedding option (-1)
64#     1997-05-25 fl   Check for broken package file
65# 1.2 1997-05-26 fl   Support uncompressed packages (-u)
66# 1.3 1997-05-27 fl   Check byte code magic, eliminated StringIO, etc.
67# 1.4 1997-06-04 fl   Removed last bits of white space, removed try/except
68# 1.5 1997-06-17 fl   Added squeeze archive capabilities (-x)
69# 1.6 1998-05-04 fl   Minor fixes in preparation for public source release
70#
71# reviews:
72#       "Fredrik Lundh is a friggin genius"
73#       -- Aaron Watters, author of 'Internet Programming with Python'
74#
75#       "I agree ... this is a friggin Good Thing"
76#       -- Paul Everitt, Digital Creations
77#
78# Copyright (c) 1997 by Fredrik Lundh.
79# Copyright (c) 1997-1998 by Secret Labs AB
80#
81# info@pythonware.com
82# http://www.pythonware.com
83#
84# --------------------------------------------------------------------
85# Permission to use, copy, modify, and distribute this software and
86# its associated documentation for any purpose and without fee is
87# hereby granted.  This software is provided as is.
88# --------------------------------------------------------------------
89#
90VERSION = "1.6/1998-05-04"
91MAGIC   = "[SQUEEZE]"
92
93import base64, imp, marshal, os, string, sys, md5
94
95# --------------------------------------------------------------------
96# usage
97
98def usage():
99    print
100    print "SQUEEZE", VERSION, "(c) 1997-1998 by Secret Labs AB"
101    print """\
102Convert a Python application to a compressed module package.
103
104Usage: squeeze [-1ux] -o app [-b start] modules... [-d files...]
105
106This utility creates a compressed package file named "app.pyz", which
107contains the given module files.  It also creates a bootstrap script
108named "app.py", which loads the package and imports the given "start"
109module to get things going.  Example:
110
111        squeeze -o app -b appMain app*.py
112
113The -1 option tells squeeze to put the package file inside the boot-
114strap script using base64 encoding.  The result is a single text file
115containing the full application.
116
117The -u option disables compression.  Otherwise, the package will be
118compressed using zlib, and the user needs zlib to run the resulting
119application.
120
121The -d option can be used to put additional files in the package file.
122You can access these files via "__main__.open(filename)" (returns a
123StringIO file object).
124
125The -x option can be used with -d to create a self-extracting archive,
126instead of a package.  When the resulting script is executed, the
127data files are extracted.  Omit the -b option in this case.
128"""
129    sys.exit(1)
130
131
132# --------------------------------------------------------------------
133# squeezer -- collect squeezed modules
134
135class Squeezer:
136
137    def __init__(self):
138
139        self.rawbytes = self.bytes = 0
140        self.modules = {}
141
142    def addmodule(self, file):
143
144        if file[-1] == "c":
145            file = file[:-1]
146
147        m = os.path.splitext(os.path.split(file)[1])[0]
148
149        # read sourcefile
150        f = open(file)
151        codestring = f.read()
152        f.close()
153        codestring = codestring.replace('\r\n', '\n')
154        codestring += '\n'
155        if m == __start_name__:
156          codestring = "__name__ = '__main__'\n" + codestring
157
158        # dump to file
159        self.modules[m] = compile(codestring, file, "exec")
160
161    def adddata(self, file):
162
163        self.modules["+"+file] = open(file, "rb").read()
164
165    def getarchive(self):
166
167        # marshal our module dictionary
168        data = marshal.dumps(self.modules)
169        self.rawbytes = len(data)
170
171        # return (compressed) dictionary
172        if zlib:
173            data = zlib.compress(data, 9)
174        self.bytes = len(data)
175
176        return data
177
178    def getstatus(self):
179        return self.bytes, self.rawbytes
180
181
182# --------------------------------------------------------------------
183# loader (used in bootstrap code)
184
185loader = """
186import ihooks
187
188PYZ_MODULE = 64
189
190from imp import C_EXTENSION, PY_SOURCE, PY_COMPILED                 
191from imp import C_BUILTIN, PY_FROZEN, PKG_DIRECTORY               
192BUILTIN_MODULE = C_BUILTIN
193FROZEN_MODULE = PY_FROZEN
194
195class Loader(ihooks.ModuleLoader):
196
197    def __init__(self, modules):
198        self.__last = None
199        self.__modules = modules
200        return ihooks.ModuleLoader.__init__(self)
201
202    def find_module(self, name, path = None):
203        try:
204            self.__modules[name]
205            if self.__last == 'compiler' and name == 'misc':
206              raise KeyError
207            return None, None, (None, None, PYZ_MODULE)
208        except KeyError:
209            self.__last = name
210            return ihooks.ModuleLoader.find_module(self, name, path)
211
212    def load_module(self, name, stuff):
213  #    try:
214            file, filename, (suff, mode, type) = stuff
215            if type != PYZ_MODULE:
216              return ihooks.ModuleLoader.load_module(self, name, stuff)
217            #print "PYZ:", "import", name
218            code = self.__modules[name]
219            del self.__modules[name] # no need to keep this one around
220            m = self.hooks.add_module(name)
221            m.__file__ = filename
222            exec code in m.__dict__
223            return m
224#      except KeyError:
225 #           return ihooks.ModuleLoader.load_module(self, name, (None, None, (None, None, BUILTIN_MODULE)))
226
227def boot(name, fp, size, offset = 0):
228
229    global data
230
231    try:
232        import %(modules)s
233    except ImportError:
234        #print "PYZ:", "failed to load marshal and zlib libraries"
235        return # cannot boot from PYZ file
236    #print "PYZ:", "boot from", name+".PYZ"
237
238    # load archive and install import hook
239    if offset:
240        data = fp[offset:]
241    else:
242        data = fp.read(size)
243        fp.close()
244
245    if len(data) != size:
246        raise IOError, "package is truncated"
247
248    data = marshal.loads(%(data)s)
249
250    ihooks.install(ihooks.ModuleImporter(Loader(data)))
251"""
252
253loaderopen = """
254def open(name):
255    import StringIO
256    try:
257        return StringIO.StringIO(data["+"+name])
258    except KeyError:
259        raise IOError, (0, "no such file")
260"""
261
262loaderexplode = """
263
264def explode():
265    for k, v in data.items():
266        if k[0] == "+":
267            try:
268                open(k[1:], "wb").write(v)
269                print k[1:], "extracted ok"
270            except IOError, v:
271                print k[1:], "failed:", "IOError", v
272
273"""
274
275def getloader(data, zlib, package):
276
277    s = loader
278
279    if data:
280        if explode:
281            s = s + loaderexplode
282        else:
283            s = s + loaderopen
284
285    if zlib:
286        dict = {
287            "modules": "marshal, zlib",
288            "data":    "zlib.decompress(data)",
289            }
290    else:
291        dict = {
292            "modules": "marshal",
293            "data":    "data",
294            }
295
296    s = s % dict
297
298    return marshal.dumps(compile(s, "<package>", "exec"))
299
300
301# --------------------------------------------------------------------
302# Main
303# --------------------------------------------------------------------
304
305#
306# parse options
307
308import getopt, glob, sys
309
310try:
311    opt, arg = getopt.getopt(sys.argv[1:], "1b:o:suzxd")
312except: 
313  raise
314  usage()
315
316app = ""
317start = ""
318embed = 0
319zlib = 1
320explode = 0
321global __start_name__
322
323data = None
324
325for i, v in opt:
326    if i == "-o":
327        app = v
328    elif i == "-b":
329        start = "import " + v
330        __start_name__ = v
331    elif i == "-d":
332        data = 0
333    elif i == "-1":
334        embed = 1
335    elif i == "-z":
336        zlib = 1
337    elif i == "-u":
338        zlib = 0
339    elif i == "-x":
340        explode = 1
341        start = "explode()"
342
343print app, start
344
345if not app or not start:
346    usage()
347
348bootstrap = app + ".py"
349archive   = app + ".pyz"
350
351archiveid = app
352if explode:
353    archiveid = "this is a self-extracting archive. run the script to unpack"
354elif embed:
355    archiveid = "this is an embedded package"
356elif zlib:
357    archiveid = "this is a bootstrap script for a compressed package"
358else:
359    archiveid = "this is a bootstrap script for an uncompressed package"
360
361#
362# import compression library (as necessary)
363
364if zlib:
365    try:
366        import zlib
367    except ImportError:
368        print "You must have the zlib module to generate compressed archives."
369        print "Squeeze will create an uncompressed archive."
370        zlib = None
371
372#
373# avoid overwriting files not generated by squeeze
374
375try:
376    fp = open(bootstrap)
377    s = fp.readline()
378    s = fp.readline()
379    string.index(s, MAGIC)
380except IOError:
381    pass
382except ValueError:
383    print bootstrap, "was not created by squeeze.  You have to manually"
384    print "remove the file to proceed."
385    sys.exit(1)
386
387#
388# collect modules
389
390sq = Squeezer()
391for patt in arg:
392    if patt == "-d":
393        data = 0
394    else:
395        for file in glob.glob(patt):
396            if file != bootstrap:
397                if data is not None:
398                    print file, "(data)"
399                    sq.adddata(file)
400                    data = data + 1
401                else:
402                    print file
403                    sq.addmodule(file)
404
405package = sq.getarchive()
406size = len(package)
407
408#
409# get loader
410
411loader = getloader(data, zlib, package)
412
413if zlib:
414    zbegin, zend = "zlib.decompress(", ")"
415    zimport = 'try:import zlib\n'\
416        'except:raise RuntimeError,"requires zlib"\n'
417    loader = zlib.compress(loader, 9)
418else:
419    zbegin = zend = zimport = ""
420
421loaderlen = len(loader)
422
423magic = repr(imp.get_magic())
424version = string.split(sys.version)[0]
425
426magictest = 'import imp\n'\
427    's="requires python %s or bytecode compatible"\n'\
428    'if imp.get_magic()!=%s:raise RuntimeError,s' % (version, magic)
429
430#
431# generate script and package files
432
433if embed:
434
435    # embedded archive
436    data = base64.encodestring(loader + package)
437
438    fp = open(bootstrap, "w")
439    fp.write('''\
440#!/usr/bin/env python
441#%(MAGIC)s %(archiveid)s
442%(magictest)s
443%(zimport)simport base64,marshal
444s=base64.decodestring("""
445%(data)s""")
446exec marshal.loads(%(zbegin)ss[:%(loaderlen)d]%(zend)s)
447boot("%(app)s",s,%(size)d,%(loaderlen)d)
448%(start)s
449''' % locals())
450    bytes = fp.tell()
451
452else:
453
454    # separate archive file
455
456    fp = open(archive, "wb")
457
458    fp.write(loader)
459    fp.write(package)
460
461    bytes = fp.tell()
462
463    #
464    # create bootstrap code
465
466    fp = open(bootstrap, "w")
467    fp.write("""\
468#!/usr/bin/env python
469#%(MAGIC)s %(archiveid)s
470%(magictest)s
471%(zimport)simport marshal,sys,os
472for p in filter(os.path.exists,map(lambda p:os.path.join(p,"%(archive)s"),sys.path)):
473 f=open(p,"rb")
474 exec marshal.loads(%(zbegin)sf.read(%(loaderlen)d)%(zend)s)
475 boot("%(app)s",f,%(size)d)
476 break
477%(start)s # failed to load package
478""" % locals())
479    bytes = bytes + fp.tell()
480
481#
482# show statistics
483
484dummy, rawbytes = sq.getstatus()
485
486print "squeezed", rawbytes, "to", bytes, "bytes",
487print "(%d%%)" % (bytes * 100 / rawbytes)
Note: See TracBrowser for help on using the browser.