temp.py 7.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252
  1. #!/usr/bin/env python
  2. #
  3. # Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
  4. # Copyright (c) 2008-2016 California Institute of Technology.
  5. # Copyright (c) 2016-2023 The Uncertainty Quantification Foundation.
  6. # License: 3-clause BSD. The full license text is available at:
  7. # - https://github.com/uqfoundation/dill/blob/master/LICENSE
  8. """
  9. Methods for serialized objects (or source code) stored in temporary files
  10. and file-like objects.
  11. """
  12. #XXX: better instead to have functions write to any given file-like object ?
  13. #XXX: currently, all file-like objects are created by the function...
  14. __all__ = ['dump_source', 'dump', 'dumpIO_source', 'dumpIO',\
  15. 'load_source', 'load', 'loadIO_source', 'loadIO',\
  16. 'capture']
  17. import contextlib
  18. @contextlib.contextmanager
  19. def capture(stream='stdout'):
  20. """builds a context that temporarily replaces the given stream name
  21. >>> with capture('stdout') as out:
  22. ... print ("foo!")
  23. ...
  24. >>> print (out.getvalue())
  25. foo!
  26. """
  27. import sys
  28. from io import StringIO
  29. orig = getattr(sys, stream)
  30. setattr(sys, stream, StringIO())
  31. try:
  32. yield getattr(sys, stream)
  33. finally:
  34. setattr(sys, stream, orig)
  35. def b(x): # deal with b'foo' versus 'foo'
  36. import codecs
  37. return codecs.latin_1_encode(x)[0]
  38. def load_source(file, **kwds):
  39. """load an object that was stored with dill.temp.dump_source
  40. file: filehandle
  41. alias: string name of stored object
  42. mode: mode to open the file, one of: {'r', 'rb'}
  43. >>> f = lambda x: x**2
  44. >>> pyfile = dill.temp.dump_source(f, alias='_f')
  45. >>> _f = dill.temp.load_source(pyfile)
  46. >>> _f(4)
  47. 16
  48. """
  49. alias = kwds.pop('alias', None)
  50. mode = kwds.pop('mode', 'r')
  51. fname = getattr(file, 'name', file) # fname=file.name or fname=file (if str)
  52. source = open(fname, mode=mode, **kwds).read()
  53. if not alias:
  54. tag = source.strip().splitlines()[-1].split()
  55. if tag[0] != '#NAME:':
  56. stub = source.splitlines()[0]
  57. raise IOError("unknown name for code: %s" % stub)
  58. alias = tag[-1]
  59. local = {}
  60. exec(source, local)
  61. _ = eval("%s" % alias, local)
  62. return _
  63. def dump_source(object, **kwds):
  64. """write object source to a NamedTemporaryFile (instead of dill.dump)
  65. Loads with "import" or "dill.temp.load_source". Returns the filehandle.
  66. >>> f = lambda x: x**2
  67. >>> pyfile = dill.temp.dump_source(f, alias='_f')
  68. >>> _f = dill.temp.load_source(pyfile)
  69. >>> _f(4)
  70. 16
  71. >>> f = lambda x: x**2
  72. >>> pyfile = dill.temp.dump_source(f, dir='.')
  73. >>> modulename = os.path.basename(pyfile.name).split('.py')[0]
  74. >>> exec('from %s import f as _f' % modulename)
  75. >>> _f(4)
  76. 16
  77. Optional kwds:
  78. If 'alias' is specified, the object will be renamed to the given string.
  79. If 'prefix' is specified, the file name will begin with that prefix,
  80. otherwise a default prefix is used.
  81. If 'dir' is specified, the file will be created in that directory,
  82. otherwise a default directory is used.
  83. If 'text' is specified and true, the file is opened in text
  84. mode. Else (the default) the file is opened in binary mode. On
  85. some operating systems, this makes no difference.
  86. NOTE: Keep the return value for as long as you want your file to exist !
  87. """ #XXX: write a "load_source"?
  88. from .source import importable, getname
  89. import tempfile
  90. kwds.setdefault('delete', True)
  91. kwds.pop('suffix', '') # this is *always* '.py'
  92. alias = kwds.pop('alias', '') #XXX: include an alias so a name is known
  93. name = str(alias) or getname(object)
  94. name = "\n#NAME: %s\n" % name
  95. #XXX: assumes kwds['dir'] is writable and on $PYTHONPATH
  96. file = tempfile.NamedTemporaryFile(suffix='.py', **kwds)
  97. file.write(b(''.join([importable(object, alias=alias),name])))
  98. file.flush()
  99. return file
  100. def load(file, **kwds):
  101. """load an object that was stored with dill.temp.dump
  102. file: filehandle
  103. mode: mode to open the file, one of: {'r', 'rb'}
  104. >>> dumpfile = dill.temp.dump([1, 2, 3, 4, 5])
  105. >>> dill.temp.load(dumpfile)
  106. [1, 2, 3, 4, 5]
  107. """
  108. import dill as pickle
  109. mode = kwds.pop('mode', 'rb')
  110. name = getattr(file, 'name', file) # name=file.name or name=file (if str)
  111. return pickle.load(open(name, mode=mode, **kwds))
  112. def dump(object, **kwds):
  113. """dill.dump of object to a NamedTemporaryFile.
  114. Loads with "dill.temp.load". Returns the filehandle.
  115. >>> dumpfile = dill.temp.dump([1, 2, 3, 4, 5])
  116. >>> dill.temp.load(dumpfile)
  117. [1, 2, 3, 4, 5]
  118. Optional kwds:
  119. If 'suffix' is specified, the file name will end with that suffix,
  120. otherwise there will be no suffix.
  121. If 'prefix' is specified, the file name will begin with that prefix,
  122. otherwise a default prefix is used.
  123. If 'dir' is specified, the file will be created in that directory,
  124. otherwise a default directory is used.
  125. If 'text' is specified and true, the file is opened in text
  126. mode. Else (the default) the file is opened in binary mode. On
  127. some operating systems, this makes no difference.
  128. NOTE: Keep the return value for as long as you want your file to exist !
  129. """
  130. import dill as pickle
  131. import tempfile
  132. kwds.setdefault('delete', True)
  133. file = tempfile.NamedTemporaryFile(**kwds)
  134. pickle.dump(object, file)
  135. file.flush()
  136. return file
  137. def loadIO(buffer, **kwds):
  138. """load an object that was stored with dill.temp.dumpIO
  139. buffer: buffer object
  140. >>> dumpfile = dill.temp.dumpIO([1, 2, 3, 4, 5])
  141. >>> dill.temp.loadIO(dumpfile)
  142. [1, 2, 3, 4, 5]
  143. """
  144. import dill as pickle
  145. from io import BytesIO as StringIO
  146. value = getattr(buffer, 'getvalue', buffer) # value or buffer.getvalue
  147. if value != buffer: value = value() # buffer.getvalue()
  148. return pickle.load(StringIO(value))
  149. def dumpIO(object, **kwds):
  150. """dill.dump of object to a buffer.
  151. Loads with "dill.temp.loadIO". Returns the buffer object.
  152. >>> dumpfile = dill.temp.dumpIO([1, 2, 3, 4, 5])
  153. >>> dill.temp.loadIO(dumpfile)
  154. [1, 2, 3, 4, 5]
  155. """
  156. import dill as pickle
  157. from io import BytesIO as StringIO
  158. file = StringIO()
  159. pickle.dump(object, file)
  160. file.flush()
  161. return file
  162. def loadIO_source(buffer, **kwds):
  163. """load an object that was stored with dill.temp.dumpIO_source
  164. buffer: buffer object
  165. alias: string name of stored object
  166. >>> f = lambda x:x**2
  167. >>> pyfile = dill.temp.dumpIO_source(f, alias='_f')
  168. >>> _f = dill.temp.loadIO_source(pyfile)
  169. >>> _f(4)
  170. 16
  171. """
  172. alias = kwds.pop('alias', None)
  173. source = getattr(buffer, 'getvalue', buffer) # source or buffer.getvalue
  174. if source != buffer: source = source() # buffer.getvalue()
  175. source = source.decode() # buffer to string
  176. if not alias:
  177. tag = source.strip().splitlines()[-1].split()
  178. if tag[0] != '#NAME:':
  179. stub = source.splitlines()[0]
  180. raise IOError("unknown name for code: %s" % stub)
  181. alias = tag[-1]
  182. local = {}
  183. exec(source, local)
  184. _ = eval("%s" % alias, local)
  185. return _
  186. def dumpIO_source(object, **kwds):
  187. """write object source to a buffer (instead of dill.dump)
  188. Loads by with dill.temp.loadIO_source. Returns the buffer object.
  189. >>> f = lambda x:x**2
  190. >>> pyfile = dill.temp.dumpIO_source(f, alias='_f')
  191. >>> _f = dill.temp.loadIO_source(pyfile)
  192. >>> _f(4)
  193. 16
  194. Optional kwds:
  195. If 'alias' is specified, the object will be renamed to the given string.
  196. """
  197. from .source import importable, getname
  198. from io import BytesIO as StringIO
  199. alias = kwds.pop('alias', '') #XXX: include an alias so a name is known
  200. name = str(alias) or getname(object)
  201. name = "\n#NAME: %s\n" % name
  202. #XXX: assumes kwds['dir'] is writable and on $PYTHONPATH
  203. file = StringIO()
  204. file.write(b(''.join([importable(object, alias=alias),name])))
  205. file.flush()
  206. return file
  207. del contextlib
  208. # EOF