crochet-1.0.0/0000755000175000017500000000000012230523106013371 5ustar roaksoaxroaksoaxcrochet-1.0.0/MANIFEST.in0000644000175000017500000000021012224642433015130 0ustar roaksoaxroaksoaxinclude LICENSE include README.rst include requirements-dev.txt recursive-include docs * prune docs/_build recursive-include examples * crochet-1.0.0/README.rst0000644000175000017500000000642512230522546015076 0ustar roaksoaxroaksoaxCrochet: Use Twisted Anywhere! ============================== Crochet is an MIT-licensed library that makes it easier for blocking or threaded applications like Flask or Django to use the Twisted networking framework. Crochet provides the following features: * Runs Twisted's reactor in a thread it manages. * The reactor shuts down automatically when the process' main thread finishes. * Hooks up Twisted's log system to the Python standard library ``logging`` framework. Unlike Twisted's built-in ``logging`` bridge, this includes support for blocking `Handler` instances. * A blocking API to eventual results (i.e. ``Deferred`` instances). This last feature can be used separately, so Crochet is also useful for normal Twisted applications that use threads. .. image:: https://travis-ci.org/itamarst/crochet.png?branch=master :target: http://travis-ci.org/itamarst/crochet :alt: Build Status Documentation can be found on `Read The Docs`_. Bugs and feature requests should be filed at the project `Github page`_. .. _Read the Docs: https://crochet.readthedocs.org/ .. _Github page: https://github.com/itamarst/crochet/ Changelog --------- **1.0.0** Documentation: * Added section on use cases and alternatives. Thanks to Tobias Oberstein for the suggestion. Bug fixes: * Twisted does not have to be pre-installed to run ``setup.py``, thanks to Paul Weaver for bug report and Chris Scutcher for patch. * Importing Crochet does not have side-effects (installing reactor event) any more. * Blocking calls are interrupted earlier in the shutdown process, to reduce scope for deadlocks. Thanks to rmorehead for bug report. **0.9.0** New features: * Expanded and much improved documentation, including a new section with design suggestions. * New decorator ``@wait_for_reactor`` added, a simpler alternative to ``@run_in_reactor``. * Refactored ``@run_in_reactor``, making it a bit more responsive. * Blocking operations which would otherwise never finish due to reactor having stopped (``EventualResult.wait()`` or ``@wait_for_reactor`` decorated call) will be interrupted with a ``ReactorStopped`` exception. Thanks to rmorehead for the bug report. Bug fixes: * ``@run_in_reactor`` decorated functions (or rather, their generated wrapper) are interrupted by Ctrl-C. * On POSIX platforms, a workaround is installed to ensure processes started by `reactor.spawnProcess` have their exit noticed. See `Twisted ticket 6378`_ for more details about the underlying issue. .. _Twisted ticket 6378: http://tm.tl/6738 **0.8.1** * ``EventualResult.wait()`` now raises error if called in the reactor thread, thanks to David Buchmann. * Unittests are now included in the release tarball. * Allow Ctrl-C to interrupt ``EventualResult.wait(timeout=None)``. **0.7.0** * Improved documentation. **0.6.0** * Renamed ``DeferredResult`` to ``EventualResult``, to reduce confusion with Twisted's ``Deferred`` class. The old name still works, but is deprecated. * Deprecated ``@in_reactor``, replaced with ``@run_in_reactor`` which doesn't change the arguments to the wrapped function. The deprecated API still works, however. * Unhandled exceptions in ``EventualResult`` objects are logged. * Added more examples. * ``setup.py sdist`` should work now. **0.5.0** * Initial release. crochet-1.0.0/setup.py0000644000175000017500000000274512230510472015115 0ustar roaksoaxroaksoaximport os try: from setuptools import setup except ImportError: from distutils.core import setup def get_crochet_version(): """ Get crochet version from version module without importing more than necessary. """ this_dir_path = os.path.dirname(__file__) crochet_module_path = os.path.join(this_dir_path, "crochet") version_module_path = os.path.join(crochet_module_path, "_version.py") # The version module contains a variable called __version__ exec(file(version_module_path).read()) return __version__ setup( classifiers=[ 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', ], name='crochet', version=get_crochet_version(), description="Use Twisted from threaded applications", install_requires=[ "Twisted>=11.1", ], keywords="twisted threading", license="MIT", packages=["crochet", "crochet.tests"], url="https://github.com/itamarst/crochet", maintainer='Itamar Turner-Trauring', maintainer_email='itamar@futurefoundries.com', long_description=open('README.rst').read(), ) crochet-1.0.0/examples/0000755000175000017500000000000012230523106015207 5ustar roaksoaxroaksoaxcrochet-1.0.0/examples/scheduling.py0000644000175000017500000000513212222412350017706 0ustar roaksoaxroaksoax#!/usr/bin/python """ An example of scheduling time-based events in the background. Download the latest EUR/USD exchange rate from Yahoo every 30 seconds in the background; the rendered Flask web page can use the latest value without having to do the request itself. Note this is example is for demonstration purposes only, and is not actually used in the real world. You should not do this in a real application without reading Yahoo's terms-of-service and following them. """ from __future__ import print_function from flask import Flask from twisted.internet.task import LoopingCall from twisted.web.client import getPage from twisted.python import log from crochet import wait_for_reactor, setup setup() # Twisted code: class _ExchangeRate(object): """Download an exchange rate from Yahoo Finance using Twisted.""" def __init__(self, name): self._value = None self._name = name # External API: def latest_value(self): """Return the latest exchange rate value. May be None if no value is available. """ return self._value def start(self): """Start the background process.""" self._lc = LoopingCall(self._download) # Run immediately, and then every 30 seconds: self._lc.start(30, now=True) def _download(self): """Download the page.""" print("Downloading!") def parse(result): print("Got %r back from Yahoo." % (result,)) values = result.strip().split(",") self._value = float(values[1]) d = getPage( "http://download.finance.yahoo.com/d/quotes.csv?e=.csv&f=c4l1&s=%s=X" % (self._name,)) d.addCallback(parse) d.addErrback(log.err) return d # Blocking wrapper: class ExchangeRate(object): """Blocking API for downloading exchange rate.""" @wait_for_reactor def __init__(self, name): self._exchange = _ExchangeRate(name) self._exchange.start() @wait_for_reactor def latest_value(self): """Return the latest exchange rate value. May be None if no value is available. """ return self._exchange.latest_value() # Start background download: EURUSD = ExchangeRate("EURUSD") # Flask application: app = Flask(__name__) @app.route('/') def index(): rate = EURUSD.latest_value() if rate is None: rate = "unavailable, please refresh the page" return "Current EUR/USD exchange rate is %s." % (rate,) if __name__ == '__main__': import sys, logging logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) app.run() crochet-1.0.0/examples/ssh.py0000644000175000017500000000367112222335170016370 0ustar roaksoaxroaksoax#!/usr/bin/python """ A demonstration of Conch, allowing you to SSH into a running Python server and inspect objects at a Python prompt. If you're using the system install of Twisted, you may need to install Conch separately, e.g. on Ubuntu: $ sudo apt-get install python-twisted-conch Once you've started the program, you can ssh in by doing: $ ssh admin@localhost -p 5022 The password is 'secret'. Once you've reached the Python prompt, you have access to the app object, and can import code, etc.: >>> 3 + 4 7 >>> print(app) """ import logging from flask import Flask from crochet import setup, wait_for_reactor setup() # Web server: app = Flask(__name__) @app.route('/') def index(): return "Welcome to my boring web server!" @wait_for_reactor def start_ssh_server(port, username, password, namespace): """ Start an SSH server on the given port, exposing a Python prompt with the given namespace. """ # This is a lot of boilerplate, see http://tm.tl/6429 for a ticket to # provide a utility function that simplifies this. from twisted.internet import reactor from twisted.conch.insults import insults from twisted.conch import manhole, manhole_ssh from twisted.cred.checkers import ( InMemoryUsernamePasswordDatabaseDontUse as MemoryDB) from twisted.cred.portal import Portal sshRealm = manhole_ssh.TerminalRealm() def chainedProtocolFactory(): return insults.ServerProtocol(manhole.Manhole, namespace) sshRealm.chainedProtocolFactory = chainedProtocolFactory sshPortal = Portal(sshRealm, [MemoryDB(**{username: password})]) reactor.listenTCP(port, manhole_ssh.ConchFactory(sshPortal), interface="127.0.0.1") if __name__ == '__main__': import sys logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) start_ssh_server(5022, "admin", "secret", {"app": app}) app.run() crochet-1.0.0/examples/timeouts.py0000644000175000017500000000231112222135252017431 0ustar roaksoaxroaksoax#!/usr/bin/python """ Example of implementing timeouts using Deferred cancellation. """ # The Twisted code we'll be using: from twisted.mail import smtp from crochet import setup, run_in_reactor, TimeoutError crochet.setup() # Crochet layer, wrapping Twisted API in blocking call: @run_in_reactor def _sendmail(to_addr, message): """Send an email to the given address with the given message.""" return smtp.sendmail("127.0.0.1", "from@example.com", [to_addr], message) # Public API, hides details of Crochet and Twisted from application code: def sendmail(to_addr, message, timeout=10): """Send an email to the given address with the given message. If the operation times out, the function will attempt to cancel the send operation and then raise a TimeoutError exception. The cancellation is best effort and not guaranteed to succeed. """ result = _sendmail(to_addr, message) try: return result.wait(timeout) except TimeoutError: # requires Twisted 13.2 to actually cancel result.cancel() raise if __name__ == '__main__': # Application code using the public API: import sys sendmail(sys.argv[1], sys.argv[2]) crochet-1.0.0/examples/fromtwisted.py~0000644000175000017500000000155412230521242020332 0ustar roaksoaxroaksoax#!/usr/bin/python """ An example of using Crochet from a normal Twisted application. """ import time from crochet import no_setup, run_in_reactor, wait_for_reactor, TimeoutError # Tell Crochet not to run the reactor: no_setup() from twisted.internet import reactor from twisted.web.client impot getPage # Blocking API: @run_in_reactor def _download_page(url): return getPage(url) def time_page_download(url, timeout=1): start = time.time() _download_page(url).wait(timeout) return time.time() - start def main(): def blockingCode(): while True: try: print "Download took", print time_page_download("http://www.example.com") except TimeoutError: print "Timeout!" threading.Thread(target=blockingCode).run() reactor.run() if __name__ == '__main__': main() crochet-1.0.0/examples/downloader.py0000644000175000017500000000237112223327230017724 0ustar roaksoaxroaksoax#!/usr/bin/python """ A flask web application that downloads a page in the background. """ import logging from flask import Flask, session, escape from crochet import setup, run_in_reactor, retrieve_result, TimeoutError # Can be called multiple times with no ill-effect: setup() app = Flask(__name__) @run_in_reactor def download_page(url): """ Download a page. """ from twisted.web.client import getPage return getPage(url) @app.route('/') def index(): if 'download' not in session: # Calling an @run_in_reactor function returns an EventualResult: result = download_page('http://www.google.com') session['download'] = result.stash() return "Starting download, refresh to track progress." # Retrieval is a one-time operation, so the uid in the session cannot be # reused: result = retrieve_result(session.pop('download')) try: download = result.wait(timeout=0.1) return "Downloaded: " + escape(download) except TimeoutError: session['download'] = result.stash() return "Download in progress..." if __name__ == '__main__': import os, sys logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) app.secret_key = os.urandom(24) app.run() crochet-1.0.0/examples/blockingdownload.py0000644000175000017500000000057012223321471021106 0ustar roaksoaxroaksoax#!/usr/bin/python """ Download a web page in a blocking manner. """ from __future__ import print_function import sys from twisted.web.client import getPage from crochet import setup, wait_for_reactor setup() @wait_for_reactor def download_page(url): return getPage(url) # download_page() now behaves like a normal blocking function: print(download_page(sys.argv[1])) crochet-1.0.0/examples/mxquery.py0000644000175000017500000000207712222411552017303 0ustar roaksoaxroaksoax#!/usr/bin/python """ A command-line application that uses Twisted to do an MX DNS query. """ from __future__ import print_function from twisted.internet.defer import inlineCallbacks, returnValue from twisted.names.client import lookupMailExchange from crochet import setup, wait_for_reactor setup() # Twisted code: def _mx(domain): """ Return Defered that fires with a list of (priority, MX domain) tuples for a given domain. """ def got_records(result): return sorted( [(int(record.payload.preference), str(record.payload.name)) for record in result[0]]) d = lookupMailExchange(domain) d.addCallback(got_records) return d # Blocking wrapper: @wait_for_reactor def mx(domain): """ Return list of (priority, MX domain) tuples for a given domain. """ return _mx(domain) # Application code: def main(domain): print("Mail servers for %s:" % (domain,)) for priority, mailserver in mx(domain): print(priority, mailserver) if __name__ == '__main__': import sys main(sys.argv[1]) crochet-1.0.0/examples/fromtwisted.py0000644000175000017500000000175112230522230020131 0ustar roaksoaxroaksoax#!/usr/bin/python """ An example of using Crochet from a normal Twisted application. """ import time import threading import sys from crochet import no_setup, run_in_reactor, wait_for_reactor, TimeoutError # Tell Crochet not to run the reactor: no_setup() from twisted.internet import reactor from twisted.web.client import getPage # Blocking API: @run_in_reactor def _download_page(url): return getPage(url) def time_page_download(url): start = time.time() _download_page(url).wait(1) return time.time() - start def main(url): def blockingCode(): print "Downloading", url while True: try: elapsed = time_page_download(url) except TimeoutError: print "Timeout!" else: print "Download took", elapsed, "seconds" threading.Thread(target=blockingCode).start() # Run the reactor as you usually would: reactor.run() if __name__ == '__main__': main(sys.argv[1]) crochet-1.0.0/docs/0000755000175000017500000000000012230523106014321 5ustar roaksoaxroaksoaxcrochet-1.0.0/docs/introduction.rst0000644000175000017500000000556712230522145017613 0ustar roaksoaxroaksoaxIntroduction ------------ What It Does ^^^^^^^^^^^^ Crochet provides the following features: * Runs Twisted's reactor in a thread it manages. * The reactor shuts down automatically when the process' main thread finishes. * Hooks up Twisted's log system to the Python standard library ``logging`` framework. Unlike Twisted's built-in ``logging`` bridge, this includes support for blocking `Handler` instances. * Provides a blocking API to eventual results (i.e. ``Deferred`` instances). This last feature can be used separately, so Crochet is also useful for normal Twisted applications that use threads. Why should you care about using Twisted? Because it gives you the full power of an event-driven networking framework from inside your applications. Some Use Cases ^^^^^^^^^^^^^^ Writing an application in a blocking framework, but want to use Twisted for some part of your application? Crochet lets you run the reactor transparently and call into Twisted in a blocking manner. If you're writing a web application you can probably use Twisted as a `WSGI container`_, in which case Crochet's reactor-running functionality isn't necessary. Crochet's APIs for calling Twisted from threads would still be useful. And of course if you don't want to use Twisted as your WSGI container then you'd want to use Crochet's full functionality. Perhaps you're writing a library that provides a blocking API, but want to use Twisted for the implementation. Running the reactor in a thread yourself is difficult... and since you can only run the reactor once this would prevent using your library in applications that already use Twisted. Crochet provides a solution for the latter issue using the ``no_setup()`` API. If you're writing a Twisted application that involves non-trivial threading interactions, and Twisted's built-in APIs (``deferToThread`` and ``blockCallFromThread``) are therefore insufficient, Crochet's API for calling into Twisted from threads will come in handy. .. _WSGI container: https://twistedmatrix.com/documents/current/web/howto/web-in-60/wsgi.html Example: Background Scheduling ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ You can use Crochet to schedule events that will run in the background without slowing down the page rendering of your web applications: .. literalinclude:: ../examples/scheduling.py Example: SSH into your Server ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ You can SSH into your Python process and get a Python prompt, allowing you to poke around in the internals of your running program: .. literalinclude:: ../examples/ssh.py Example: DNS Query ^^^^^^^^^^^^^^^^^^ Twisted also has a fully featured DNS library: .. literalinclude:: ../examples/mxquery.py Example: Using Crochet in Normal Twisted Code ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ You can use Crochet's APIs for calling into the reactor thread from normal Twisted applications: .. literalinclude:: ../examples/fromtwisted.py crochet-1.0.0/docs/conf.py0000644000175000017500000001670112217617723015643 0ustar roaksoaxroaksoax# -*- coding: utf-8 -*- # # Crochet documentation build configuration file, created by # sphinx-quickstart on Mon Sep 16 19:37:18 2013. # # This file is execfile()d with the current directory set to its containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import sys, os # Make sure local crochet is used when importing: sys.path.insert(0, os.path.abspath('..')) # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. #source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = u'Crochet' copyright = u'2013, Itamar Turner-Trauring' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. import crochet version = crochet.__version__ # The full version, including alpha/beta/rc tags. release = crochet.__version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = ['_build'] # The reST default role (used for this markup: `text`) to use for all documents. #default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). #add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = 'default' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If false, no module index is generated. #html_domain_indices = True # If false, no index is generated. html_use_index = False # If true, the index is split into individual pages for each letter. #html_split_index = False # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. #html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. #html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'crochetdoc' # -- Options for LaTeX output -------------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). #'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). #'pointsize': '10pt', # Additional stuff for the LaTeX preamble. #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ ('index', 'Crochet.tex', u'Crochet Documentation', u'Itamar Turner-Trauring', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # If true, show page references after internal links. #latex_show_pagerefs = False # If true, show URL addresses after external links. #latex_show_urls = False # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. #latex_domain_indices = True # -- Options for manual page output -------------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ ('index', 'crochet', u'Crochet Documentation', [u'Itamar Turner-Trauring'], 1) ] # If true, show URL addresses after external links. #man_show_urls = False # -- Options for Texinfo output ------------------------------------------------ # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ ('index', 'Crochet', u'Crochet Documentation', u'Itamar Turner-Trauring', 'Crochet', 'One line description of project.', 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. #texinfo_appendices = [] # If false, no module index is generated. #texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. #texinfo_show_urls = 'footnote' crochet-1.0.0/docs/index.rst0000644000175000017500000000161412224575333016200 0ustar roaksoaxroaksoax.. Crochet documentation master file, created by sphinx-quickstart on Mon Sep 16 19:37:18 2013. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. Crochet: Use Twisted Anywhere! ============================== Crochet is an MIT-licensed library that makes it easier for blocking and threaded applications like Flask or Django to use the Twisted networking framework. Here's an example of a program using Crochet: .. literalinclude:: ../examples/blockingdownload.py Run on the command line:: $ python blockingdownload.py http://google.com print(download_page(sys.argv[1])) File "/home/itamar/devel/python/crochet/crochet/_eventloop.py", line 300, in wrapper function, *args, **kwargs) File "/usr/lib/python2.7/dist-packages/twisted/internet/threads.py", line 118, in blockingCallFromThread result.raiseException() File "/usr/lib/python2.7/dist-packages/twisted/python/failure.py", line 370, in raiseException raise self.type, self.value, self.tb twisted.internet.error.DNSLookupError: DNS lookup failed: address 'notarealsite.atall' not found: [Errno -5] No address associated with hostname. For comparison, here's what the Twisted version would look like; notice the use of ``addCallback`` since ``getPage()`` returns a ``Deferred``: .. code-block:: python #!/usr/bin/python """ Download a web page in a non-blocking manner. """ from __future__ import print_function import sys from twisted.web.client import getPage from twisted.internet.task import react def main(reactor, url): return getPage(url).addCallback(print) react(main, sys.argv[1:]) @run_in_reactor: Asynchronous Results ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``run_in_reactor`` is a more sophisticated alternative to ``wait_for_reactor``. Rather than waiting until a result is available, it returns a special object supporting timeouts and cancellation. Decorating a function that calls Twisted APIs with ``run_in_reactor`` has two consequences: * When the function is called, the code will not run in the calling thread, but rather in the reactor thread. * The return result from a decorated function is an ``EventualResult`` instance, wrapping the result of the underlying code, with particular support for ``Deferred`` instances. ``EventualResult`` has the following methods: * ``wait(timeout=None)``: Return the result when it becomes available; if the result is an exception it will be raised. If an optional timeout is given (in seconds), ``wait()`` will throw ``crochet.TimeoutError`` if the timeout is hit, rather than blocking indefinitely. * ``cancel()``: Cancel the operation tied to the underlying ``Deferred``. Many, but not all, ``Deferred`` results returned from Twisted allow the underlying operation to be canceled. In any case this should be considered a best effort cancellation. * ``stash()``: Sometimes you want to store the ``EventualResult`` in memory for later retrieval. This is specifically useful when you want to store a reference to the ``EventualResult`` in a web session like Flask's (see the example below). ``stash()`` stores the ``EventualResult`` in memory, and returns an integer uid that can be used to retrieve the result using ``crochet.retrieve_result(uid)``. Note that retrieval works only once per uid. You will need the stash the ``EventualResult`` again (with a new resulting uid) if you want to retrieve it again later. In the following example, you can see all of these APIs in use. For each user session, a download is started in the background. Subsequent page refreshes will eventually show the downloaded page. .. literalinclude:: ../examples/downloader.py Using Crochet from Twisted Applications ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If your application is already planning on running the Twisted reactor itself (e.g. you're using Twisted as a WSGI container), Crochet's default behavior of running the reactor in a thread is a problem. To solve this, Crochet provides the ``no_setup()`` function, which causes future calls to ``setup()`` to do nothing. Thus, an application that will run the Twisted reactor but also wants to use a Crochet-using library must run it first: .. code-block:: python from crochet import no_setup no_setup() # Only now do we import libraries that might run crochet.setup(): import blockinglib # ... setup application ... from twisted.internet import reactor reactor.run() Reducing Twisted Log Messages ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Twisted can be rather verbose with its log messages. If you wish to reduce the message flow you can limit them to error messages only: .. code-block:: python import logging logging.getLogger('twisted').setLevel(logging.ERROR) crochet-1.0.0/docs/Makefile0000644000175000017500000001270012215713056015771 0ustar roaksoaxroaksoax# Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " text to make text files" @echo " man to make manual pages" @echo " texinfo to make Texinfo files" @echo " info to make Texinfo files and run them through makeinfo" @echo " gettext to make PO message catalogs" @echo " changes to make an overview of all changed/added/deprecated items" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: -rm -rf $(BUILDDIR)/* html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Crochet.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Crochet.qhc" devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @echo "# mkdir -p $$HOME/.local/share/devhelp/Crochet" @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Crochet" @echo "# devhelp" epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." texinfo: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." crochet-1.0.0/docs/using.rst0000644000175000017500000001155112230514763016214 0ustar roaksoaxroaksoaxUsing Crochet ------------- @wait_for_reactor vs. @run_in_reactor ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``@wait_for_reactor`` is a much simpler API to use, but also suffers from this simplicity. In particular, it will indefinitely for the result to be available. If you're dealing with an in-memory structure this may be fine. Any time you deal with an external system this can lead to infinite or at least extremely long wait times. Your network connection may be down, you may be hitting a firewall that swallows all packets, you might be talking to a process that got suspended... relying on an external system to respond in a timely manner is a bad idea. This is where ``@run_in_reactor`` becomes useful. The ability to check if a result has arrived yet means the action can run in the background, as we saw with the background download example. Support for timeouts and cancellation means you're never left waiting for an unbounded amount of time. But, that does mean you're using ``EventualResult`` instances. Luckily, you can often hide their existence from the calling code as we'll see in the next sections. Hide Twisted and Crochet ^^^^^^^^^^^^^^^^^^^^^^^^ Consider some synchronous do-one-thing-after-the-other application code that wants to use event-driven Twisted-using code. We have two threads at a minimum: the application thread(s) and the reactor thread. There are also multiple layers of code involved in this interaction: * **Twisted code:** Should only be called in reactor thread. This may be code from the Twisted package itself, or more likely code you have written that is built on top of Twisted. * **@wait_for_reactor/@run_in_reactor wrappers:** The body of the functions runs in the reactor thread... but the caller should be in the application thread. * **The application code:** Runs in the application thread, expects synchronous/blocking calls. Sometimes the first two layers suffice, but there are some issues with only having these. First, if you're using ``@run_in_reactor`` it requires the application code to understand Crochet's API, i.e. ``EventualResult`` objects. Second, if the wrapped function returns an object that expects to interact with Twisted, the application code will not be able to use that object since it will be called in the wrong thread. A better solution is to have an additional layer in-between the application code and ``@wait_for_reactor/@run_in_reactor`` wrappers. This layer can hide the details of the Crochet API and wrap returned Twisted objects if necessary. As a result the application code simply seems a normal API, with no need to understand ``EventualResult`` objects or Twisted. Implementing Timeouts ^^^^^^^^^^^^^^^^^^^^^ To see how this might work, let's consider for example how one might use Twisted and Crochet's features to expose timeouts to application code. Twisted's ``Deferred`` objects can support cancellation: when ``Deferred.cancel()`` is called the underlying operation is canceled. The API documentation for Twisted will tell you when this is the case, and you can add support to your own ``Deferred``-creating code. ``EventualResult.cancel()`` exposes this functionality. ``EventualResult.wait()`` also has the ability to time out if no result becomes available within the given amount of time. A layer wrapping ``@run_in_reactor`` is an excellent place to combine the two. Notice in the following example the different layers of the code. .. literalinclude:: ../examples/timeouts.py Minimize Decorated Code ^^^^^^^^^^^^^^^^^^^^^^^ It's best to have as little code as possible in the ``@wait_for_reactor/@run_in_reactor`` wrappers. As this code straddles two worlds (or at least, two threads) it is more difficult to unit test. Having an extra layer between this code and the application code is also useful in this regard as well: Twisted code can be pushed into the lower-level Twisted layer, and code hiding the Twisted details from the application code can be pushed into the higher-level layer. Preventing Deadlocks on Shutdown ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ In general, you should avoid using calls that block indefinitely, i.e. ``@wait_for_reactor`` or ``EventualResult.wait()`` with no timeout. Crochet will however try to interrupt these during reactor shutdown with a ``crochet.ReactorStopped`` exception. This is still not a complete solution, unfortunately. If you are shutting down a thread pool as part of Twisted's reactor shutdown, this will wait until all threads are done. If you're blocking indefinitely, this may rely on Crochet interrupting those blocking calls... but Crochet's shutdown may be delayed until the thread pool finishes shutting down, depending on the ordering of shutdown events. The solution is to interrupt all blocking calls yourself. You can do this by firing or canceling any ``Deferred`` instances you are waiting on as part of your application shutdown, and do so before you stop any thread pools. crochet-1.0.0/docs/make.bat0000644000175000017500000001175212215713056015744 0ustar roaksoaxroaksoax@ECHO OFF REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) set BUILDDIR=_build set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . set I18NSPHINXOPTS=%SPHINXOPTS% . if NOT "%PAPER%" == "" ( set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% ) if "%1" == "" goto help if "%1" == "help" ( :help echo.Please use `make ^` where ^ is one of echo. html to make standalone HTML files echo. dirhtml to make HTML files named index.html in directories echo. singlehtml to make a single large HTML file echo. pickle to make pickle files echo. json to make JSON files echo. htmlhelp to make HTML files and a HTML help project echo. qthelp to make HTML files and a qthelp project echo. devhelp to make HTML files and a Devhelp project echo. epub to make an epub echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter echo. text to make text files echo. man to make manual pages echo. texinfo to make Texinfo files echo. gettext to make PO message catalogs echo. changes to make an overview over all changed/added/deprecated items echo. linkcheck to check all external links for integrity echo. doctest to run all doctests embedded in the documentation if enabled goto end ) if "%1" == "clean" ( for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i del /q /s %BUILDDIR%\* goto end ) if "%1" == "html" ( %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html if errorlevel 1 exit /b 1 echo. echo.Build finished. The HTML pages are in %BUILDDIR%/html. goto end ) if "%1" == "dirhtml" ( %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml if errorlevel 1 exit /b 1 echo. echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. goto end ) if "%1" == "singlehtml" ( %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml if errorlevel 1 exit /b 1 echo. echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. goto end ) if "%1" == "pickle" ( %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle if errorlevel 1 exit /b 1 echo. echo.Build finished; now you can process the pickle files. goto end ) if "%1" == "json" ( %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json if errorlevel 1 exit /b 1 echo. echo.Build finished; now you can process the JSON files. goto end ) if "%1" == "htmlhelp" ( %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp if errorlevel 1 exit /b 1 echo. echo.Build finished; now you can run HTML Help Workshop with the ^ .hhp project file in %BUILDDIR%/htmlhelp. goto end ) if "%1" == "qthelp" ( %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp if errorlevel 1 exit /b 1 echo. echo.Build finished; now you can run "qcollectiongenerator" with the ^ .qhcp project file in %BUILDDIR%/qthelp, like this: echo.^> qcollectiongenerator %BUILDDIR%\qthelp\Crochet.qhcp echo.To view the help file: echo.^> assistant -collectionFile %BUILDDIR%\qthelp\Crochet.ghc goto end ) if "%1" == "devhelp" ( %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp if errorlevel 1 exit /b 1 echo. echo.Build finished. goto end ) if "%1" == "epub" ( %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub if errorlevel 1 exit /b 1 echo. echo.Build finished. The epub file is in %BUILDDIR%/epub. goto end ) if "%1" == "latex" ( %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex if errorlevel 1 exit /b 1 echo. echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. goto end ) if "%1" == "text" ( %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text if errorlevel 1 exit /b 1 echo. echo.Build finished. The text files are in %BUILDDIR%/text. goto end ) if "%1" == "man" ( %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man if errorlevel 1 exit /b 1 echo. echo.Build finished. The manual pages are in %BUILDDIR%/man. goto end ) if "%1" == "texinfo" ( %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo if errorlevel 1 exit /b 1 echo. echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. goto end ) if "%1" == "gettext" ( %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale if errorlevel 1 exit /b 1 echo. echo.Build finished. The message catalogs are in %BUILDDIR%/locale. goto end ) if "%1" == "changes" ( %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes if errorlevel 1 exit /b 1 echo. echo.The overview file is in %BUILDDIR%/changes. goto end ) if "%1" == "linkcheck" ( %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck if errorlevel 1 exit /b 1 echo. echo.Link check complete; look for any errors in the above output ^ or in %BUILDDIR%/linkcheck/output.txt. goto end ) if "%1" == "doctest" ( %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest if errorlevel 1 exit /b 1 echo. echo.Testing of doctests in the sources finished, look at the ^ results in %BUILDDIR%/doctest/output.txt. goto end ) :end crochet-1.0.0/PKG-INFO0000644000175000017500000001143412230523106014471 0ustar roaksoaxroaksoaxMetadata-Version: 1.1 Name: crochet Version: 1.0.0 Summary: Use Twisted from threaded applications Home-page: https://github.com/itamarst/crochet Author: Itamar Turner-Trauring Author-email: itamar@futurefoundries.com License: MIT Description: Crochet: Use Twisted Anywhere! ============================== Crochet is an MIT-licensed library that makes it easier for blocking or threaded applications like Flask or Django to use the Twisted networking framework. Crochet provides the following features: * Runs Twisted's reactor in a thread it manages. * The reactor shuts down automatically when the process' main thread finishes. * Hooks up Twisted's log system to the Python standard library ``logging`` framework. Unlike Twisted's built-in ``logging`` bridge, this includes support for blocking `Handler` instances. * A blocking API to eventual results (i.e. ``Deferred`` instances). This last feature can be used separately, so Crochet is also useful for normal Twisted applications that use threads. .. image:: https://travis-ci.org/itamarst/crochet.png?branch=master :target: http://travis-ci.org/itamarst/crochet :alt: Build Status Documentation can be found on `Read The Docs`_. Bugs and feature requests should be filed at the project `Github page`_. .. _Read the Docs: https://crochet.readthedocs.org/ .. _Github page: https://github.com/itamarst/crochet/ Changelog --------- **1.0.0** Documentation: * Added section on use cases and alternatives. Thanks to Tobias Oberstein for the suggestion. Bug fixes: * Twisted does not have to be pre-installed to run ``setup.py``, thanks to Paul Weaver for bug report and Chris Scutcher for patch. * Importing Crochet does not have side-effects (installing reactor event) any more. * Blocking calls are interrupted earlier in the shutdown process, to reduce scope for deadlocks. Thanks to rmorehead for bug report. **0.9.0** New features: * Expanded and much improved documentation, including a new section with design suggestions. * New decorator ``@wait_for_reactor`` added, a simpler alternative to ``@run_in_reactor``. * Refactored ``@run_in_reactor``, making it a bit more responsive. * Blocking operations which would otherwise never finish due to reactor having stopped (``EventualResult.wait()`` or ``@wait_for_reactor`` decorated call) will be interrupted with a ``ReactorStopped`` exception. Thanks to rmorehead for the bug report. Bug fixes: * ``@run_in_reactor`` decorated functions (or rather, their generated wrapper) are interrupted by Ctrl-C. * On POSIX platforms, a workaround is installed to ensure processes started by `reactor.spawnProcess` have their exit noticed. See `Twisted ticket 6378`_ for more details about the underlying issue. .. _Twisted ticket 6378: http://tm.tl/6738 **0.8.1** * ``EventualResult.wait()`` now raises error if called in the reactor thread, thanks to David Buchmann. * Unittests are now included in the release tarball. * Allow Ctrl-C to interrupt ``EventualResult.wait(timeout=None)``. **0.7.0** * Improved documentation. **0.6.0** * Renamed ``DeferredResult`` to ``EventualResult``, to reduce confusion with Twisted's ``Deferred`` class. The old name still works, but is deprecated. * Deprecated ``@in_reactor``, replaced with ``@run_in_reactor`` which doesn't change the arguments to the wrapped function. The deprecated API still works, however. * Unhandled exceptions in ``EventualResult`` objects are logged. * Added more examples. * ``setup.py sdist`` should work now. **0.5.0** * Initial release. Keywords: twisted threading Platform: UNKNOWN Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: MIT License Classifier: Operating System :: OS Independent Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 2.6 Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3.3 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy crochet-1.0.0/crochet.egg-info/0000755000175000017500000000000012230523106016512 5ustar roaksoaxroaksoaxcrochet-1.0.0/crochet.egg-info/top_level.txt0000644000175000017500000000001012230523103021230 0ustar roaksoaxroaksoaxcrochet crochet-1.0.0/crochet.egg-info/requires.txt0000644000175000017500000000001512230523103021103 0ustar roaksoaxroaksoaxTwisted>=11.1crochet-1.0.0/crochet.egg-info/PKG-INFO0000644000175000017500000001143412230523103017607 0ustar roaksoaxroaksoaxMetadata-Version: 1.1 Name: crochet Version: 1.0.0 Summary: Use Twisted from threaded applications Home-page: https://github.com/itamarst/crochet Author: Itamar Turner-Trauring Author-email: itamar@futurefoundries.com License: MIT Description: Crochet: Use Twisted Anywhere! ============================== Crochet is an MIT-licensed library that makes it easier for blocking or threaded applications like Flask or Django to use the Twisted networking framework. Crochet provides the following features: * Runs Twisted's reactor in a thread it manages. * The reactor shuts down automatically when the process' main thread finishes. * Hooks up Twisted's log system to the Python standard library ``logging`` framework. Unlike Twisted's built-in ``logging`` bridge, this includes support for blocking `Handler` instances. * A blocking API to eventual results (i.e. ``Deferred`` instances). This last feature can be used separately, so Crochet is also useful for normal Twisted applications that use threads. .. image:: https://travis-ci.org/itamarst/crochet.png?branch=master :target: http://travis-ci.org/itamarst/crochet :alt: Build Status Documentation can be found on `Read The Docs`_. Bugs and feature requests should be filed at the project `Github page`_. .. _Read the Docs: https://crochet.readthedocs.org/ .. _Github page: https://github.com/itamarst/crochet/ Changelog --------- **1.0.0** Documentation: * Added section on use cases and alternatives. Thanks to Tobias Oberstein for the suggestion. Bug fixes: * Twisted does not have to be pre-installed to run ``setup.py``, thanks to Paul Weaver for bug report and Chris Scutcher for patch. * Importing Crochet does not have side-effects (installing reactor event) any more. * Blocking calls are interrupted earlier in the shutdown process, to reduce scope for deadlocks. Thanks to rmorehead for bug report. **0.9.0** New features: * Expanded and much improved documentation, including a new section with design suggestions. * New decorator ``@wait_for_reactor`` added, a simpler alternative to ``@run_in_reactor``. * Refactored ``@run_in_reactor``, making it a bit more responsive. * Blocking operations which would otherwise never finish due to reactor having stopped (``EventualResult.wait()`` or ``@wait_for_reactor`` decorated call) will be interrupted with a ``ReactorStopped`` exception. Thanks to rmorehead for the bug report. Bug fixes: * ``@run_in_reactor`` decorated functions (or rather, their generated wrapper) are interrupted by Ctrl-C. * On POSIX platforms, a workaround is installed to ensure processes started by `reactor.spawnProcess` have their exit noticed. See `Twisted ticket 6378`_ for more details about the underlying issue. .. _Twisted ticket 6378: http://tm.tl/6738 **0.8.1** * ``EventualResult.wait()`` now raises error if called in the reactor thread, thanks to David Buchmann. * Unittests are now included in the release tarball. * Allow Ctrl-C to interrupt ``EventualResult.wait(timeout=None)``. **0.7.0** * Improved documentation. **0.6.0** * Renamed ``DeferredResult`` to ``EventualResult``, to reduce confusion with Twisted's ``Deferred`` class. The old name still works, but is deprecated. * Deprecated ``@in_reactor``, replaced with ``@run_in_reactor`` which doesn't change the arguments to the wrapped function. The deprecated API still works, however. * Unhandled exceptions in ``EventualResult`` objects are logged. * Added more examples. * ``setup.py sdist`` should work now. **0.5.0** * Initial release. Keywords: twisted threading Platform: UNKNOWN Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: MIT License Classifier: Operating System :: OS Independent Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 2.6 Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3.3 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy crochet-1.0.0/crochet.egg-info/SOURCES.txt0000644000175000017500000000147312230523106020403 0ustar roaksoaxroaksoaxLICENSE MANIFEST.in README.rst requirements-dev.txt setup.py crochet/__init__.py crochet/_eventloop.py crochet/_resultstore.py crochet/_shutdown.py crochet/_util.py crochet/_version.py crochet.egg-info/PKG-INFO crochet.egg-info/SOURCES.txt crochet.egg-info/dependency_links.txt crochet.egg-info/requires.txt crochet.egg-info/top_level.txt crochet/tests/__init__.py crochet/tests/test_api.py crochet/tests/test_process.py crochet/tests/test_resultstore.py crochet/tests/test_setup.py crochet/tests/test_shutdown.py crochet/tests/test_util.py docs/Makefile docs/api.rst docs/conf.py docs/index.rst docs/introduction.rst docs/make.bat docs/using.rst examples/blockingdownload.py examples/downloader.py examples/fromtwisted.py examples/fromtwisted.py~ examples/mxquery.py examples/scheduling.py examples/ssh.py examples/timeouts.pycrochet-1.0.0/crochet.egg-info/dependency_links.txt0000644000175000017500000000000112230523103022555 0ustar roaksoaxroaksoax crochet-1.0.0/requirements-dev.txt0000644000175000017500000000002712217622714017442 0ustar roaksoaxroaksoaxTwisted>=11.1.0 sphinx crochet-1.0.0/LICENSE0000644000175000017500000000210212224075317014402 0ustar roaksoaxroaksoaxCopyright (c) 2013 Itamar Turner-Trauring and Twisted Matrix Labs Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. crochet-1.0.0/crochet/0000755000175000017500000000000012230523106015020 5ustar roaksoaxroaksoaxcrochet-1.0.0/crochet/_eventloop.py0000644000175000017500000003311112230516537017555 0ustar roaksoaxroaksoax""" Expose Twisted's event loop to threaded programs. """ from __future__ import absolute_import import threading import weakref try: from queue import Queue, Empty except ImportError: from Queue import Queue, Empty from functools import wraps from twisted.python import threadable from twisted.python.runtime import platform from twisted.python.failure import Failure from twisted.python.log import PythonLoggingObserver, err from twisted.internet.defer import maybeDeferred from twisted.internet import reactor from twisted.internet.task import LoopingCall from ._util import synchronized from ._resultstore import ResultStore _store = ResultStore() if hasattr(weakref, "WeakSet"): WeakSet = weakref.WeakSet else: class WeakSet(object): """ Minimal WeakSet emulation. """ def __init__(self): self._items = weakref.WeakKeyDictionary() def add(self, value): self._items[value] = True def __iter__(self): return iter(self._items) class TimeoutError(Exception): """ A timeout has been hit. """ class ReactorStopped(Exception): """ The reactor has stopped, and therefore no result will ever become available from this EventualResult. """ class ResultRegistry(object): """ Keep track of EventualResults. Once the reactor has shutdown: 1. Registering new EventualResult instances is an error, since no results will ever become available. 2. Already registered EventualResult instances are "fired" with a ReactorStopped exception to unblock any remaining EventualResult.wait() calls. """ def __init__(self, reactor): self._results = WeakSet() self._stopped = False self._lock = threading.Lock() @synchronized def register(self, result): """ Register an EventualResult. May be called in any thread. """ if self._stopped: raise ReactorStopped() self._results.add(result) @synchronized def stop(self): """ Indicate no more results will get pushed into EventualResults, since the reactor has stopped. This should be called in the reactor thread. """ self._stopped = True for result in self._results: result._set_result(Failure(ReactorStopped())) class EventualResult(object): """ A blocking interface to Deferred results. This allows you to access results from Twisted operations that may not be available immediately, using the wait() method. In general you should not create these directly; instead use functions decorated with @run_in_reactor. """ def __init__(self, deferred, _reactor=reactor): """ The deferred parameter should be a Deferred or None indicating _connect_deferred will be called separately later. """ self._deferred = deferred self._reactor = _reactor self._queue = Queue() self._result_retrieved = False self._result_set = False if deferred is not None: self._connect_deferred(deferred) def _connect_deferred(self, deferred): """ Hook up the Deferred that that this will be the result of. Should only be run in Twisted thread, and only called once. """ self._deferred = deferred # Because we use __del__, we need to make sure there are no cycles # involving this object, which is why we use a weakref: def put(result, eventual=weakref.ref(self)): eventual = eventual() if eventual: eventual._set_result(result) else: err(result, "Unhandled error in EventualResult") deferred.addBoth(put) def _set_result(self, result): """ Set the result of the EventualResult, if not already set. This can only happen in the reactor thread, either as a result of Deferred firing, or as a result of ResultRegistry.stop(). So, no need for thread-safety. """ if self._result_set: return self._result_set = True self._queue.put(result) def __del__(self): if self._result_retrieved: return try: result = self._queue.get(timeout=0) except Empty: return if isinstance(result, Failure): err(result, "Unhandled error in EventualResult") def cancel(self): """ Try to cancel the operation by cancelling the underlying Deferred. Cancellation of the operation may or may not happen depending on underlying cancellation support and whether the operation has already finished. In any case, however, the underlying Deferred will be fired. Multiple calls will have no additional effect. """ self._reactor.callFromThread(lambda: self._deferred.cancel()) def _result(self, timeout=None): """ Return the result, if available. It may take an unknown amount of time to return the result, so a timeout option is provided. If the given number of seconds pass with no result, a TimeoutError will be thrown. If a previous call timed out, additional calls to this function will still wait for a result and return it if available. If a result was returned on one call, additional calls will return/raise the same result. """ if timeout is None: # Queue.get(None) won't get interrupted by Ctrl-C... timeout = 2 ** 31 try: result = self._queue.get(timeout=timeout) except Empty: raise TimeoutError() self._result_retrieved = True self._queue.put(result) # allow next _result() call to get a value out return result def wait(self, timeout=None): """ Return the result, or throw the exception if result is a failure. It may take an unknown amount of time to return the result, so a timeout option is provided. If the given number of seconds pass with no result, a TimeoutError will be thrown. If a previous call timed out, additional calls to this function will still wait for a result and return it if available. If a result was returned or raised on one call, additional calls will return/raise the same result. """ if threadable.isInIOThread(): raise RuntimeError( "EventualResult.wait() must not be run in the reactor thread.") result = self._result(timeout) if isinstance(result, Failure): result.raiseException() return result def stash(self): """ Store the EventualResult in memory for later retrieval. Returns a integer uid which can be passed to crochet.retrieve_result() to retrieve the instance later on. """ return _store.store(self) def original_failure(self): """ Return the underlying Failure object, if the result is an error. If no result is yet available, or the result was not an error, None is returned. This method is useful if you want to get the original traceback for an error result. """ try: result = self._result(0.0) except TimeoutError: return None if isinstance(result, Failure): return result else: return None class ThreadLogObserver(object): """ A log observer that wraps another observer, and calls it in a thread. In particular, used to wrap PythonLoggingObserver, so that blocking logging.py Handlers don't block the event loop. """ _DONE = object() def __init__(self, observer): self._observer = observer self._queue = Queue() self._thread = threading.Thread(target=self._reader) self._thread.start() def _reader(self): """ Runs in a thread, reads messages from a queue and writes them to the wrapped observer. """ while True: item = self._queue.get() if item is self._DONE: return self._observer(item) def stop(self): """ Stop the thread. """ self._queue.put(self._DONE) def __call__(self, msg): """ A log observer that writes to a queue. """ self._queue.put(msg) class EventLoop(object): """ Initialization infrastructure for running a reactor in a thread. """ def __init__(self, reactor, atexit_register, startLoggingWithObserver=None, watchdog_thread=None, reapAllProcesses=None): self._reactor = reactor self._atexit_register = atexit_register self._startLoggingWithObserver = startLoggingWithObserver self._started = False self._lock = threading.Lock() self._watchdog_thread = watchdog_thread self._reapAllProcesses = reapAllProcesses self._registry = ResultRegistry(self._reactor) def _startReapingProcesses(self): """ Start a LoopingCall that calls reapAllProcesses. """ lc = LoopingCall(self._reapAllProcesses) lc.clock = self._reactor lc.start(0.1, False) def _common_setup(self): """ The minimal amount of setup done by both setup() and no_setup(). """ self._started = True # We want to unblock EventualResult regardless of how the reactor is # run, so we always register this: self._reactor.addSystemEventTrigger( "before", "shutdown", self._registry.stop) @synchronized def setup(self): """ Initialize the crochet library. This starts the reactor in a thread, and connect's Twisted's logs to Python's standard library logging module. This must be called at least once before the library can be used, and can be called multiple times. """ if self._started: return self._common_setup() if platform.type == "posix": self._reactor.callFromThread(self._startReapingProcesses) if self._startLoggingWithObserver: observer = ThreadLogObserver(PythonLoggingObserver().emit) self._reactor.callFromThread( self._startLoggingWithObserver, observer, False) # We only want to stop the logging thread once the reactor has # shut down: self._reactor.addSystemEventTrigger("after", "shutdown", observer.stop) t = threading.Thread( target=lambda: self._reactor.run(installSignalHandlers=False)) t.start() self._atexit_register(self._reactor.callFromThread, self._reactor.stop) self._atexit_register(_store.log_errors) if self._watchdog_thread is not None: self._watchdog_thread.start() @synchronized def no_setup(self): """ Initialize the crochet library with no side effects. No reactor will be started, logging is uneffected, etc.. Future calls to setup() will have no effect. This is useful for applications that intend to run Twisted's reactor themselves, and so do not want libraries using crochet to attempt to start it on their own. If no_setup() is called after setup(), a RuntimeError is raised. """ if self._started: raise RuntimeError("no_setup() is intended to be called once, by a" " Twisted application, before any libraries " "using crochet are imported and call setup().") self._common_setup() def run_in_reactor(self, function): """ A decorator that ensures the wrapped function runs in the reactor thread. When the wrapped function is called, an EventualResult is returned. """ def runs_in_reactor(result, args, kwargs): d = maybeDeferred(function, *args, **kwargs) result._connect_deferred(d) @wraps(function) def wrapper(*args, **kwargs): result = EventualResult(None, self._reactor) self._registry.register(result) self._reactor.callFromThread(runs_in_reactor, result, args, kwargs) return result return wrapper def wait_for_reactor(self, function): """ A decorator that ensures the wrapped function runs in the reactor thread. When the wrapped function is called, its result is returned or its exception raised. Deferreds are handled transparently. """ @wraps(function) def wrapper(*args, **kwargs): @self.run_in_reactor def run(): return function(*args, **kwargs) return run().wait() return wrapper def in_reactor(self, function): """ DEPRECATED, use run_in_reactor. A decorator that ensures the wrapped function runs in the reactor thread. The wrapped function will get the reactor passed in as a first argument, in addition to any arguments it is called with. When the wrapped function is called, an EventualResult is returned. """ import warnings warnings.warn("@in_reactor is deprecated, use @run_in_reactor", DeprecationWarning, stacklevel=2) @self.run_in_reactor @wraps(function) def add_reactor(*args, **kwargs): return function(self._reactor, *args, **kwargs) return add_reactor crochet-1.0.0/crochet/__init__.py0000644000175000017500000000241212230510363017131 0ustar roaksoaxroaksoax""" Crochet: Use Twisted Anywhere! """ from __future__ import absolute_import import sys from twisted.internet import reactor from twisted.python.log import startLoggingWithObserver try: from twisted.internet.process import reapAllProcesses except SyntaxError: if sys.version_info < (3, 3, 0): raise else: # Process support is still not ported to Python 3 on some versions of # Twisted. reapAllProcesses = lambda: None from ._shutdown import _watchdog, register from ._eventloop import (EventualResult, TimeoutError, EventLoop, _store, ReactorStopped) from ._version import __version__ _main = EventLoop(reactor, register, startLoggingWithObserver, _watchdog, reapAllProcesses) setup = _main.setup no_setup = _main.no_setup run_in_reactor = _main.run_in_reactor wait_for_reactor = _main.wait_for_reactor retrieve_result = _store.retrieve # Backwards compatibility with 0.5.0: in_reactor = _main.in_reactor DeferredResult = EventualResult __all__ = ["setup", "run_in_reactor", "EventualResult", "TimeoutError", "retrieve_result", "no_setup", "wait_for_reactor", "ReactorStopped", # Backwards compatibility: "DeferredResult", "in_reactor", ] crochet-1.0.0/crochet/_version.py0000644000175000017500000000016312230522774017230 0ustar roaksoaxroaksoax""" Store version in its own module so we can access it from both setup.py and __init__. """ __version__ = "1.0.0" crochet-1.0.0/crochet/tests/0000755000175000017500000000000012230523106016162 5ustar roaksoaxroaksoaxcrochet-1.0.0/crochet/tests/__init__.py0000644000175000017500000000002712127024027020275 0ustar roaksoaxroaksoax""" Crochet tests. """ crochet-1.0.0/crochet/tests/test_process.py0000644000175000017500000000356412224577036021277 0ustar roaksoaxroaksoax""" Tests for IReactorProcess. """ import subprocess import sys from twisted.trial.unittest import TestCase from twisted.python.runtime import platform class ProcessTests(TestCase): """ Tests for process support. """ def test_processExit(self): """ A Crochet-managed reactor notice when a process it started exits. On POSIX platforms this requies waitpid() to be called, which in default Twisted implementation relies on a SIGCHLD handler which is not installed by Crochet at the moment. """ program = """\ from crochet import setup, run_in_reactor setup() import sys import os from twisted.internet.protocol import ProcessProtocol from twisted.internet.defer import Deferred from twisted.internet import reactor class Waiter(ProcessProtocol): def __init__(self): self.result = Deferred() def processExited(self, reason): self.result.callback(None) @run_in_reactor def run(): waiter = Waiter() # Closing FDs before exit forces us to rely on SIGCHLD to notice process # exit: reactor.spawnProcess(waiter, sys.executable, [sys.executable, '-c', 'import os; os.close(0); os.close(1); os.close(2)'], env=os.environ) return waiter.result run().wait(10) # If we don't notice process exit, TimeoutError will be thrown and we won't # reach the next line: sys.stdout.write("abc") """ process = subprocess.Popen([sys.executable, "-c", program], stdout=subprocess.PIPE) result = process.stdout.read() self.assertEqual(result, b"abc") if platform.type != "posix": test_processExit.skip = "SIGCHLD is a POSIX-specific issue" if sys.version_info >= (3, 0, 0): test_processExit.skip = "Twisted does not support processes on Python 3 yet" crochet-1.0.0/crochet/tests/test_util.py0000644000175000017500000000327112127024027020556 0ustar roaksoaxroaksoax""" Tests for crochet._util. """ from __future__ import absolute_import from twisted.trial.unittest import TestCase from .._util import synchronized class FakeLock(object): locked = False def __enter__(self): self.locked = True def __exit__(self, type, value, traceback): self.locked = False class Lockable(object): def __init__(self): self._lock = FakeLock() @synchronized def check(self, x, y): if not self._lock.locked: raise RuntimeError() return x, y @synchronized def raiser(self): if not self._lock.locked: raise RuntimeError() raise ZeroDivisionError() class SynchronizedTests(TestCase): """ Tests for the synchronized decorator. """ def test_return(self): """ A method wrapped with @synchronized is called with the lock acquired, and it is released on return. """ obj = Lockable() self.assertEqual(obj.check(1, y=2), (1, 2)) self.assertFalse(obj._lock.locked) def test_raise(self): """ A method wrapped with @synchronized is called with the lock acquired, and it is released on exception raise. """ obj = Lockable() self.assertRaises(ZeroDivisionError, obj.raiser) self.assertFalse(obj._lock.locked) def test_name(self): """ A method wrapped with @synchronized preserves its name. """ self.assertEqual(Lockable.check.__name__, "check") def test_marked(self): """ A method wrapped with @synchronized is marked as synchronized. """ self.assertEqual(Lockable.check.synchronized, True) crochet-1.0.0/crochet/tests/test_shutdown.py0000644000175000017500000000633412131246444021463 0ustar roaksoaxroaksoax""" Tests for _shutdown. """ from __future__ import absolute_import import sys import subprocess import time from twisted.trial.unittest import TestCase from crochet._shutdown import (Watchdog, FunctionRegistry, _watchdog, register, _registry) class ShutdownTests(TestCase): """ Tests for shutdown registration. """ def test_shutdown(self): """ A function registered with _shutdown.register() is called when the main thread exits. """ program = """\ import threading, sys from crochet._shutdown import register, _watchdog _watchdog.start() end = False def thread(): while not end: pass sys.stdout.write("byebye") sys.stdout.flush() def stop(x, y): # Move this into separate test at some point. assert x == 1 assert y == 2 global end end = True threading.Thread(target=thread).start() register(stop, 1, y=2) sys.exit() """ process = subprocess.Popen([sys.executable, "-c", program], stdout=subprocess.PIPE) result = process.stdout.read() self.assertEqual(process.wait(), 0) self.assertEqual(result, b"byebye") def test_watchdog(self): """ The watchdog thread exits when the thread it is watching exits, and calls its shutdown function. """ done = [] alive = True class FakeThread: def is_alive(self): return alive w = Watchdog(FakeThread(), lambda: done.append(True)) w.start() time.sleep(0.2) self.assertTrue(w.is_alive()) self.assertFalse(done) alive = False time.sleep(0.2) self.assertTrue(done) self.assertFalse(w.is_alive()) def test_api(self): """ The module exposes a shutdown thread that will call a global registry's run(), and a register function tied to the global registry. """ self.assertIsInstance(_registry, FunctionRegistry) self.assertEqual(register, _registry.register) self.assertIsInstance(_watchdog, Watchdog) self.assertEqual(_watchdog._shutdown_function, _registry.run) class FunctionRegistryTests(TestCase): """ Tests for FunctionRegistry. """ def test_called(self): """ Functions registered with a FunctionRegistry are called in reverse order by run(). """ result = [] registry = FunctionRegistry() registry.register(lambda: result.append(1)) registry.register(lambda x: result.append(x), 2) registry.register(lambda y: result.append(y), y=3) registry.run() self.assertEqual(result, [3, 2, 1]) def test_log_errors(self): """ Registered functions that raise an error have the error logged, and run() continues processing. """ result = [] registry = FunctionRegistry() registry.register(lambda: result.append(2)) registry.register(lambda: 1/0) registry.register(lambda: result.append(1)) registry.run() self.assertEqual(result, [1, 2]) excs = self.flushLoggedErrors(ZeroDivisionError) self.assertEqual(len(excs), 1) crochet-1.0.0/crochet/tests/test_api.py0000644000175000017500000006067212230515263020364 0ustar roaksoaxroaksoax""" Tests for the crochet APIs. """ from __future__ import absolute_import import threading import subprocess import time import gc import sys import weakref from twisted.trial.unittest import TestCase, SkipTest from twisted.internet.defer import succeed, Deferred, fail, CancelledError from twisted.python.failure import Failure from twisted.python import threadable from twisted.python.runtime import platform try: from twisted.internet.process import reapAllProcesses except ImportError: reapAllProcesses = None except SyntaxError: if sys.version_info < (3, 3, 0): raise else: # Process support is still not ported to Python 3 on some versions of # Twisted. reapAllProcesses = None from .._eventloop import (EventLoop, EventualResult, TimeoutError, ResultRegistry, ReactorStopped) from .test_setup import FakeReactor from .. import (_main, setup, in_reactor, retrieve_result, _store, no_setup, run_in_reactor, wait_for_reactor) class ResultRegistryTests(TestCase): """ Tests for ResultRegistry. """ def test_stopped_registered(self): """ ResultRegistery.stop() fires registered EventualResult with ReactorStopped. """ registry = ResultRegistry(FakeReactor()) er = EventualResult(None) registry.register(er) registry.stop() self.assertRaises(ReactorStopped, er.wait, timeout=0) def test_stopped_new_registration(self): """ After ResultRegistery.stop() is called subsequent register() calls raise ReactorStopped. """ registry = ResultRegistry(FakeReactor()) er = EventualResult(None) registry.stop() self.assertRaises(ReactorStopped, registry.register, er) def test_stopped_already_have_result(self): """ ResultRegistery.stop() has no impact on registered EventualResult which already have a result. """ registry = ResultRegistry(FakeReactor()) er = EventualResult(succeed(123)) registry.register(er) registry.stop() self.assertEqual(er.wait(), 123) self.assertEqual(er.wait(), 123) self.assertEqual(er.wait(), 123) def test_weakref(self): """ Registering an EventualResult with a ResultRegistry does not prevent it from being garbage collected. """ registry = ResultRegistry(FakeReactor()) er = EventualResult(None) registry.register(er) ref = weakref.ref(er) del er gc.collect() self.assertIdentical(ref(), None) def test_runs_with_lock(self): """ All code in ResultRegistry.stop() and register() is protected by a lock. """ self.assertTrue(ResultRegistry.stop.synchronized) self.assertTrue(ResultRegistry.register.synchronized) class EventualResultTests(TestCase): """ Tests for EventualResult. """ def setUp(self): self.patch(threadable, "isInIOThread", lambda: False) def test_success_result(self): """ wait() returns the value the Deferred fired with. """ dr = EventualResult(succeed(123)) self.assertEqual(dr.wait(), 123) def test_later_success_result(self): """ wait() returns the value the Deferred fired with, in the case where the Deferred is fired after wait() is called. """ d = Deferred() def fireSoon(): import time; time.sleep(0.01) d.callback(345) threading.Thread(target=fireSoon).start() dr = EventualResult(d) self.assertEqual(dr.wait(), 345) def test_success_result_twice(self): """ A second call to wait() returns same value as the first call. """ dr = EventualResult(succeed(123)) self.assertEqual(dr.wait(), 123) self.assertEqual(dr.wait(), 123) def test_failure_result(self): """ wait() raises the exception the Deferred fired with. """ dr = EventualResult(fail(RuntimeError())) self.assertRaises(RuntimeError, dr.wait) def test_later_failure_result(self): """ wait() raises the exception the Deferred fired with, in the case where the Deferred is fired after wait() is called. """ d = Deferred() def fireSoon(): time.sleep(0.01) d.errback(RuntimeError()) threading.Thread(target=fireSoon).start() dr = EventualResult(d) self.assertRaises(RuntimeError, dr.wait) def test_failure_result_twice(self): """ A second call to wait() raises same value as the first call. """ dr = EventualResult(fail(ZeroDivisionError())) self.assertRaises(ZeroDivisionError, dr.wait) self.assertRaises(ZeroDivisionError, dr.wait) def test_timeout(self): """ If no result is available, wait(timeout) will throw a TimeoutError. """ start = time.time() dr = EventualResult(Deferred()) self.assertRaises(TimeoutError, dr.wait, timeout=0.03) self.assertTrue(abs(time.time() - start - 0.03) < 0.005) def test_timeout_twice(self): """ If no result is available, a second call to wait(timeout) will also result in a TimeoutError exception. """ dr = EventualResult(Deferred()) self.assertRaises(TimeoutError, dr.wait, timeout=0.01) self.assertRaises(TimeoutError, dr.wait, timeout=0.01) def test_timeout_then_result(self): """ If a result becomes available after a timeout, a second call to wait() will return it. """ d = Deferred() dr = EventualResult(d) self.assertRaises(TimeoutError, dr.wait, timeout=0.01) d.callback(u"value") self.assertEqual(dr.wait(), u"value") self.assertEqual(dr.wait(), u"value") def test_reactor_thread_disallowed(self): """ wait() cannot be called from the reactor thread. """ self.patch(threadable, "isInIOThread", lambda: True) d = Deferred() dr = EventualResult(d) self.assertRaises(RuntimeError, dr.wait, 0) def test_cancel(self): """ cancel() cancels the wrapped Deferred, running cancellation in the event loop thread. """ reactor = FakeReactor() cancelled = [] def error(f): cancelled.append(reactor.in_call_from_thread) cancelled.append(f) d = Deferred().addErrback(error) dr = EventualResult(d, _reactor=reactor) dr.cancel() self.assertTrue(cancelled[0]) self.assertIsInstance(cancelled[1].value, CancelledError) def test_stash(self): """ EventualResult.stash() stores the object in the global ResultStore. """ dr = EventualResult(Deferred()) uid = dr.stash() self.assertIdentical(dr, _store.retrieve(uid)) def test_original_failure(self): """ original_failure() returns the underlying Failure of the Deferred wrapped by the EventualResult. """ try: 1/0 except: f = Failure() dr = EventualResult(fail(f)) self.assertIdentical(dr.original_failure(), f) def test_original_failure_no_result(self): """ If there is no result yet, original_failure() returns None. """ dr = EventualResult(Deferred()) self.assertIdentical(dr.original_failure(), None) def test_original_failure_not_error(self): """ If the result is not an error, original_failure() returns None. """ dr = EventualResult(succeed(3)) self.assertIdentical(dr.original_failure(), None) def test_error_logged_no_wait(self): """ If the result is an error and wait() was never called, the error will be logged once the EventualResult is garbage-collected. """ dr = EventualResult(fail(ZeroDivisionError())) del dr gc.collect() excs = self.flushLoggedErrors(ZeroDivisionError) self.assertEqual(len(excs), 1) def test_error_logged_wait_timeout(self): """ If the result is an error and wait() was called but timed out, the error will be logged once the EventualResult is garbage-collected. """ d = Deferred() dr = EventualResult(d) try: dr.wait(0) except TimeoutError: pass d.errback(ZeroDivisionError()) del dr if sys.version_info[0] == 2: sys.exc_clear() gc.collect() excs = self.flushLoggedErrors(ZeroDivisionError) self.assertEqual(len(excs), 1) def test_error_after_gc_logged(self): """ If the result is an error that occurs after all user references to the EventualResult are lost, the error is still logged. """ d = Deferred() dr = EventualResult(d) del dr d.errback(ZeroDivisionError()) gc.collect() excs = self.flushLoggedErrors(ZeroDivisionError) self.assertEqual(len(excs), 1) def test_control_c_is_possible(self): """ If you're wait()ing on an EventualResult in main thread, make sure the KeyboardInterrupt happens in timely manner. """ program = """\ import os, threading, signal, time, sys import crochet crochet.setup() from twisted.internet.defer import Deferred def interrupt(): time.sleep(0.1) # Make sure we've hit wait() os.kill(os.getpid(), signal.SIGINT) time.sleep(1) # Still running, test shall fail... os.kill(os.getpid(), signal.SIGKILL) t = threading.Thread(target=interrupt) t.setDaemon(True) t.start() d = Deferred() e = crochet.EventualResult(d) try: # Queue.get() has special non-interruptible behavior if not given timeout, # so don't give timeout here. e.wait() except KeyboardInterrupt: sys.exit(23) """ process = subprocess.Popen([sys.executable, "-c", program]) self.assertEqual(process.wait(), 23) def test_reactor_stop_unblocks_EventualResult(self): """ Any EventualResult.wait() calls still waiting when the reactor has stopped will get a ReactorStopped exception. """ raise SkipTest("Not done yet.") def test_connect_deferred(self): """ If an EventualResult is created with None, EventualResult._connect_deferred can be called later to register a Deferred as the one it is wrapping. """ er = EventualResult(None) self.assertRaises(TimeoutError, er.wait, 0) d = Deferred() er._connect_deferred(d) self.assertRaises(TimeoutError, er.wait, 0) d.callback(123) self.assertEqual(er.wait(), 123) def test_reactor_stop_unblocks_EventualResult(self): """ Any EventualResult.wait() calls still waiting when the reactor has stopped will get a ReactorStopped exception. """ program = """\ import os, threading, signal, time, sys from twisted.internet.defer import Deferred from twisted.internet import reactor import crochet crochet.setup() @crochet.run_in_reactor def run(): reactor.callLater(0.1, reactor.stop) return Deferred() er = run() try: er.wait(timeout=10) except crochet.ReactorStopped: sys.exit(23) """ process = subprocess.Popen([sys.executable, "-c", program]) self.assertEqual(process.wait(), 23) def test_reactor_stop_unblocks_EventualResult_in_threadpool(self): """ Any EventualResult.wait() calls still waiting when the reactor has stopped will get a ReactorStopped exception, even if it is running in Twisted's thread pool. """ program = """\ import os, threading, signal, time, sys from twisted.internet.defer import Deferred from twisted.internet import reactor import crochet crochet.setup() @crochet.run_in_reactor def run(): reactor.callLater(0.1, reactor.stop) return Deferred() result = [13] def inthread(): er = run() try: er.wait(timeout=10) except crochet.ReactorStopped: result[0] = 23 reactor.callInThread(inthread) time.sleep(1) sys.exit(result[0]) """ process = subprocess.Popen([sys.executable, "-c", program]) self.assertEqual(process.wait(), 23) def test_immediate_cancel(self): """ Immediately cancelling the result of @run_in_reactor function will still cancel the Deferred. """ # This depends on the way reactor runs callFromThread calls, so need # real functional test. program = """\ import os, threading, signal, time, sys from twisted.internet.defer import Deferred, CancelledError import crochet crochet.setup() @crochet.run_in_reactor def run(): return Deferred() er = run() er.cancel() try: er.wait(1) except CancelledError: sys.exit(23) else: sys.exit(3) """ process = subprocess.Popen([sys.executable, "-c", program]) self.assertEqual(process.wait(), 23) class InReactorTests(TestCase): """ Tests for the deprecated in_reactor decorator. """ def test_name(self): """ The function decorated with in_reactor has the same name as the original function. """ c = EventLoop(FakeReactor(), lambda f, g: None) @c.in_reactor def some_name(reactor): pass self.assertEqual(some_name.__name__, "some_name") def test_in_reactor_thread(self): """ The function decorated with in_reactor is run in the reactor thread, and takes the reactor as its first argument. """ myreactor = FakeReactor() c = EventLoop(myreactor, lambda f, g: None) calls = [] @c.in_reactor def func(reactor, a, b, c): self.assertIdentical(reactor, myreactor) self.assertTrue(reactor.in_call_from_thread) calls.append((a, b, c)) func(1, 2, c=3) self.assertEqual(calls, [(1, 2, 3)]) def test_run_in_reactor_wrapper(self): """ in_reactor is implemented on top of run_in_reactor. """ wrapped = [False] def fake_run_in_reactor(function): def wrapper(*args, **kwargs): wrapped[0] = True result = function(*args, **kwargs) wrapped[0] = False return result return wrapper myreactor = FakeReactor() c = EventLoop(myreactor, lambda f, g: None) c.run_in_reactor = fake_run_in_reactor @c.in_reactor def func(reactor): self.assertTrue(wrapped[0]) return 17 result = func() self.assertFalse(wrapped[0]) self.assertEqual(result, 17) class RunInReactorTests(TestCase): """ Tests for the run_in_reactor decorator. """ def test_name(self): """ The function decorated with run_in_reactor has the same name as the original function. """ c = EventLoop(FakeReactor(), lambda f, g: None) @c.run_in_reactor def some_name(): pass self.assertEqual(some_name.__name__, "some_name") def test_run_in_reactor_thread(self): """ The function decorated with run_in_reactor is run in the reactor thread. """ myreactor = FakeReactor() c = EventLoop(myreactor, lambda f, g: None) calls = [] @c.run_in_reactor def func(a, b, c): self.assertTrue(myreactor.in_call_from_thread) calls.append((a, b, c)) func(1, 2, c=3) self.assertEqual(calls, [(1, 2, 3)]) def make_wrapped_function(self): """ Return a function wrapped with run_in_reactor that returns its first argument. """ myreactor = FakeReactor() c = EventLoop(myreactor, lambda f, g: None) @c.run_in_reactor def passthrough(argument): return argument return passthrough def test_deferred_success_result(self): """ If the underlying function returns a Deferred, the wrapper returns a EventualResult hooked up to the Deferred. """ passthrough = self.make_wrapped_function() result = passthrough(succeed(123)) self.assertIsInstance(result, EventualResult) self.assertEqual(result.wait(), 123) def test_deferred_failure_result(self): """ If the underlying function returns a Deferred, the wrapper returns a EventualResult hooked up to the Deferred that can deal with failures as well. """ passthrough = self.make_wrapped_function() result = passthrough(fail(ZeroDivisionError())) self.assertIsInstance(result, EventualResult) self.assertRaises(ZeroDivisionError, result.wait) def test_regular_result(self): """ If the underlying function returns a non-Deferred, the wrapper returns a EventualResult hooked up to a Deferred wrapping the result. """ passthrough = self.make_wrapped_function() result = passthrough(123) self.assertIsInstance(result, EventualResult) self.assertEqual(result.wait(), 123) def test_exception_result(self): """ If the underlying function throws an exception, the wrapper returns a EventualResult hooked up to a Deferred wrapping the exception. """ myreactor = FakeReactor() c = EventLoop(myreactor, lambda f, g: None) @c.run_in_reactor def raiser(): 1/0 result = raiser() self.assertIsInstance(result, EventualResult) self.assertRaises(ZeroDivisionError, result.wait) def test_registry(self): """ @run_in_reactor registers the EventualResult in the ResultRegistry. """ myreactor = FakeReactor() c = EventLoop(myreactor, lambda f, g: None) @c.run_in_reactor def run(): return result = run() self.assertIn(result, c._registry._results) class WaitForReactorTests(TestCase): """ Tests for the wait_for_reactor decorator. """ def test_name(self): """ The function decorated with run_in_reactor has the same name as the original function. """ c = EventLoop(FakeReactor(), lambda f, g: None) @c.wait_for_reactor def some_name(): pass self.assertEqual(some_name.__name__, "some_name") def test_reactor_thread_disallowed(self): """ Functions decorated with wait_for_reactor() cannot be called from the reactor thread. """ self.patch(threadable, "isInIOThread", lambda: True) c = EventLoop(FakeReactor(), lambda f, g: None) @c.wait_for_reactor def f(): pass self.assertRaises(RuntimeError, f) def test_wait_for_reactor_thread(self): """ The function decorated with wait_for_reactor is run in the reactor thread. """ myreactor = FakeReactor() c = EventLoop(myreactor, lambda f, g: None) calls = [] @c.wait_for_reactor def func(a, b, c): self.assertTrue(myreactor.in_call_from_thread) calls.append((a, b, c)) func(1, 2, c=3) self.assertEqual(calls, [(1, 2, 3)]) def make_wrapped_function(self): """ Return a function wrapped with wait_for_reactor that returns its first argument. """ myreactor = FakeReactor() c = EventLoop(myreactor, lambda f, g: None) @c.wait_for_reactor def passthrough(argument): return argument return passthrough def test_deferred_success_result(self): """ If the underlying function returns a Deferred, the wrapper returns a the Deferred's result. """ passthrough = self.make_wrapped_function() result = passthrough(succeed(123)) self.assertEqual(result, 123) def test_deferred_failure_result(self): """ If the underlying function returns a Deferred with an errback, the wrapper throws an exception. """ passthrough = self.make_wrapped_function() self.assertRaises( ZeroDivisionError, passthrough, fail(ZeroDivisionError())) def test_regular_result(self): """ If the underlying function returns a non-Deferred, the wrapper returns that result. """ passthrough = self.make_wrapped_function() result = passthrough(123) self.assertEqual(result, 123) def test_exception_result(self): """ If the underlying function throws an exception, the wrapper raises that exception. """ myreactor = FakeReactor() c = EventLoop(myreactor, lambda f, g: None) @c.wait_for_reactor def raiser(): 1/0 self.assertRaises(ZeroDivisionError, raiser) def test_control_c_is_possible(self): """ A call to a decorated function responds to a Ctrl-C (i.e. with a KeyboardInterrupt) in a timely manner. """ program = """\ import os, threading, signal, time, sys import crochet crochet.setup() from twisted.internet.defer import Deferred def interrupt(): time.sleep(0.1) # Make sure we've hit wait() os.kill(os.getpid(), signal.SIGINT) time.sleep(1) # Still running, test shall fail... os.kill(os.getpid(), signal.SIGKILL) t = threading.Thread(target=interrupt) t.setDaemon(True) t.start() @crochet.wait_for_reactor def wait(): return Deferred() try: wait() except KeyboardInterrupt: sys.exit(23) """ process = subprocess.Popen([sys.executable, "-c", program]) self.assertEqual(process.wait(), 23) def test_reactor_stop_unblocks(self): """ Any @wait_for_reactor-decorated calls still waiting when the reactor has stopped will get a ReactorStopped exception. """ program = """\ import os, threading, signal, time, sys from twisted.internet.defer import Deferred from twisted.internet import reactor import crochet crochet.setup() @crochet.wait_for_reactor def run(): reactor.callLater(0.1, reactor.stop) return Deferred() try: er = run() except crochet.ReactorStopped: sys.exit(23) """ process = subprocess.Popen([sys.executable, "-c", program]) self.assertEqual(process.wait(), 23) class PublicAPITests(TestCase): """ Tests for the public API. """ def test_no_sideeffects(self): """ Creating an EventLoop object, as is done in crochet.__init__, does not call any methods on the objects it is created with. """ c = EventLoop(None, lambda f, g: 1/0, lambda *args: 1/0, watchdog_thread=object(), reapAllProcesses=lambda: 1/0) del c def test_eventloop_api(self): """ An EventLoop object configured with the real reactor and _shutdown.register is exposed via its public methods. """ from twisted.internet import reactor from twisted.python.log import startLoggingWithObserver from crochet import _shutdown self.assertIsInstance(_main, EventLoop) self.assertEqual(_main.setup, setup) self.assertEqual(_main.no_setup, no_setup) self.assertEqual(_main.in_reactor, in_reactor) self.assertEqual(_main.run_in_reactor, run_in_reactor) self.assertEqual(_main.wait_for_reactor, wait_for_reactor) self.assertIdentical(_main._reactor, reactor) self.assertIdentical(_main._atexit_register, _shutdown.register) self.assertIdentical(_main._startLoggingWithObserver, startLoggingWithObserver) self.assertIdentical(_main._watchdog_thread, _shutdown._watchdog) def test_retrieve_result(self): """ retrieve_result() calls retrieve() on the global ResultStore. """ dr = EventualResult(Deferred()) uid = dr.stash() self.assertIdentical(dr, retrieve_result(uid)) def test_reapAllProcesses(self): """ An EventLoop object configured with the real reapAllProcesses on POSIX plaforms. """ self.assertIdentical(_main._reapAllProcesses, reapAllProcesses) if platform.type != "posix": test_reapAllProcesses.skip = "Only relevant on POSIX platforms" if reapAllProcesses is None: test_reapAllProcesses.skip = "Twisted does not yet support processes" crochet-1.0.0/crochet/tests/test_resultstore.py0000644000175000017500000000440112215625072022175 0ustar roaksoaxroaksoax""" Tests for _resultstore. """ from twisted.trial.unittest import TestCase from twisted.internet.defer import Deferred, fail, succeed from .._resultstore import ResultStore from .._eventloop import EventualResult class ResultStoreTests(TestCase): """ Tests for ResultStore. """ def test_store_and_retrieve(self): """ EventualResult instances be be stored in a ResultStore and then retrieved using the id returned from store(). """ store = ResultStore() dr = EventualResult(Deferred()) uid = store.store(dr) self.assertIdentical(store.retrieve(uid), dr) def test_retrieve_only_once(self): """ Once a result is retrieved, it can no longer be retrieved again. """ store = ResultStore() dr = EventualResult(Deferred()) uid = store.store(dr) store.retrieve(uid) self.assertRaises(KeyError, store.retrieve, uid) def test_synchronized(self): """ store() and retrieve() are synchronized. """ self.assertTrue(ResultStore.store.synchronized) self.assertTrue(ResultStore.retrieve.synchronized) self.assertTrue(ResultStore.log_errors.synchronized) def test_uniqueness(self): """ Each store() operation returns a larger number, ensuring uniqueness. """ store = ResultStore() dr = EventualResult(Deferred()) previous = store.store(dr) for i in range(100): store.retrieve(previous) dr = EventualResult(Deferred()) uid = store.store(dr) self.assertTrue(uid > previous) previous = uid def test_log_errors(self): """ Unretrieved EventualResults have their errors, if any, logged on shutdown. """ store = ResultStore() store.store(EventualResult(Deferred())) store.store(EventualResult(fail(ZeroDivisionError()))) store.store(EventualResult(succeed(1))) store.store(EventualResult(fail(RuntimeError()))) store.log_errors() excs = self.flushLoggedErrors(ZeroDivisionError) self.assertEqual(len(excs), 1) excs = self.flushLoggedErrors(RuntimeError) self.assertEqual(len(excs), 1) crochet-1.0.0/crochet/tests/test_setup.py0000644000175000017500000002074612230515344020751 0ustar roaksoaxroaksoax""" Tests for the initial setup. """ from __future__ import absolute_import import threading from twisted.trial.unittest import TestCase from twisted.python.log import PythonLoggingObserver from twisted.python.runtime import platform from twisted.internet.task import Clock from .._eventloop import EventLoop, ThreadLogObserver, _store class FakeReactor(Clock): """ A fake reactor for testing purposes. """ thread_id = None runs = 0 in_call_from_thread = False def __init__(self): Clock.__init__(self) self.started = threading.Event() self.stopping = False self.events = [] def run(self, installSignalHandlers=True): self.runs += 1 self.thread_id = threading.current_thread().ident self.installSignalHandlers = installSignalHandlers self.started.set() def callFromThread(self, f, *args, **kwargs): self.in_call_from_thread = True f(*args, **kwargs) self.in_call_from_thread = False def stop(self): self.stopping = True def addSystemEventTrigger(self, when, event, f): self.events.append((when, event, f)) class FakeThread: started = False def start(self): self.started = True class SetupTests(TestCase): """ Tests for setup(). """ def test_first_runs_reactor(self): """ With it first call, setup() runs the reactor in a thread. """ reactor = FakeReactor() EventLoop(reactor, lambda f, *g: None).setup() reactor.started.wait(5) self.assertNotEqual(reactor.thread_id, None) self.assertNotEqual(reactor.thread_id, threading.current_thread().ident) self.assertFalse(reactor.installSignalHandlers) def test_second_does_nothing(self): """ The second call to setup() does nothing. """ reactor = FakeReactor() s = EventLoop(reactor, lambda f, *g: None) s.setup() s.setup() reactor.started.wait(5) self.assertEqual(reactor.runs, 1) def test_stop_on_exit(self): """ setup() registers an exit handler that stops the reactor, and an exit handler that logs stashed EventualResults. """ atexit = [] reactor = FakeReactor() s = EventLoop(reactor, lambda f, *args: atexit.append((f, args))) s.setup() self.assertEqual(len(atexit), 2) self.assertFalse(reactor.stopping) f, args = atexit[0] self.assertEqual(f, reactor.callFromThread) self.assertEqual(args, (reactor.stop,)) f(*args) self.assertTrue(reactor.stopping) f, args = atexit[1] self.assertEqual(f, _store.log_errors) self.assertEqual(args, ()) f(*args) # make sure it doesn't throw an exception def test_runs_with_lock(self): """ All code in setup() and no_setup() is protected by a lock. """ self.assertTrue(EventLoop.setup.synchronized) self.assertTrue(EventLoop.no_setup.synchronized) def test_logging(self): """ setup() registers a PythonLoggingObserver wrapped in a ThreadLogObserver, removing the default log observer. """ logging = [] def fakeStartLoggingWithObserver(observer, setStdout=1): self.assertIsInstance(observer, ThreadLogObserver) wrapped = observer._observer expected = PythonLoggingObserver.emit # Python 3 and 2 differ in value of __func__: expected = getattr(expected, "__func__", expected) self.assertIdentical(wrapped.__func__, expected) self.assertEqual(setStdout, False) self.assertTrue(reactor.in_call_from_thread) logging.append(observer) reactor = FakeReactor() loop = EventLoop(reactor, lambda f, *g: None, fakeStartLoggingWithObserver) loop.setup() self.assertTrue(logging) logging[0].stop() def test_stop_logging_on_exit(self): """ setup() registers a reactor shutdown event that stops the logging thread. """ observers = [] reactor = FakeReactor() s = EventLoop(reactor, lambda f, *arg: None, lambda observer, setStdout=1: observers.append(observer)) s.setup() self.addCleanup(observers[0].stop) self.assertIn(("after", "shutdown", observers[0].stop), reactor.events) def test_start_watchdog_thread(self): """ setup() starts the shutdown watchdog thread. """ thread = FakeThread() reactor = FakeReactor() loop = EventLoop(reactor, lambda *args: None, watchdog_thread=thread) loop.setup() self.assertTrue(thread.started) def test_no_setup(self): """ If called first, no_setup() makes subsequent calls to setup() do nothing. """ observers = [] atexit = [] thread = FakeThread() reactor = FakeReactor() loop = EventLoop(reactor, lambda f, *arg: atexit.append(f), lambda observer, *a, **kw: observers.append(observer), watchdog_thread=thread) loop.no_setup() loop.setup() self.assertFalse(observers) self.assertFalse(atexit) self.assertFalse(reactor.runs) self.assertFalse(thread.started) def test_no_setup_after_setup(self): """ If called after setup(), no_setup() throws an exception. """ reactor = FakeReactor() s = EventLoop(reactor, lambda f, *g: None) s.setup() self.assertRaises(RuntimeError, s.no_setup) def test_setup_registry_shutdown(self): """ ResultRegistry.stop() is registered to run before reactor shutdown by setup(). """ reactor = FakeReactor() s = EventLoop(reactor, lambda f, *g: None) s.setup() self.assertEqual(reactor.events, [("before", "shutdown", s._registry.stop)]) def test_no_setup_registry_shutdown(self): """ ResultRegistry.stop() is registered to run before reactor shutdown by setup(). """ reactor = FakeReactor() s = EventLoop(reactor, lambda f, *g: None) s.no_setup() self.assertEqual(reactor.events, [("before", "shutdown", s._registry.stop)]) class ProcessSetupTests(TestCase): """ setup() enables support for IReactorProcess on POSIX plaforms. """ def test_posix(self): """ On POSIX systems, setup() installs a LoopingCall that runs t.i.process.reapAllProcesses() 10 times a second. """ reactor = FakeReactor() reaps = [] s = EventLoop(reactor, lambda f, *g: None, reapAllProcesses=lambda: reaps.append(1)) s.setup() reactor.advance(0.1) self.assertEquals(reaps, [1]) reactor.advance(0.1) self.assertEquals(reaps, [1, 1]) reactor.advance(0.1) self.assertEquals(reaps, [1, 1, 1]) if platform.type != "posix": test_posix.skip = "SIGCHLD is a POSIX-specific issue" def test_non_posix(self): """ On POSIX systems, setup() does not install a LoopingCall. """ reactor = FakeReactor() s = EventLoop(reactor, lambda f, *g: None) s.setup() self.assertFalse(reactor.getDelayedCalls()) if platform.type == "posix": test_non_posix.skip = "SIGCHLD is a POSIX-specific issue" class ThreadLogObserverTest(TestCase): """ Tests for ThreadLogObserver. """ def test_stop(self): """ ThreadLogObserver.stop() stops the thread started in __init__. """ threadLog = ThreadLogObserver(None) self.assertTrue(threadLog._thread.is_alive()) threadLog.stop() threadLog._thread.join() self.assertFalse(threadLog._thread.is_alive()) def test_emit(self): """ ThreadLogObserver.emit runs the wrapped observer's in its thread, with the given message. """ log = [] def observer(msg): log.append((threading.current_thread().ident, msg)) threadLog = ThreadLogObserver(observer) ident = threadLog._thread.ident msg1 = {} msg2 = {"a": "b"} threadLog(msg1) threadLog(msg2) threadLog.stop() # Wait for writing to finish: threadLog._thread.join() self.assertEqual(log, [(ident, msg1), (ident, msg2)]) crochet-1.0.0/crochet/_util.py0000644000175000017500000000055612127024027016517 0ustar roaksoaxroaksoax""" Utility functions and classes. """ from functools import wraps def synchronized(method): """ Decorator that wraps a method with an acquire/release of self._lock. """ @wraps(method) def synced(self, *args, **kwargs): with self._lock: return method(self, *args, **kwargs) synced.synchronized = True return synced crochet-1.0.0/crochet/_shutdown.py0000644000175000017500000000300312130041116017373 0ustar roaksoaxroaksoax""" Support for calling code when the main thread exits. atexit cannot be used, since registered atexit functions only run after *all* threads have exited. The watchdog thread will be started by crochet.setup(). """ import threading import time from twisted.python import log class Watchdog(threading.Thread): """ Watch a given thread, call a list of functions when that thread exits. """ def __init__(self, canary, shutdown_function): threading.Thread.__init__(self) self._canary = canary self._shutdown_function = shutdown_function def run(self): while self._canary.is_alive(): time.sleep(0.1) self._shutdown_function() class FunctionRegistry(object): """ A registry of functions that can be called all at once. """ def __init__(self): self._functions = [] def register(self, f, *args, **kwargs): """ Register a function and arguments to be called later. """ self._functions.append(lambda: f(*args, **kwargs)) def run(self): """ Run all registered functions in reverse order of registration. """ for f in reversed(self._functions): try: f() except: log.err() # This is... fragile. Not sure how else to do it though. _registry = FunctionRegistry() _watchdog = Watchdog([t for t in threading.enumerate() if t.name == "MainThread"][0], _registry.run) register = _registry.register crochet-1.0.0/crochet/_resultstore.py0000644000175000017500000000272012215625072020135 0ustar roaksoaxroaksoax""" In-memory store for EventualResults. """ import threading from twisted.python import log from ._util import synchronized class ResultStore(object): """ An in-memory store for EventualResult instances. Each EventualResult put in the store gets a unique identifier, which can be used to retrieve it later. This is useful for referring to results in e.g. web sessions. EventualResults that are not retrieved by shutdown will be logged if they have an error result. """ def __init__(self): self._counter = 0 self._stored = {} self._lock = threading.Lock() @synchronized def store(self, deferred_result): """ Store a EventualResult. Return an integer, a unique identifier that can be used to retrieve the object. """ self._counter += 1 self._stored[self._counter] = deferred_result return self._counter @synchronized def retrieve(self, result_id): """ Return the given EventualResult, and remove it from the store. """ return self._stored.pop(result_id) @synchronized def log_errors(self): """ Log errors for all stored EventualResults that have error results. """ for result in self._stored.values(): failure = result.original_failure() if failure is not None: log.err(failure, "Unhandled error in stashed EventualResult:") crochet-1.0.0/setup.cfg0000644000175000017500000000007312230523106015212 0ustar roaksoaxroaksoax[egg_info] tag_build = tag_date = 0 tag_svn_revision = 0