From 3cfb32d8a2b55592bc514ec3114b163afbb7e598 Mon Sep 17 00:00:00 2001 From: chang-ning Date: Mon, 29 Feb 2016 11:14:38 +0800 Subject: [PATCH] First commit of Python cheatsheet Separate from old "cheatsheet" project --- .gitignore | 3 + Makefile | 5 + README.rst | 12 + app.py | 24 + docs/Makefile | 216 ++++++ docs/source/conf.py | 294 ++++++++ docs/source/index.rst | 29 + docs/source/notes/python-asyncio.rst | 628 ++++++++++++++++ docs/source/notes/python-basic.rst | 870 +++++++++++++++++++++++ docs/source/notes/python-capi.rst | 384 ++++++++++ docs/source/notes/python-concurrency.rst | 834 ++++++++++++++++++++++ docs/source/notes/python-cstyle.rst | 642 +++++++++++++++++ docs/source/notes/python-generator.rst | 728 +++++++++++++++++++ docs/source/notes/python-socket.rst | 437 ++++++++++++ docs/source/notes/python-tests.rst | 639 +++++++++++++++++ requirements.txt | 18 + 16 files changed, 5763 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 README.rst create mode 100644 app.py create mode 100644 docs/Makefile create mode 100644 docs/source/conf.py create mode 100644 docs/source/index.rst create mode 100644 docs/source/notes/python-asyncio.rst create mode 100644 docs/source/notes/python-basic.rst create mode 100644 docs/source/notes/python-capi.rst create mode 100644 docs/source/notes/python-concurrency.rst create mode 100644 docs/source/notes/python-cstyle.rst create mode 100644 docs/source/notes/python-generator.rst create mode 100644 docs/source/notes/python-socket.rst create mode 100644 docs/source/notes/python-tests.rst create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..346dd5e8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*.swp +*.pyc +docs/build/ diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..a07b9dbb --- /dev/null +++ b/Makefile @@ -0,0 +1,5 @@ +.PHONY: build +build: html + +%: + cd docs && make $@ diff --git a/README.rst b/README.rst new file mode 100644 index 00000000..33389ef3 --- /dev/null +++ b/README.rst @@ -0,0 +1,12 @@ +How to run the server +===================== + +.. code-block:: console + + $ virtualenv venv + $ . venv/bin/activate + $ pip install -r requirements.txt + $ make + $ python app.py + + # URL: localhost:5000 diff --git a/app.py b/app.py new file mode 100644 index 00000000..430adf41 --- /dev/null +++ b/app.py @@ -0,0 +1,24 @@ +""" +This is a simple cheatsheet webapp. +""" +import os + +from flask import Flask, send_from_directory + +DIR = os.path.dirname(os.path.realpath(__file__)) +ROOT = os.path.join(DIR, 'docs', 'build', 'html') + +app = Flask(__name__) + +@app.route('/') +def static_proxy(path): + """Static files proxy""" + return send_from_directory(ROOT, path) + +@app.route('/') +def index_redirection(): + """Redirecting index file""" + return send_from_directory(ROOT, 'index.html') + +if __name__ == "__main__": + app.run(debug=True) diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..30bde125 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,216 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = build + +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) +$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source + +.PHONY: help +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " applehelp to make an Apple Help Book" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " coverage to run coverage check of the documentation (if enabled)" + +.PHONY: clean +clean: + rm -rf $(BUILDDIR)/* + +.PHONY: html +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +.PHONY: dirhtml +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +.PHONY: singlehtml +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +.PHONY: pickle +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +.PHONY: json +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +.PHONY: htmlhelp +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +.PHONY: qthelp +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/python-cheatsheet.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/python-cheatsheet.qhc" + +.PHONY: applehelp +applehelp: + $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp + @echo + @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." + @echo "N.B. You won't be able to view it unless you put it in" \ + "~/Library/Documentation/Help or install it in your application" \ + "bundle." + +.PHONY: devhelp +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/python-cheatsheet" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/python-cheatsheet" + @echo "# devhelp" + +.PHONY: epub +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +.PHONY: latex +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +.PHONY: latexpdf +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: latexpdfja +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: text +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +.PHONY: man +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +.PHONY: texinfo +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +.PHONY: info +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +.PHONY: gettext +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +.PHONY: changes +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +.PHONY: linkcheck +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +.PHONY: doctest +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +.PHONY: coverage +coverage: + $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage + @echo "Testing of coverage in the sources finished, look at the " \ + "results in $(BUILDDIR)/coverage/python.txt." + +.PHONY: xml +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +.PHONY: pseudoxml +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 00000000..dd12b88e --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,294 @@ +# -*- coding: utf-8 -*- +# +# python-cheatsheet documentation build configuration file, created by +# sphinx-quickstart on Sun Feb 28 09:26:04 2016. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.mathjax', + 'sphinx.ext.viewcode', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'python-cheatsheet' +copyright = u'2016, crazyguitar' +author = u'crazyguitar' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'0.1.0' +# The full version, including alpha/beta/rc tags. +release = u'0.1.0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +html_theme_options = { + 'show_powered_by': False, + 'github_user': 'crazyguitar', + 'github_repo': 'cheatsheet', + 'github_banner': True, + 'show_related': False +} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' +#html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# Now only 'ja' uses this config value +#html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +#html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'python-cheatsheetdoc' + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', + +# Latex figure (float) alignment +#'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'python-cheatsheet.tex', u'python-cheatsheet Documentation', + u'crazyguitar', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'python-cheatsheet', u'python-cheatsheet Documentation', + [author], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'python-cheatsheet', u'python-cheatsheet Documentation', + author, 'python-cheatsheet', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 00000000..ae3a4586 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,29 @@ +.. python-cheatsheet documentation master file, created by + sphinx-quickstart on Sun Feb 28 09:26:04 2016. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to python-cheatsheet's documentation! +============================================= + +Contents: + +.. toctree:: + :maxdepth: 2 + + notes/python-basic + notes/python-generator + notes/python-socket + notes/python-concurrency + notes/python-asyncio + notes/python-tests + notes/python-capi + notes/python-cstyle + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`search` + diff --git a/docs/source/notes/python-asyncio.rst b/docs/source/notes/python-asyncio.rst new file mode 100644 index 00000000..55a1eace --- /dev/null +++ b/docs/source/notes/python-asyncio.rst @@ -0,0 +1,628 @@ +Python asyncio cheatsheet +========================= + +What is @asyncio.coroutine? +--------------------------- + +.. code-block:: python + + import asyncio + import inspect + from functools import wraps + + Future = asyncio.futures.Future + def coroutine(func): + """Simple prototype of coroutine""" + @wraps(func) + def coro(*a, **k): + res = func(*a, **k) + if isinstance(res, Future) or inspect.isgenerator(res): + res = yield from res + return res + return coro + + @coroutine + def foo(): + yield from asyncio.sleep(1) + print("Hello Foo") + + @asyncio.coroutine + def bar(): + print("Hello Bar") + + loop = asyncio.get_event_loop() + tasks = [loop.create_task(foo()), + loop.create_task(bar())] + loop.run_until_complete( + asyncio.wait(tasks)) + loop.close() + +output: + +.. code-block:: console + + $ python test.py + Hello Bar + Hello Foo + + +What is a Task? +--------------- + +.. code-block:: python + + # goal: supervise coroutine run state + # ref: asyncio/tasks.py + + import asyncio + Future = asyncio.futures.Future + + class Task(Future): + """Simple prototype of Task""" + + def __init__(self, gen, *,loop): + super().__init__(loop=loop) + self._gen = gen + self._loop.call_soon(self._step) + + def _step(self, val=None, exc=None): + try: + if exc: + f = self._gen.throw(exc) + else: + f = self._gen.send(val) + except StopIteration as e: + self.set_result(e.value) + except Exception as e: + self.set_exception(e) + else: + f.add_done_callback( + self._wakeup) + + def _wakeup(self, fut): + try: + res = fut.result() + except Exception as e: + self._step(None, e) + else: + self._step(res, None) + + @asyncio.coroutine + def foo(): + yield from asyncio.sleep(3) + print("Hello Foo") + + @asyncio.coroutine + def bar(): + yield from asyncio.sleep(1) + print("Hello Bar") + + loop = asyncio.get_event_loop() + tasks = [Task(foo(), loop=loop), + loop.create_task(bar())] + loop.run_until_complete( + asyncio.wait(tasks)) + loop.close() + +output: + +.. code-block:: console + + $ python test.py + Hello Bar + hello Foo + + +What event loop doing? (Without polling) +---------------------------------------- + +.. code-block:: python + + import asyncio + from collections import deque + + def done_callback(fut): + fut._loop.stop() + + class Loop: + """Simple event loop prototype""" + + def __init__(self): + self._ready = deque() + self._stopping = False + + def create_task(self, coro): + Task = asyncio.tasks.Task + task = Task(coro, loop=self) + return task + + def run_until_complete(self, fut): + tasks = asyncio.tasks + # get task + fut = tasks.ensure_future( + fut, loop=self) + # add task to ready queue + fut.add_done_callback(done_callback) + # run tasks + self.run_forever() + # remove task from ready queue + fut.remove_done_callback(done_callback) + + def run_forever(self): + """Run tasks until stop""" + try: + while True: + self._run_once() + if self._stopping: + break + finally: + self._stopping = False + + def call_soon(self, cb, *args): + """Append task to ready queue""" + self._ready.append((cb, args)) + def call_exception_handler(self, c): + pass + + def _run_once(self): + """Run task at once""" + ntodo = len(self._ready) + for i in range(ntodo): + t, a = self._ready.popleft() + t(*a) + + def stop(self): + self._stopping = True + + def close(self): + self._ready.clear() + + def get_debug(self): + return False + + @asyncio.coroutine + def foo(): + print("Foo") + + @asyncio.coroutine + def bar(): + print("Bar") + + loop = Loop() + tasks = [loop.create_task(foo()), + loop.create_task(bar())] + loop.run_until_complete( + asyncio.wait(tasks)) + loop.close() + + output: + +.. code-block:: console + + $ python test.py + Foo + Bar + +Socket with asyncio +------------------- + +.. code-block:: python + + import asyncio + import socket + + host = 'localhost' + port = 9527 + loop = asyncio.get_event_loop() + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.setblocking(False) + s.bind((host, port)) + s.listen(10) + + async def handler(conn): + while True: + msg = await loop.sock_recv(conn, 1024) + if not msg: + break + await loop.sock_sendall(conn, msg) + conn.close() + + async def server(): + while True: + conn, addr = await loop.sock_accept(s) + loop.create_task(handler(conn)) + + loop.create_task(server()) + loop.run_forever() + loop.close() + +output: (bash 1) + +.. code-block:: console + + $ nc localhost 9527 + Hello + Hello + +output: (bash 2) + +.. code-block:: console + + $ nc localhost 9527 + World + World + + +Event Loop with polling +----------------------- + +.. code-block:: python + + # using selectors + # ref: PyCon 2015 - David Beazley + + import asyncio + import socket + import selectors + from collections import deque + + @asyncio.coroutine + def read_wait(s): + yield 'read_wait', s + + @asyncio.coroutine + def write_wait(s): + yield 'write_wait', s + + class Loop: + """Simple loop prototype""" + + def __init__(self): + self.ready = deque() + self.selector = selectors.DefaultSelector() + + @asyncio.coroutine + def sock_accept(self, s): + yield from read_wait(s) + return s.accept() + + @asyncio.coroutine + def sock_recv(self, c, mb): + yield from read_wait(c) + return c.recv(mb) + + @asyncio.coroutine + def sock_sendall(self, c, m): + while m: + yield from write_wait(c) + nsent = c.send(m) + m = m[nsent:] + + def create_task(self, coro): + self.ready.append(coro) + + def run_forever(self): + while True: + self._run_once() + + def _run_once(self): + while not self.ready: + events = self.selector.select() + for k, _ in events: + self.ready.append(k.data) + self.selector.unregister(k.fileobj) + + while self.ready: + self.cur_t = ready.popleft() + try: + op, *a = self.cur_t.send(None) + getattr(self, op)(*a) + except StopIteration: + pass + + def read_wait(self, s): + self.selector.register(s, selectors.EVENT_READ, self.cur_t) + + def write_wait(self, s): + self.selector.register(s, selectors.EVENT_WRITE, self.cur_t) + + loop = Loop() + host = 'localhost' + port = 9527 + + s = socket.socket( + socket.AF_INET, + socket.SOCK_STREAM, 0) + s.setsockopt( + socket.SOL_SOCKET, + socket.SO_REUSEADDR, 1) + s.setblocking(False) + s.bind((host, port)) + s.listen(10) + + @asyncio.coroutine + def handler(c): + while True: + msg = yield from loop.sock_recv(c, 1024) + if not msg: + break + yield from loop.sock_sendall(c, msg) + c.close() + + @asyncio.coroutine + def server(): + while True: + c, addr = yield from loop.sock_accept(s) + loop.create_task(handler(c)) + + loop.create_task(server()) + loop.run_forever() + + +Inline callback +--------------- + +.. code-block:: python + + >>> import asyncio + >>> async def foo(): + ... await asyncio.sleep(1) + ... return "foo done" + ... + >>> async def bar(): + ... await asyncio.sleep(.5) + ... return "bar done" + ... + >>> async def ker(): + ... await asyncio.sleep(3) + ... return "ker done" + ... + >>> async def task(): + ... res = await foo() + ... print(res) + ... res = await bar() + ... print(res) + ... res = await ker() + ... print(res) + ... + >>> loop = asyncio.get_event_loop() + >>> loop.run_until_complete(task()) + foo done + bar done + ker done + +Asynchronous Iterator +--------------------- + +.. code-block:: python + + # ref: PEP-0492 + # need Python >= 3.5 + + >>> class AsyncIter: + ... def __init__(self, it): + ... self._it = iter(it) + ... async def __aiter__(self): + ... return self + ... async def __anext__(self): + ... await asyncio.sleep(1) + ... try: + ... val = next(self._it) + ... except StopIteration: + ... raise StopAsyncIteration + ... return val + ... + >>> async def foo(): + ... it = [1,2,3] + ... async for _ in AsyncIter(it): + ... print(_) + ... + >>> loop = asyncio.get_event_loop() + >>> loop.run_until_complete(foo()) + 1 + 2 + 3 + +Asynchronous context manager +---------------------------- + +.. code-block:: python + + # ref: PEP-0492 + # need Python >= 3.5 + + >>> class AsyncCtxMgr: + ... async def __aenter__(self): + ... await asyncio.sleep(3) + ... print("__anter__") + ... return self + ... async def __aexit__(self, *exc): + ... await asyncio.sleep(1) + ... print("__aexit__") + ... + >>> async def hello(): + ... async with AsyncCtxMgr() as m: + ... print("hello block") + ... + >>> async def world(): + ... print("world block") + ... + >>> t = loop.create_task(world()) + >>> loop.run_until_complete(hello()) + world block + __anter__ + hello block + __aexit__ + +Simple asyncio web server +------------------------- + +.. code-block:: python + + import asyncio + import socket + + host = 'localhost' + port = 9527 + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.setblocking(False) + s.bind((host, port)) + s.listen(10) + + loop = asyncio.get_event_loop() + + def make_header(): + header = b"HTTP/1.1 200 OK\r\n" + header += b"Content-Type: text/html\r\n" + header += b"\r\n" + return header + + def make_body(): + resp = b'' + resp += b'

Hello World

' + resp += b'' + return resp + + async def handler(conn): + req = await loop.sock_recv(conn, 1024) + if req: + resp = make_header() + resp += make_body() + await loop.sock_sendall(conn, resp) + conn.close() + + async def server(sock, loop): + while True: + conn, addr = await loop.sock_accept(sock) + loop.create_task(handler(conn)) + + try: + loop.run_until_complete(server(s, loop)) + except KeyboardInterrupt: + pass + finally: + loop.close() + s.close() + # Then open browser with url: localhost:9527 + + +Simple asyncio WSGI web server +------------------------------ + +.. code-block:: python + + # ref: PEP333 + + import asyncio + import socket + import io + import sys + + from flask import Flask, Response + + host = 'localhost' + port = 9527 + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.setblocking(False) + s.bind((host, port)) + s.listen(10) + + loop = asyncio.get_event_loop() + + class WSGIServer(object): + + def __init__(self, sock, app): + self._sock = sock + self._app = app + self._header = [] + + def parse_request(self, req): + """ HTTP Request Format: + + GET /hello.htm HTTP/1.1\r\n + Accept-Language: en-us\r\n + ... + Connection: Keep-Alive\r\n + """ + # bytes to string + req_info = req.decode('utf-8') + first_line = req_info.splitlines()[0] + method, path, ver = first_line.split() + return method, path, ver + + def get_environ(self, req, method, path): + env = {} + + # Required WSGI variables + env['wsgi.version'] = (1, 0) + env['wsgi.url_scheme'] = 'http' + env['wsgi.input'] = req + env['wsgi.errors'] = sys.stderr + env['wsgi.multithread'] = False + env['wsgi.multiprocess'] = False + env['wsgi.run_once'] = False + + # Required CGI variables + env['REQUEST_METHOD'] = method # GET + env['PATH_INFO'] = path # /hello + env['SERVER_NAME'] = host # localhost + env['SERVER_PORT'] = str(port) # 9527 + return env + + def start_response(self, status, resp_header, exc_info=None): + header = [('Server', 'WSGIServer 0.2')] + self.headers_set = [status, resp_header + header] + + async def finish_response(self, conn, data, headers): + status, resp_header = headers + + # make header + resp = 'HTTP/1.1 {0}\r\n'.format(status) + for header in resp_header: + resp += '{0}: {1}\r\n'.format(*header) + resp += '\r\n' + + # make body + resp += '{0}'.format(data) + try: + await loop.sock_sendall(conn, str.encode(resp)) + finally: + conn.close() + + async def run_server(self): + while True: + conn, addr = await loop.sock_accept(self._sock) + loop.create_task(self.handle_request(conn)) + + async def handle_request(self, conn): + # get request data + req = await loop.sock_recv(conn, 1024) + if req: + method, path, ver = self.parse_request(req) + # get environment + env = self.get_environ(req, method, path) + # get application execute result + res = self._app(env, self.start_response) + res = [_.decode('utf-8') for _ in list(res)] + res = ''.join(res) + loop.create_task( + self.finish_response(conn, res, self.headers_set)) + + app = Flask(__name__) + + @app.route('/hello') + def hello(): + return Response("Hello WSGI",mimetype="text/plain") + + server = WSGIServer(s, app.wsgi_app) + try: + loop.run_until_complete(server.run_server()) + except: + pass + finally: + loop.close() + + # Then open browser with url: localhost:9527/hello diff --git a/docs/source/notes/python-basic.rst b/docs/source/notes/python-basic.rst new file mode 100644 index 00000000..e800f038 --- /dev/null +++ b/docs/source/notes/python-basic.rst @@ -0,0 +1,870 @@ +======================= +Python basic cheatsheet +======================= + +Python Naming Styles +-------------------- + +.. code-block:: python + + # see: PEP8 + + # class: camel style only + MyClass + + # func, module, package + var_underscore_separate + + # for public use + var + + # for internal use + _var + + # convention to avoid conflict keyword + var_ + + # for private use in class + __var + + # for protect use in class + _var_ + + # "magic" method or attributes + # ex: __init__, __file__, __main__ + __var__ + + # for "internal" use throwaway variable + # usually used in loop + # ex: [_ for _ in range(10)] + # or variable not used + # for _, a in [(1,2),(3,4)]: print a + _ + +Check object attributes +----------------------- + +.. code-block:: python + + # example of check list attributes + >>> dir(list) + ['__add__', '__class__', ...] + +Define a function __doc__ +------------------------- + +.. code-block:: python + + # Define a function document + >>> def Example(): + ... """ This is an example function """ + ... print "Example function" + ... + >>> Example.__doc__ + ' This is an example function ' + + # Or using help function + >>> help(Example) + +Check instance type +------------------- + +.. code-block:: python + + >>> ex = 10 + >>> isinstance(ex,int) + True + +Check, Get, Set attribute +------------------------- + +.. code-block:: python + + >>> class example(object): + ... def __init__(self): + ... self.name = "ex" + ... def printex(self): + ... print "This is an example" + ... + + # Check object has attributes + # hasattr(obj, 'attr') + >>> ex = example() + >>> hasattr(ex,"name") + True + >>> hasattr(ex,"printex") + True + >>> hasattr(ex,"print") + False + + # Get object attribute + # getattr(obj, 'attr') + >>> getattr(ex,'name') + 'ex' + + # Set object attribute + # setattr(obj, 'attr', value) + >>> setattr(ex,'name','example') + >>> ex.name + 'example' + +Check inheritance +----------------- + +.. code-block:: python + + >>> class example(object): + ... def __init__(self): + ... self.name = "ex" + ... def printex(self): + ... print "This is an example" + ... + >>> issubclass(example,object) + True + +Check all global variables +-------------------------- + +.. code-block:: python + + # globals() return a dictionary + # {'variable name': variable value} + >>> globals() + {'args': (1, 2, 3, 4, 5), ...} + +Check "callable" +---------------- + +.. code-block:: python + + >>> a = 10 + >>> def fun(): + ... print "I am callable" + ... + >>> callable(a) + False + >>> callable(fun) + True + +Get function/class name +----------------------- + +.. code-block:: python + + >>> class excls(object): + ... pass + ... + >>> def exfun(): + ... pass + ... + >>> ex.__class__.__name__ + 'excls' + >>> exfun.__name__ + 'exfun' + +Representations of your class behave +------------------------------------ + +.. code-block:: python + + >>> class example(object): + ... def __str__(self): + ... return "example __str__" + ... def __repr__(self): + ... return "example __repr__" + ... + >>> print str(example()) + example __str__ + >>> example() + example __repr__ + +Get list item "SMART" +--------------------- + +.. code-block:: python + + >>> a = [1,2,3,4,5] + >>> a[0] + 1 + >>>a[-1] + 5 + >>> a[0:] + [1,2,3,4,5] + >>> a[:-1] + [1,2,3,4] + + # a[start:end:step] + >>> a[0:-1:2] + [1,3] + + # using slice object + # slice(start,end,step) + >>> s = slice(0,-1,2) + >>> a[s] + [1,3] + + # Get index and item in loop + >>> a = range(3) + >>> for idx,item in enumerate(a): + ... print (idx,item), + ... + (0, 0) (1, 1) (2, 2) + + # Transfer two list into tuple list + >>> a = [1,2,3,4,5] + >>> b = [2,4,5,6,8] + >>> zip(a,b) + [(1, 2), (2, 4), (3, 5), (4, 6), (5, 8)] + + # with filter + >>> [x for x in range(5) if x>1] + [2, 3, 4] + >>> _ = ['1','2',3,'Hello',4] + >>> f = lambda x: isinstance(x,int) + >>> filter(f,_) + [3, 4] + +Get dictionary item "SMART" +--------------------------- + +.. code-block:: python + + # get dictionary all keys + >>> a={"1":1,"2":2,"3":3} + >>> b={"2":2,"3":3,"4":4} + >>> a.keys() + ['1', '3', '2'] + + # get dictionary key and value as tuple + >>> a.items() + [('1', 1), ('3', 3), ('2', 2)] + + # find same key between two dictionary + >>> [_ for _ in a.keys() if _ in b.keys()] + ['3', '2'] + # better way + >>> c = set(a).intersection(set(b)) + >>> list(c) + ['3', '2'] + # or + >>> [_ for _ in a if _ in b] + ['3', '2'] + + # update dictionary + >>> a.update(b) + >>> a + {'1': 1, '3': 3, '2': 2, '4': 4} + +Set a list/dict "SMART" +----------------------- + +.. code-block:: python + + # get a list with init value + >>> ex = [0]*10 + >>> ex + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + + # extend two list + >>> a = [1,2,3]; b=['a','b'] + >>> a+b + [1, 2, 3, 'a', 'b'] + + # using generator + >>> [x for x in range(10)] + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + >>> fn = lambda x: x**2 + >>> [fn(x) for x in range(5)] + [0, 1, 4, 9, 16] + >>> {'{0}'.format(x): x for x in range(3)} + {'1': 1, '0': 0, '2': 2} + + # using builtin function "map" + >>> map(fn,range(5)) + [0, 1, 4, 9, 16] + +NamedTuple +---------- + +.. code-block:: python + + # namedtuple(typename, field_names) + # replace define class without method + >>> from collections import namedtuple + >>> ex = namedtuple("ex",'a b c') + >>> e = ex(1,2,3) + >>> print e.a, e[1], e[1]+e.b + 1 2 4 + +Delegating Iteration (__iter__) +-------------------------------- + +.. code-block:: python + + # __iter__ return an iterator object + # Be careful: list is an "iterable" object not an "iterator" + >>> class example(object): + ... def __init__(self,list_): + ... self._list = list_ + ... def __iter__(self): + ... return iter(self._list) + ... + >>> ex = example([1,2,3,4,5]) + >>> for _ in ex: print _, + ... + 1 2 3 4 5 + +Using Generator as Iterator +--------------------------- + +.. code-block:: python + + # see: PEP289 + >>> a = (_ for _ in range(10)) + >>> for _ in a: print _, + ... + 0 1 2 3 4 5 6 7 8 9 + + # equivalent to + >>> def _gen(): + ... for _ in range(10): + ... yield _ + ... + >>> a = _gen() + >>> for _ in a: print _, + ... + 0 1 2 3 4 5 6 7 8 9 + +Emulating a list +---------------- + +.. code-block:: python + + >>> class emulist(object): + ... def __init__(self,list_): + ... self._list = list_ + ... def __repr__(self): + ... return "emulist: "+str(self._list) + ... def append(self,item): + ... self._list.append(item) + ... def remove(self,item): + ... self._list.remove(item) + ... def __len__(self): + ... return len(self._list) + ... def __getitem__(self,sliced): + ... return self._list[sliced] + ... def __setitem__(self,sliced,val): + ... self._list[sliced] = val + ... def __delitem__(self,sliced): + ... del self._list[sliced] + ... def __contains__(self,item): + ... return item in self._list + ... def __iter__(self): + ... return iter(self._list) + ... + >>> emul = emulist(range(5)) + >>> emul + emulist: [0, 1, 2, 3, 4] + >>> emul[1:3] + [1, 2] + >>> emul[0:4:2] + [0, 2] + >>> len(emul) + 5 + >>> emul.append(5) + >>> emul + emulist: [0, 1, 2, 3, 4, 5] + >>> emul.remove(2) + >>> emul + emulist: [0, 1, 3, 4, 5] + >>> emul[3] = 6 + >>> emul + emulist: [0, 1, 3, 6, 5] + >>> 0 in emul + True + +Emulating a dictionary +---------------------- + +.. code-block:: python + + >>> class emudict(object): + ... def __init__(self,dict_): + ... self._dict = dict_ + ... def __repr__(self): + ... return "emudict: "+str(self._dict) + ... def __getitem__(self,key): + ... return self._dict[key] + ... def __setitem__(self,key,val): + ... self._dict[key] = val + ... def __delitem__(self,key): + ... del self._dict[key] + ... def __contains__(self,key): + ... return key in self._dict + ... def __iter__(self): + ... return iter(self._dict.keys()) + ... + >>> _ = {"1":1,"2":2,"3":3} + >>> emud = emudict(_) + >>> emud + emudict: {'1': 1, '3': 3, '2': 2} + >>> emud['1'] + 1 + >>> emud['5'] = 5 + >>> emud + emudict: {'1': 1, '3': 3, '2': 2, '5': 5} + >>> del emud['2'] + >>> emud + emudict: {'1': 1, '3': 3, '5': 5} + >>> for _ in emud: print emud[_], + ... + 1 3 5 + >>> '1' in emudict + True + +Decorator +--------- + +.. code-block:: python + + # see: PEP318 + >>> def decor(func): + ... def wrapper(*args,**kwargs): + ... print "wrapper" + ... func() + ... print "-------" + ... return wrapper + ... + >>> @decor + ... def example(): + ... print "Example" + ... + >>> example() + wrapper + Example + ------- + + # equivalent to + >>> def example(): + ... print "Example" + ... + >>> example = decor(example) + >>> example() + wrapper + Example + ------- + +Decorator with arguments +------------------------ + +.. code-block:: python + + >>> def example(val): + ... def decor(func): + ... def wrapper(*args,**kargs): + ... print "Val is {0}".format(val) + ... func() + ... return wrapper + ... return decor + ... + >>> @example(10) + ... def undecor(): + ... print "This is undecor func" + ... + >>> undecor() + Val is 10 + This is undecor func + + # equivalent to + >>> def undecor(): + ... print "This is undecor func" + ... + >>> d = example(10) + >>> undecor = d(undecor) + >>> undecor() + Val is 10 + This is undecor func + +for: exp else: exp +------------------ + +.. code-block:: python + + # see document: More Control Flow Tools + # forloop’s else clause runs when no break occurs + >>> for _ in range(5): + ... print _, + ... else: + ... print "\nno break occur" + ... + 0 1 2 3 4 + no break occur + >>> for _ in range(5): + ... if _ % 2 ==0: + ... print "break occur" + ... break + ... else: + ... print "else not occur" + ... + break occur + + # above statement equivalent to + flag = False + for _ in range(5): + if _ % 2 == 0: + flag = True + print "break occur" + break + if flag == False: + print "else not occur" + +try: exp else: exp +------------------ + +.. code-block:: python + + # No exception occur will go into else. + >>> try: + ... print "No exception" + ... except: + ... pass + ... else: + ... print "No exception occur" + ... + No exception + No exception occur + +Lambda function +--------------- + +.. code-block:: python + + >>> fn = lambda x: x**2 + >>> fn(3) + 9 + >>> (lambda x:x**2)(3) + 9 + >>> (lambda x: [x*_ for _ in range(5)])(2) + [0, 2, 4, 6, 8] + >>> (lambda x: x if x>3 else 3)(5) + 5 + + # multiline lambda example + >>> (lambda x: + ... True + ... if x>0 + ... else + ... False)(3) + True + +Option arguments - (\*args, \*\*kwargs) +--------------------------------------- + +.. code-block:: python + + >>> def example(a,b=None,*args,**kwargs): + ... print a, b + ... print args + ... print kwargs + ... + >>> example(1,"var",2,3,word="hello") + 1 var + (2, 3) + {'word': 'hello'} + >>> _args = (1,2,3,4,5) + >>> _kwargs = {"1":1,"2":2,"3":3} + >>> example(1,"var",*_args,**_kwargs) + 1 var + (1, 2, 3, 4, 5) + {'1': 1, '3': 3, '2': 2} + +Callable object +--------------- + +.. code-block:: python + + >>> class calobj(object): + ... def example(self): + ... print "I am callable!" + ... def __call__(self): + ... self.example() + ... + >>> ex = calobj() + >>> ex() + I am callable! + +Context Manager - "with" statement +---------------------------------- + +.. code-block:: python + + # replace try: ... finally: ... + # see: PEP343 + # common use in open and close + + import socket + + class Socket(object): + def __init__(self,host,port): + self.host = host + self.port = port + def __enter__(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind((self.host,self.port)) + sock.listen(5) + self.sock = sock + return self.sock + + def __exit__(self,*exc_info): + if exc_ty is not None: + import traceback + traceback.print_exception(*exc_info) + self.sock.close() + + if __name__=="__main__": + host = 'localhost' + port 5566 + with Socket(host,port) as s: + while True: + conn, addr = s.accept() + msg = conn.recv(1024) + print msg + conn.send(msg) + conn.close() + +Using @contextmanager +--------------------- + +.. code-block:: python + + from contextlib import contextmanager + + @contextmanager + def opening(filename): + f = open(filename) + try: + yield f + finally: + f.close() + + with opening('example.txt','r') as fd: + fd.read() + +Using "with" statement open file +-------------------------------- + +.. code-block:: python + + >>> with open("/etc/passwd",'r') as f: + ... content = f.read() + +Property - Managed attributes +----------------------------- + +.. code-block:: python + + >>> class example(object): + ... def __init__(self,value): + ... self._val = value + ... @property + ... def val(self): + ... return self._val + ... @val.setter + ... def val(self,value): + ... if not isintance(value,int): + ... raise TypeError("Expect int") + ... self._val = value + ... @val.deleter + ... def val(self): + ... del self._val + ... + >>> ex = example(123) + >>> ex.val = "str" + Traceback (most recent call last): + File "", line 1, in + File "test.py", line 12, in val + raise TypeError("Expect int") + TypeError: Expect int + +Computed attribures - Using property +------------------------------------ + +Concept: Attribure's value is not store in memory. Computing the value only +when we need. + +.. code-block:: python + + >>> class example(object): + ... @property + ... def square3(self): + ... return 2**3 + ... + >>> ex = example() + >>> ex.square3 + 8 + +Descriptor - manage attributes +------------------------------ + +.. code-block:: python + + >>> class Integer(object): + ... def __init__(self,name): + ... self._name = name + ... def __get__(self,inst,cls): + ... if inst is None: + ... return self + ... else: + ... return inst.__dict__[self._name] + ... def __set__(self,inst,value): + ... if not isinstance(value,int): + ... raise TypeError("Expected INT") + ... inst.__dict__[self._name] = value + ... def __delete__(self,inst): + ... del inst.__dict__[self._name] + ... + >>> class example(object): + ... x = Integer('x') + ... def __init__(self,val): + ... self.x = val + ... + >>> ex = example(1) + >>> ex = example("str") + Traceback (most recent call last): + File "", line 1, in + File "", line 4, in __init__ + File "", line 11, in __set__ + TypeError: Expected an int + +@staticmethod, @classmethod +--------------------------- + +.. code-block:: python + + # @classmethod: bound to class + # @staticmethod: like python function but in class + >>> class example(object): + ... @classmethod + ... def clsmethod(cls): + ... print "I am classmethod" + ... @staticmethod + ... def stmethod(): + ... print "I am staticmethod" + ... def instmethod(self): + ... print "I am instancemethod" + ... + >>> ex = example() + >>> ex.clsmethod() + I am classmethod + >>> ex.stmethod() + I am staticmethod + >>> ex.instmethod() + I am instancemethod + >>> example.clsmethod() + I am classmethod + >>> example.stmethod() + I am staticmethod + >>> example.instmethod() + Traceback (most recent call last): + File "", line 1, in + TypeError: unbound method instmethod() ... + +Abstract method - Metaclass +--------------------------- + +.. code-block:: python + + # usually using in define methods but not implement + from abc import ABCMeta, abstractmethod + + >>> class base(object): + ... __metaclass__ = ABCMeta + ... @abstractmethod + ... def absmethod(self): + ... """ Abstract method """ + ... + >>> class example(base): + ... def absmethod(self): + ... print "abstract" + ... + >>> ex = example() + >>> ex.absmethod() + abstract + + # another better way to define a meta class + >>> class base(object): + ... def absmethod(self): + ... raise NotImplementedError + ... + >>> class example(base): + ... def absmethod(self): + ... print "abstract" + ... + >>> ex = example() + >>> ex.absmethod() + abstract + +Common Use "Magic" +------------------ + +.. code-block:: python + + # see python document: data model + # For command class + __main__ + __name__ + __file__ + __module__ + __all__ + __dict__ + __class__ + __doc__ + __init__(self, [...) + __str__(self) + __repr__(self) + __del__(self) + + # For Descriptor + __get__(self, instance, owner) + __set__(self, instance, value) + __delete__(self, instance) + + # For Context Manager + __enter__(self) + __exit__(self, exc_ty, exc_val, tb) + + # Emulating container types + __len__(self) + __getitem__(self, key) + __setitem__(self, key, value) + __delitem__(self, key) + __iter__(self) + __contains__(self, value) + + # Controlling Attribute Access + __getattr__(self, name) + __setattr__(self, name, value) + __delattr__(self, name) + __getattribute__(self, name) + + # Callable object + __call__(self, [args...]) + + # Compare related + __cmp__(self, other) + __eq__(self, other) + __ne__(self, other) + __lt__(self, other) + __gt__(self, other) + __le__(self, other) + __ge__(self, other) + + # arithmetical operation related + __add__(self, other) + __sub__(self, other) + __mul__(self, other) + __div__(self, other) + __mod__(self, other) + __and__(self, other) + __or__(self, other) + __xor__(self, other) diff --git a/docs/source/notes/python-capi.rst b/docs/source/notes/python-capi.rst new file mode 100644 index 00000000..9d2af08e --- /dev/null +++ b/docs/source/notes/python-capi.rst @@ -0,0 +1,384 @@ +======================= +Python C API cheatsheet +======================= + +PyObject header +--------------- + +.. code-block:: c + + // ref: python source code + // Python/Include/object.c + #define _PyObject_HEAD_EXTRA \ + struct _object *_ob_next;\ + struct _object *_ob_prev; + + #define PyObject_HEAD \ + _PyObject_HEAD_EXTRA \ + Py_ssize_t ob_refcnt;\ + struct _typeobject *ob_type; + +Python C API Template +--------------------- + +C API source +~~~~~~~~~~~~ + +.. code-block:: c + + #include + + typedef struct { + PyObject_HEAD + } spamObj; + + static PyTypeObject spamType = { + PyObject_HEAD_INIT(&PyType_Type) + 0, //ob_size + "spam.Spam", //tp_name + sizeof(spamObj), //tp_basicsize + 0, //tp_itemsize + 0, //tp_dealloc + 0, //tp_print + 0, //tp_getattr + 0, //tp_setattr + 0, //tp_compare + 0, //tp_repr + 0, //tp_as_number + 0, //tp_as_sequence + 0, //tp_as_mapping + 0, //tp_hash + 0, //tp_call + 0, //tp_str + 0, //tp_getattro + 0, //tp_setattro + 0, //tp_as_buffer + Py_TPFLAGS_DEFAULT, //tp_flags + "spam objects", //tp_doc + }; + + static PyMethodDef spam_methods[] = { + {NULL} /* Sentinel */ + }; + + /* declarations for DLL import */ + #ifndef PyMODINIT_FUNC + #define PyMODINIT_FUNC void + #endif + + PyMODINIT_FUNC + initspam(void) + { + PyObject *m; + spamType.tp_new = PyType_GenericNew; + if (PyType_Ready(&spamType) < 0) { + goto END; + } + m = Py_InitModule3("spam", spam_methods, "Example of Module"); + Py_INCREF(&spamType); + PyModule_AddObject(m, "spam", (PyObject *)&spamType); + END: + return; + } + +Prepare setup.py +~~~~~~~~~~~~~~~~ + +.. code-block:: python + + from distutils.core import setup + from distutils.core import Extension + + setup(name="spam", + version="1.0", + ext_modules=[Extension("spam", ["spam.c"])]) + +Build C API source +~~~~~~~~~~~~~~~~~~ + +.. code-block:: console + + $ python setup.py build + $ python setup.py install + +Run the C module +~~~~~~~~~~~~~~~~ + +.. code-block:: python + + >>> import spam + >>> spam.__doc__ + 'Example of Module' + >>> spam.spam + + +PyObject with Member and Methods +-------------------------------- + +C API source +~~~~~~~~~~~~ + + +.. code-block:: c + + #include + #include + + typedef struct { + PyObject_HEAD + PyObject *hello; + PyObject *world; + int spam_id; + } spamObj; + + static void + spamdealloc(spamObj *self) + { + Py_XDECREF(self->hello); + Py_XDECREF(self->world); + self->ob_type + ->tp_free((PyObject*)self); + } + + /* __new__ */ + static PyObject * + spamNew(PyTypeObject *type, PyObject *args, PyObject *kwds) + { + spamObj *self = NULL; + + self = (spamObj *) + type->tp_alloc(type, 0); + if (self == NULL) { + goto END; + } + /* alloc str to hello */ + self->hello = + PyString_FromString(""); + if (self->hello == NULL) + { + Py_XDECREF(self); + self = NULL; + goto END; + } + /* alloc str to world */ + self->world = + PyString_FromString(""); + if (self->world == NULL) + { + Py_XDECREF(self); + self = NULL; + goto END; + } + self->spam_id = 0; + END: + return (PyObject *)self; + } + + /* __init__ */ + static int + spamInit(spamObj *self, PyObject *args, PyObject *kwds) + { + int ret = -1; + PyObject *hello=NULL, + *world=NULL, + *tmp=NULL; + + static char *kwlist[] = { + "hello", + "world", + "spam_id", NULL}; + + /* parse input arguments */ + if (! PyArg_ParseTupleAndKeywords( + args, kwds, + "|OOi", + kwlist, + &hello, &world, + &self->spam_id)) { + goto END; + } + /* set attr hello */ + if (hello) { + tmp = self->hello; + Py_INCREF(hello); + self->hello = hello; + Py_XDECREF(tmp); + } + /* set attr world */ + if (world) { + tmp = self->world; + Py_INCREF(world); + self->world = world; + Py_XDECREF(tmp); + } + ret = 0; + END: + return ret; + } + + static long + fib(long n) { + if (n<=2) { + return 1; + } + return fib(n-1)+fib(n-2); + } + + static PyObject * + spamFib(spamObj *self, PyObject *args) + { + PyObject *ret = NULL; + long arg = 0; + + if (!PyArg_ParseTuple(args, "i", &arg)) { + goto END; + } + ret = PyInt_FromLong(fib(arg)); + END: + return ret; + } + + //ref: python doc + static PyMemberDef spam_members[] = { + /* spameObj.hello*/ + {"hello", //name + T_OBJECT_EX, //type + offsetof(spamObj, hello), //offset + 0, //flags + "spam hello"}, //doc + /* spamObj.world*/ + {"world", + T_OBJECT_EX, + offsetof(spamObj, world), + 0, + "spam world"}, + /* spamObj.spam_id*/ + {"spam_id", + T_INT, + offsetof(spamObj, spam_id), + 0, + "spam id"}, + /* Sentiel */ + {NULL} + }; + + static PyMethodDef spam_methods[] = { + /* fib */ + {"spam_fib", + (PyCFunction)spamFib, + METH_VARARGS, + "Calculate fib number"}, + /* Sentiel */ + {NULL} + }; + + static PyMethodDef module_methods[] = { + {NULL} /* Sentinel */ + }; + + static PyTypeObject spamKlass = { + PyObject_HEAD_INIT(NULL) + 0, //ob_size + "spam.spamKlass", //tp_name + sizeof(spamObj), //tp_basicsize + 0, //tp_itemsize + (destructor) spamdealloc, //tp_dealloc + 0, //tp_print + 0, //tp_getattr + 0, //tp_setattr + 0, //tp_compare + 0, //tp_repr + 0, //tp_as_number + 0, //tp_as_sequence + 0, //tp_as_mapping + 0, //tp_hash + 0, //tp_call + 0, //tp_str + 0, //tp_getattro + 0, //tp_setattro + 0, //tp_as_buffer + Py_TPFLAGS_DEFAULT | + Py_TPFLAGS_BASETYPE, //tp_flags + "spamKlass objects", //tp_doc + 0, //tp_traverse + 0, //tp_clear + 0, //tp_richcompare + 0, //tp_weaklistoffset + 0, //tp_iter + 0, //tp_iternext + spam_methods, //tp_methods + spam_members, //tp_members + 0, //tp_getset + 0, //tp_base + 0, //tp_dict + 0, //tp_descr_get + 0, //tp_descr_set + 0, //tp_dictoffset + (initproc)spamInit, //tp_init + 0, //tp_alloc + spamNew, //tp_new + }; + + /* declarations for DLL import */ + #ifndef PyMODINIT_FUNC + #define PyMODINIT_FUNC void + #endif + + PyMODINIT_FUNC + initspam(void) + { + PyObject* m; + + if (PyType_Ready(&spamKlass) < 0) { + goto END; + } + + m = Py_InitModule3( + "spam", // Mod name + module_methods, // Mod methods + "Spam Module"); // Mod doc + + if (m == NULL) { + goto END; + } + Py_INCREF(&spamKlass); + PyModule_AddObject( + m, // Module + "SpamKlass", // Class Name + (PyObject *) &spamKlass); // Class + END: + return; + } + +Compare performance with pure Python +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + >>> import spam + >>> o = spam.SpamKlass() + >>> def profile(func): + ... def wrapper(*args, **kwargs): + ... s = time.time() + ... ret = func(*args, **kwargs) + ... e = time.time() + ... print e-s + ... return wrapper + ... + >>> def fib(n): + ... if n <= 2: + ... return n + ... return fib(n-1)+fib(n-2) + ... + >>> @profile + ... def cfib(n): + ... o.spam_fib(n) + ... + >>> @profile + ... def pyfib(n): + ... fib(n) + ... + >>> cfib(30) + 0.0106310844421 + >>> pyfib(30) + 0.399799108505 diff --git a/docs/source/notes/python-concurrency.rst b/docs/source/notes/python-concurrency.rst new file mode 100644 index 00000000..912f29ca --- /dev/null +++ b/docs/source/notes/python-concurrency.rst @@ -0,0 +1,834 @@ +============================= +Python Concurrency Cheatsheet +============================= + +Create a thread via "threading" +------------------------------- + +.. code-block:: python + + >>> from threading import Thread + >>> class Worker(Thread): + ... def __init__(self, id): + ... super(Worker, self).__init__() + ... self._id = id + ... def run(self): + ... print "I am worker %d" % self._id + ... + >>> t1 = Worker(1) + >>> t2 = Worker(2) + >>> t1.start(); t2.start() + I am worker 1 + I am worker 2 + + # using function could be more flexible + >>> def Worker(worker_id): + ... print "I am worker %d" % worker_id + ... + >>> from threading import Thread + >>> t1 = Thread(target=Worker, args=(1,)) + >>> t2 = Thread(target=Worker, args=(2,)) + >>> t1.start() + I am worker 1 + I am worker 2 + +Performance Problem - GIL +------------------------- + +.. code-block:: python + + # GIL - Global Interpreter Lock + # see: Ungerstanging the Python GIL + >>> from threading import Thread + >>> def profile(func): + ... def wrapper(*args, **kwargs): + ... import time + ... start = time.time() + ... func(*args, **kwargs) + ... end = time.time() + ... print end - start + ... return wrapper + ... + >>> @profile + ... def nothread(): + ... fib(35) + ... fib(35) + ... + >>> @profile + ... def hasthread(): + ... t1=Thread(target=fib, args=(35,)) + ... t2=Thread(target=fib, args=(35,)) + ... t1.start(); t2.start() + ... t1.join(); t2.join() + ... + >>> nothread() + 9.51164007187 + >>> hasthread() + 11.3131771088 + # !Thread get bad Performance + # since cost on context switch + +Consumer and Producer +--------------------- + +.. code-block:: python + + # This architecture make concurrency easy + >>> from threading import Thread + >>> from Queue import Queue + >>> from random import random + >>> import time + >>> q = Queue() + >>> def fib(n): + ... if n<=2: + ... return 1 + ... return fib(n-1)+fib(n-2) + ... + >>> def producer(): + ... while True: + ... wt = random()*5 + ... time.sleep(wt) + ... q.put((fib,35)) + ... + >>> def consumer(): + ... while True: + ... task,arg = q.get() + ... print task(arg) + ... q.task_done() + ... + >>> t1 = Thread(target=producer) + >>> t2 = Thread(target=consumer) + >>> t1.start();t2.start() + +Thread Pool Templeate +--------------------- + +.. code-block:: python + + # producer and consumer architecture + from Queue import Queue + from threading import Thread + + class Worker(Thread): + def __init__(self,queue): + super(Worker, self).__init__() + self._q = queue + self.daemon = True + self.start() + def run(self): + while True: + f,args,kwargs = self._q.get() + try: + print f(*args, **kwargs) + except Exception as e: + print e + self._q.task_done() + + class ThreadPool(object): + def __init__(self, num_t=5): + self._q = Queue(num_t) + # Create Worker Thread + for _ in range(num_t): + Worker(self._q) + def add_task(self,f,*args,**kwargs): + self._q.put((f, args, kwargs)) + def wait_complete(self): + self._q.join() + + def fib(n): + if n <= 2: + return 1 + return fib(n-1)+fib(n-2) + + if __name__ == '__main__': + pool = ThreadPool() + for _ in range(3): + pool.add_task(fib,35) + pool.wait_complete() + + +Using multiprocessing ThreadPool +-------------------------------- + +.. code-block:: python + + # ThreadPool is not in python doc + >>> from multiprocessing.pool import ThreadPool + >>> pool = ThreadPool(5) + >>> pool.map(lambda x: x**2, range(5)) + [0, 1, 4, 9, 16] + +Compare with "map" performance + +.. code-block:: python + + # pool will get bad result since GIL + import time + from multiprocessing.pool import \ + ThreadPool + + pool = ThreadPool(10) + def profile(func): + def wrapper(*args, **kwargs): + print func.__name__ + s = time.time() + func(*args, **kwargs) + e = time.time() + print "cost: {0}".format(e-s) + return wrapper + + @profile + def pool_map(): + res = pool.map(lambda x:x**2, + range(999999)) + + @profile + def ordinary_map(): + res = map(lambda x:x**2, + range(999999)) + + pool_map() + ordinary_map() + +output: + +.. code-block:: console + + $ python test_theadpool.py + pool_map + cost: 0.562669038773 + ordinary_map + cost: 0.38525390625 + +Mutex lock +---------- + +Simplest synchronization primitive lock + +.. code-block:: python + + >>> from threading import Thread + >>> from threading import Lock + >>> lock = Lock() + >>> def getlock(id): + ... lock.acquire() + ... print "task{0} get".format(id) + ... lock.release() + ... + >>> t1=Thread(target=getlock,args=(1,)) + >>> t2=Thread(target=getlock,args=(2,)) + >>> t1.start();t2.start() + task1 get + task2 get + + # using lock manager + >>> def getlock(id): + ... with lock: + ... print "task%d get" % id + ... + >>> t1=Thread(target=getlock,args=(1,)) + >>> t2=Thread(target=getlock,args=(2,)) + >>> t1.start();t2.start() + task1 get + task2 get + + +Deadlock +-------- + +Happen when more than one mutex lock. + +.. code-block:: python + + >>> import threading + >>> import time + >>> lock1 = threading.Lock() + >>> lock2 = threading.Lock() + >>> def task1(): + ... with lock1: + ... print "get lock1" + ... time.sleep(3) + ... with lock2: + ... print "No deadlock" + ... + >>> def task2(): + ... with lock2: + ... print "get lock2" + ... with lock1: + ... print "No deadlock" + ... + >>> t1=threading.Thread(target=task1) + >>> t2=threading.Thread(target=task2) + >>> t1.start();t2.start() + get lock1 + get lock2 + + >>> t1.isAlive() + True + >>> t2.isAlive() + True + + +Implement "Monitor" +------------------- + +Using RLock + +.. code-block:: python + + # ref: An introduction to Python Concurrency - David Beazley + from threading import Thread + from threading import RLock + import time + + class monitor(object): + lock = RLock() + def foo(self,tid): + with monitor.lock: + print "%d in foo" % tid + time.sleep(5) + self.ker(tid) + + def ker(self,tid): + with monitor.lock: + print "%d in ker" % tid + m = monitor() + def task1(id): + m.foo(id) + + def task2(id): + m.ker(id) + + t1 = Thread(target=task1,args=(1,)) + t2 = Thread(target=task2,args=(2,)) + t1.start() + t2.start() + t1.join() + t2.join() + +output: + +.. code-block:: console + + $ python monitor.py + 1 in foo + 1 in ker + 2 in ker + +Control primitive resources +--------------------------- + +Using Semaphore + +.. code-block:: python + + from threading import Thread + from threading import Semaphore + from random import random + import time + + # limit resource to 3 + sema = Semaphore(3) + def foo(tid): + with sema: + print "%d acquire sema" % tid + wt = random()*5 + time.sleep(wt) + print "%d release sema" % tid + + threads = [] + for _t in range(5): + t = Thread(target=foo,args=(_t,)) + threads.append(t) + t.start() + for _t in threads: + _t.join() + +output: + +.. code-block:: console + + python semaphore.py + 0 acquire sema + 1 acquire sema + 2 acquire sema + 0 release sema + 3 acquire sema + 2 release sema + 4 acquire sema + 1 release sema + 4 release sema + 3 release sema + + +Ensure tasks has done +--------------------- + +Using 'event' + +.. code-block:: python + + from threading import Thread + from threading import Event + import time + + e = Event() + + def worker(id): + print "%d wait event" % id + e.wait() + print "%d get event set" % id + + t1=Thread(target=worker,args=(1,)) + t2=Thread(target=worker,args=(2,)) + t3=Thread(target=worker,args=(3,)) + t1.start() + t2.start() + t3.start() + + # wait sleep task(event) happen + time.sleep(3) + e.set() + +output: + +.. code-block:: console + + python event.py + 1 wait event + 2 wait event + 3 wait event + 2 get event set + 3 get event set + 1 get event set + +Thread-safe priority queue +-------------------------- + +Using 'condition' + +.. code-block:: python + + import threading + import heapq + import time + import random + + class PriorityQueue(object): + def __init__(self): + self._q = [] + self._count = 0 + self._cv = threading.Condition() + + def __str__(self): + return str(self._q) + + def __repr__(self): + return self._q + + def put(self, item, priority): + with self._cv: + heapq.heappush(self._q, (-priority,self._count,item)) + self._count += 1 + self._cv.notify() + + def pop(self): + with self._cv: + while len(self._q) == 0: + print("wait...") + self._cv.wait() + ret = heapq.heappop(self._q)[-1] + return ret + + priq = PriorityQueue() + def producer(): + while True: + print(priq.pop()) + + def consumer(): + while True: + time.sleep(3) + print("consumer put value") + priority = random.random() + priq.put(priority,priority*10) + + for _ in range(3): + priority = random.random() + priq.put(priority,priority*10) + + t1=threading.Thread(target=producer) + t2=threading.Thread(target=consumer) + t1.start();t2.start() + t1.join();t2.join() + +output: + +.. code-block:: console + + python3 thread_safe.py + 0.6657491871045683 + 0.5278797439991247 + 0.20990624606296315 + wait... + consumer put value + 0.09123101305407577 + wait... + +Multiprocessing +--------------- + +Solving GIL problem via processes + +.. code-block:: python + + >>> from multiprocessing import Pool + >>> def fib(n): + ... if n >= 2: + ... return 1 + ... return fib(n-1)+fib(n-2) + ... + >>> def profile(func): + ... def wrapper(*args, **kwargs): + ... import time + ... start = time.time() + ... func(*args, **kwargs) + ... end = time.time() + ... print end - start + ... return wrapper + ... + >>> @profile + ... def nomultiprocess(): + ... map(fib,[35]*5) + ... + >>> @profile + ... def hasmultiprocess(): + ... pool = Pool(5) + ... pool.map(fib,[35]*5) + ... + >>> nomultiprocess() + 23.8454811573 + >>> hasmultiprocess() + 13.2433719635 + +Custom multiprocessing map +-------------------------- + +.. code-block:: python + + from multiprocessing import Process, Pipe + from itertools import izip + + def spawn(f): + def fun(pipe,x): + pipe.send(f(x)) + pipe.close() + return fun + + def parmap(f,X): + pipe=[Pipe() for x in X] + proc=[Process(target=spawn(f), + args=(c,x)) + for x,(p,c) in izip(X,pipe)] + [p.start() for p in proc] + [p.join() for p in proc] + return [p.recv() for (p,c) in pipe] + + print parmap(lambda x:x**x,range(1,5)) + +Simple round-robin scheduler +---------------------------- + +.. code-block:: python + + >>> def fib(n): + ... if n <= 2: + ... return 1 + ... return fib(n-1)+fib(n-2) + ... + >>> def gen_fib(n): + ... for _ in range(1,n+1): + ... yield fib(_) + ... + >>> t=[gen_fib(5),gen_fib(3)] + >>> from collections import deque + >>> tasks = deque() + >>> tasks.extend(t) + >>> def run(tasks): + ... while tasks: + ... try: + ... task = tasks.popleft() + ... print task.next() + ... tasks.append(task) + ... except StopIteration: + ... print "done" + ... + >>> run(tasks) + 1 + 1 + 1 + 1 + 2 + 2 + 3 + done + 5 + done + +Scheduler with blocking function + +.. code-block:: python + + # ref: PyCon 2015 - David Beazley + import socket + from select import select + from collections import deque + + tasks = deque() + r_wait = {} + s_wait = {} + + def fib(n): + if n <= 2: + return 1 + return fib(n-1)+fib(n-2) + + def run(): + while any([tasks,r_wait,s_wait]): + while not tasks: + # polling + rr, sr, _ = select(r_wait, s_wait, {}) + for _ in rr: + tasks.append(r_wait.pop(_)) + for _ in sr: + tasks.append(s_wait.pop(_)) + try: + task = tasks.popleft() + why,what = task.next() + if why == 'recv': + r_wait[what] = task + elif why == 'send': + s_wait[what] = task + else: + raise RuntimeError + except StopIteration: + pass + + def fib_server(): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,1) + sock.bind(('localhost',5566)) + sock.listen(5) + while True: + yield 'recv', sock + c, a = sock.accept() + tasks.append(fib_handler(c)) + + def fib_handler(client): + while True: + yield 'recv', client + req = client.recv(1024) + if not req: + break + resp = fib(int(req)) + yield 'send', client + client.send(str(resp)+'\n') + client.close() + + tasks.append(fib_server()) + run() + +output: (bash 1) + +.. code-block:: console + + $ nc loalhost 5566 + 20 + 6765 + +output: (bash 2) + +.. code-block:: console + + $ nc localhost 5566 + 10 + 55 + +PoolExecutor +------------ + +.. code-block:: python + + # python2.x is module futures on PyPI + # new in Python3.2 + >>> from concurrent.futures import \ + ... ThreadPoolExecutor + >>> def fib(n): + ... if n<=2: + ... return 1 + ... return fib(n-1) + fib(n-2) + ... + >>> with ThreadPoolExecutor(3) as e: + ... res= e.map(fib,[1,2,3,4,5]) + ... for _ in res: + ... print(_, end=' ') + ... + 1 1 2 3 5 >>> + # result is generator?! + >>> with ThreadPoolExecutor(3) as e: + ... res = e.map(fib, [1,2,3]) + ... inspect.isgenerator(res) + ... + True + + # demo GIL + from concurrent import futures + import time + + def fib(n): + if n <= 2: + return 1 + return fib(n-1) + fib(n-2) + + def thread(): + s = time.time() + with futures.ThreadPoolExecutor(2) as e: + res = e.map(fib, [35]*2) + for _ in res: + print(_) + e = time.time() + print("thread cost: {}".format(e-s)) + + def process(): + s = time.time() + with futures.ProcessPoolExecutor(2) as e: + res = e.map(fib, [35]*2) + for _ in res: + print(_) + e = time.time() + print("pocess cost: {}".format(e-s)) + + + # bash> python3 -i test.py + >>> thread() + 9227465 + 9227465 + thread cost: 12.550225019454956 + >>> process() + 9227465 + 9227465 + pocess cost: 5.538189888000488 + +What "with ThreadPoolExecutor" doing? +------------------------------------- + +.. code-block:: python + + from concurrent import futures + + def fib(n): + if n <= 2: + return 1 + return fib(n-1) + fib(n-2) + + with futures.ThreadPoolExecutor(3) as e: + fut = e.submit(fib, 30) + res = fut.result() + print(res) + + # equal to + e = futures.ThreadPoolExecutor(3) + fut = e.submit(fib, 30) + fut.result() + e.shutdown(wait=True) + print(res) + +output: + +.. code-block:: console + + $ python3 thread_pool_exec.py + 832040 + 832040 + +Future Object +------------- + +.. code-block:: python + + # future: deferred computation + # add_done_callback + from concurrent import futures + + def fib(n): + if n <= 2: + return 1 + return fib(n-1) + fib(n-2) + + def handler(future): + res = future.result() + print("res: {}".format(res)) + + def thread_v1(): + with futures.ThreadPoolExecutor(3) as e: + for _ in range(3): + f = e.submit(fib, 30+_) + f.add_done_callback(handler) + print("end") + + def thread_v2(): + to_do = [] + with futures.ThreadPoolExecutor(3) as e: + for _ in range(3): + fut = e.submit(fib, 30+_) + to_do.append(fut) + for _f in futures.as_completed(to_do): + res = _f.result() + print("res: {}".format(res)) + print("end") + +output: + +.. code-block:: console + + $ python3 -i fut.py + >>> thread_v1() + res: 832040 + res: 1346269 + res: 2178309 + end + >>> thread_v2() + res: 832040 + res: 1346269 + res: 2178309 + end + +Future error handling +--------------------- + +.. code-block:: python + + from concurrent import futures + + def spam(): + raise RuntimeError + + def handler(future): + print("callback handler") + try: + res = future.result() + except RuntimeError: + print("get RuntimeError") + + def thread_spam(): + with futures.ThreadPoolExecutor(2) as e: + f = e.submit(spam) + f.add_done_callback(handler) + +output: + +.. code-block:: console + + $ python -i fut_err.py + >>> thread_spam() + callback handler + get RuntimeError diff --git a/docs/source/notes/python-cstyle.rst b/docs/source/notes/python-cstyle.rst new file mode 100644 index 00000000..db74f7a3 --- /dev/null +++ b/docs/source/notes/python-cstyle.rst @@ -0,0 +1,642 @@ +Python Design Pattern in C +========================== + +Decorator in C +-------------- + +Python + +.. code-block:: python + + >>> def decorator(func): + ... def wrapper(*args, **kwargs): + ... print("I am decorator") + ... ret = func(*args, **kwargs) + ... return ret + ... return wrapper + ... + >>> @decorator + ... def hello(str): + ... print("Hello {0}".format(str)) + ... + >>> @decorator + ... def add(a,b): + ... print("add %d+%d=%d" % (a,b,a+b)) + ... return a+b + ... + >>> hello("KerKer") + I am decorator + Hello KerKer + >>> add(1,2) + I am decorator + add 1+2=3 + 3 + +C + +.. code-block:: c + + #include + + #define DECORATOR(t, f, declar, input) \ + t decor_##f(declar) { \ + printf("I am decorator\n"); \ + return f(input);\ + } + #define FUNC_DEC(func, ...) \ + decor_##func(__VA_ARGS__) + + // Original function + void hello(char *str) { + printf("Hello %s\n", str); + } + + int add(int a, int b) { + printf("add %d + %d = %d\n",a,b,a+b); + return a+b; + } + // Patch function + #define DECLAR char *str + #define INPUT str + DECORATOR(void, hello, DECLAR, INPUT) + #undef DECLAR + #undef INPUT + + #define DECLAR int a, int b + #define INPUT a,b + DECORATOR(int, add, DECLAR, INPUT) + #undef DECLAR + #undef INPUT + + int main(int argc, char *argv[]) { + FUNC_DEC(hello, "KerKer"); + FUNC_DEC(add,1,2); + + return 0; + } + +output: + +.. code-block:: console + + $ gcc example.c + $ ./a.out + I am decorator + Hello KerKer + I am decorator + add 1 + 2 = 3 + +A Set of Functions +------------------ + +Python + +.. code-block:: python + + >>> def func_1(): + ... print "Hello" + ... + >>> def func_2(): + ... print "World" + ... + >>> def func_3(): + ... print "!!!" + ... + >>> s = [func_1,func_2,func_3] + >>> for _ in s: _() + ... + Hello + World + !!! + +C + +.. code-block:: c + + #include + + typedef void (*func)(void); + + enum func_id{ + FUNC_1,FUNC_2,FUNC_3 + }; + + void func_1() { + printf("Hello "); + } + void func_2() { + printf("World "); + } + void func_3() { + printf("!!!\n"); + } + + func gFuncTable[] = { + func_1,func_2,func_3 + }; + + int main(int argc, char *argv[]) { + gFuncTable[FUNC_1](); + gFuncTable[FUNC_2](); + gFuncTable[FUNC_3](); + + return 0; + } + +Closure in C +------------ + +Python + +.. code-block:: python + + # implement via __call__ + >>> class closure(object): + ... def __init__(self): + ... self.val = 5566 + ... def __call__(self,var): + ... self.val += var + ... + >>> c = closure() + >>> c(9527) + >>> print c.val + 15093 + # using "global" keyword + >>> x = 0 + >>> def closure(val): + ... def wrapper(): + ... global x + ... x += val + ... print x + ... wrapper() + ... + >>> closure(5566) + 5566 + >>> closure(9527) + 15093 + # using "nonlocal" (only in python3) + >>> def closure(val): + ... x = 0 + ... def wrapper(): + ... nonlocal x + ... x += val + ... print(x) + ... wrapper() + ... + >>> closure(5566) + 5566 + >>> closure(9527) + 9527 + +C + +.. code-block:: c + + #include + #include + + typedef struct Closure { + int val; + void (*add) (struct Closure **, int); + }closure; + + void add_func(closure **c, int var) { + (*c)->val += var; + } + + int main(int argc, char *argv[]) { + closure *c = NULL; + c = malloc(sizeof(closure)); + c->val = 5566; + c->add = add_func; + c->add(&c,9527); + printf("result: %d\n",c->val); + + return 0; + } + + +Generator +--------- + +Python + +.. code-block:: python + + >>> def gen(): + ... var = 0 + ... while True: + ... var += 1 + ... yield var + ... + >>> g = gen() + >>> for _ in range(3): + ... print next(g), + ... + 1 2 3 + +C + +.. code-block:: c + + #include + #include + + struct gen { + int (*next) (struct gen *); + int var; + }; + + int next_func(struct gen *g) { + printf("var = %d\n",g->var); + g->var +=1; + return g->var; + } + + struct gen * new_gen() { + struct gen *g = NULL; + g = (struct gen*) + malloc(sizeof(struct gen)); + g->var = 0; + g->next = next_func; + return g; + } + + int main(int argc, char *argv[]) { + struct gen *g = new_gen(); + int i = 0; + for (i=0;i<3;i++) { + printf("gen var = %d\n",g->next(g)); + } + return 0; + } + +Context Manager in C +-------------------- + +Python + +.. code-block:: python + + >>> class CtxManager(object): + ... def __enter__(self): + ... self._attr = "KerKer" + ... return self._attr + ... def __exit__(self,*e_info): + ... del self._attr + ... + >>> with CtxManager() as c: + ... print c + ... + KerKer + +C + +.. code-block:: c + + #include + #include + + #define ENTER(type,ptr,len) \ + printf("enter context manager\n");\ + ptr = malloc(sizeof(type)*len);\ + if (NULL == ptr) { \ + printf("malloc get error\n");\ + goto exit;\ + }\ + + #define EXIT(ptr) \ + exit:\ + printf("exit context manager\n");\ + if (NULL != ptr) {\ + free(ptr);\ + ptr = NULL; \ + }\ + + #define CONTEXT_MANAGER(t, p, l,...){\ + ENTER(t,p,l)\ + __VA_ARGS__ \ + EXIT(p)\ + } + + int main(int argc, char *argv[]) { + char *ptr; + CONTEXT_MANAGER(char, ptr, 128, + sprintf(ptr, "Hello World"); + printf("%s\n",ptr); + ); + printf("ptr = %s\n",ptr); + return 0; + } + +Tuple in C +---------- + +Python + +.. code-block:: python + + >>> a = ("Hello","World",123) + >>> for _ in a: print _, + ... + Hello World 123 + +C + +.. code-block:: c + + #include + + int main(int argc, char *argv[]) { + int a = 123; + void * const x[4] = {"Hello", + "World",&a}; + printf("%s %s, %d\n",x[0],x[1],*(int *)x[2]); + return 0; + } + +Error Handling +-------------- + +Python + +.. code-block:: python + + >>> import os + >>> def spam(a,b): + ... try: + ... os.listdir('.') + ... except OSError: + ... print "listdir get error" + ... return + ... try: + ... a/b + ... except ZeroDivisionError: + ... print "zero division" + ... return + ... + >>> spam(1,0) + zero division + # single exit -> using decorator + >>> import time + >>> def profile(func): + ... def wrapper(*args, **kwargs): + ... s = time.time() + ... ret = func(*args, **kwargs) + ... e = time.time() + ... print e - s + ... return ret + ... return wrapper + ... + >>> @profile + ... def spam(a,b): + ... try: + ... os.listdir('.') + ... except OSError: + ... return + ... try: + ... a/b + ... except ZeroDivisionError: + ... return + ... + >>> spam(1,0) + 0.000284910202026 + +C + +.. code-block:: c + + #include + #include + #include + + int main(int argc, char *argv[]) { + int ret = -1; + char *ptr; + ptr = malloc(sizeof(char)*128); + if (NULL == ptr) { + perror("malloc get error"); + goto exit; + } + strcpy(ptr,"KerKer"); + printf("%s\n", ptr); + ret = 0; + exit: + if (ptr) { + free(ptr); + ptr = NULL; + } + return ret; + } + +Keyword Arguments in C +---------------------- + +Python + +.. code-block:: python + + >>> def f(str_, float_,int_=0): + ... print(str_, float_, int_) + ... + >>> f("KerKer",2.0,2) + KerKer 2.0 2 + >>> f("HaHa",3.) + HaHa 3.0 0 + +C + +.. code-block:: c + + #include + + #define FUNC(...) \ + base_func((struct input ){.var=0, ##__VA_ARGS__}); + + struct input { + char *str; + int var; + double dvar; + }; + + void base_func(struct input in){ + printf("str = %s, var = %d" + ", dvar = %lf\n", + in.str, in.var,in.dvar); + } + + int main(int argc, char *argv[]) { + FUNC(.str="KerKer", 2.0); + FUNC(2, .str="KerKer"); + FUNC(.var=10, .dvar=2.0, .str="HAHA"); + return 0; + } + +Function "MAP" +-------------- + +Python + +.. code-block:: python + + >>> x = [1,2,3,4,5] + >>> y = map(lambda x:2*x, x) + >>> print y + [2, 4, 6, 8, 10] + #or + >>> x = [1,2,3,4,5] + >>> y = [2*_ for _ in x] + >>> print y + [2, 4, 6, 8, 10] + +C + +.. code-block:: c + + #include + + #define MAP(func, src, dst, len) \ + do {\ + unsigned i=0;\ + for(i=0; i>> x = ["Hello","World","!!!"] + >>> for _ in x:print _, + ... + Hello World !!! + +C + +.. code-block:: c + + #include + + #define foreach(it, x,...) \ + for(char **it=x;*it;it++) {__VA_ARGS__} + + int main(int argc, char *argv[]) { + char *x[] = {"Hello","World", + "!!!",NULL}; + foreach(it,x, + printf("%s ",*it); + ) + printf("\n"); + return 0; + } + +Simple OOP in C +--------------- + +Python + +.. code-block:: python + + # common declaration + >>> class obj(object): + ... def __init__(self): + ... self.a = 0 + ... self.b = 0 + ... def add(self): + ... return self.a + self.b + ... def sub(self): + ... return self.a - self.b + ... + >>> o = obj() + >>> o.a = 9527 + >>> o.b = 5566 + >>> o.add() + 15093 + >>> o.sub() + 3961 + # patch class (more like ooc) + >>> class obj(object): + ... def __init__(self): + ... self.a = 0 + ... self.b = 0 + ... + >>> def add(self): + ... return self.a+self.b + ... + >>> def sub(self): + ... return self.a - self.b + ... + >>> obj.add = add + >>> obj.sub = sub + >>> o = obj() + >>> o.a = 9527 + >>> o.b = 5566 + >>> o.add() + 15093 + >>> o.sub() + 3961 + +C + +.. code-block:: c + + #include + #include + + typedef struct object Obj; + typedef int (*func)(Obj *); + + struct object { + int a; + int b; + // virtual + func add; + func sub; + }; + int add_func(Obj *self) { + return self-&t;a + self->b; + } + int sub_func(Obj *self) { + return self->a - self->b; + } + int init_obj(Obj **self) { + *self = malloc(sizeof(Obj)); + if (NULL == *self) { + return -1; + } + (*self)->a = 0; + (*self)->b = 0; + (*self)->add = add_func; + (*self)->sub = sub_func; + return 0; + } + + int main(int argc, char *argv[]) { + Obj *o = NULL; + init_obj(&o); + o->a = 9527; + o->b = 5566; + printf("add = %d\n",o->add(o)); + printf("sub = %d\n",o->sub(o)); + return 0; + } diff --git a/docs/source/notes/python-generator.rst b/docs/source/notes/python-generator.rst new file mode 100644 index 00000000..c1204caf --- /dev/null +++ b/docs/source/notes/python-generator.rst @@ -0,0 +1,728 @@ +=========================== +Python generator cheatsheet +=========================== + +Glossary of Generator +--------------------- + +.. code-block:: python + + # generator function + >>> def gen_func(): + ... yield 5566 + ... + >>> gen_func + + + # generator + >>> g = gen_func() + >>> g + + >>> next(g) + 5566 + >>> next(g) + Traceback (most recent call last): + File "", line 1, in + StopIteration + + # generator expression + >>> g = (_ for _ in range(2)) + >>> g + at 0x10a9c191> + >>> next(g) + 0 + >>> next(g) + 1 + >>> next(g) + Traceback (most recent call last): + File "", line 1, in + StopIteration + +Produce value via generator +--------------------------- + +.. code-block:: python + + >>> def prime(n): + ... p = 2 + ... while n > 0: + ... for _ in range(2,p): + ... if p % _ == 0: + ... break + ... else: + ... yield p + ... n-=1 + ... p+=1 + ... + >>> p = prime(3) + >>> next(p) + 2 + >>> next(p) + 3 + >>> + >>> next(p) + 5 + >>> next(p) + Traceback (most recent call last): + File "", line 1, in + StopIteration + >>> for _ in prime(5): + ... print(_, end=" ") + ... + 2 3 5 7 11 >>> + +Implement Iterable object via generator +--------------------------------------- + +.. code-block:: python + + >>> class count(object): + ... def __init__(self,n): + ... self._n = n + ... def __iter__(self): + ... n = self._n + ... while n>0: + ... yield n + ... n-=1 + ... def __reversed__(self): + ... n = 0 + ... while n>> for _ in count(5): + ... print(_, end=" ") + ... + 5 4 3 2 1 >>> + >>> for _ in reversed(count(5)): + ... print(_, end=" ") + ... + 0 1 2 3 4 >>> + +Send message to generator +------------------------- + +.. code-block:: python + + >>> def spam(): + ... msg = yield + ... print("Message:",msg) + ... + >>> try: + ... g = spam() + ... # start generator + ... next(g) + ... # send message to generator + ... g.send("Hello World!") + ... except StopIteration: + ... pass + ... + Message: Hello World! + +"yield from" expression +----------------------- + +.. code-block:: python + + # delegating gen do nothing(pipe) + >>> def subgen(): + ... try: + ... yield 9527 + ... except ValueError: + ... print("get value error") + ... + >>> def delegating_gen(): + ... yield from subgen() + ... + >>> g = delegating_gen() + >>> try: + ... next(g) + ... g.throw(ValueError) + ... except StopIteration: + ... print("gen stop") + ... + 9527 + get value error + gen stop + + # yield from + yield from + >>> import inspect + >>> def subgen(): + ... yield from range(5) + ... + >>> def delegating_gen(): + ... yield from subgen() + ... + >>> g = delegating_gen() + >>> inspect.getgeneratorstate(g) + 'GEN_CREATED' + >>> next(g) + 0 + >>> inspect.getgeneratorstate(g) + 'GEN_SUSPENDED' + >>> g.close() + >>> inspect.getgeneratorstate(g) + 'GEN_CLOSED' + +yield (from) EXPR return RES +---------------------------- + +.. code-block:: python + + >>> def average(): + ... total = .0 + ... count = 0 + ... avg = None + ... while True: + ... val = yield + ... if not val: + ... break + ... total += val + ... count += 1 + ... avg = total / count + ... return avg + ... + >>> g = average() + >>> g = average() + >>> next(g) # start gen + >>> g.send(3) + >>> g.send(5) + >>> try: + ... g.send(None) + ... except StopIteration as e: + ... ret = e.value + ... + >>> ret + 4.0 + + # yield from EXP return RES + >>> def subgen(): + ... yield 9527 + ... + >>> def delegating_gen(): + ... yield from subgen() + ... return 5566 + ... + >>> try: + ... g = delegating_gen() + ... next(g) + ... next(g) + ... except StopIteration as _e: + ... print(_e.value) + ... + 9527 + 5566 + +Generate sequences +------------------ + +.. code-block:: python + + # get a list via generator + >>> def chain(): + ... for _ in 'ab': + ... yield _ + ... for _ in range(3): + ... yield _ + ... + >>> a = list(chain()) + >>> a + ['a', 'b', 0, 1, 2] + # equivalent to + >>> def chain(): + ... yield from 'ab' + ... yield range(3) + ... + >>> a = list(chain()) + >>> a + ['a', 'b', range(0, 3)] + +What "RES = yield from EXP" actually do? +---------------------------------------- + +.. code-block:: python + + # ref: pep380 + >>> def subgen(): + ... for _ in range(3): + ... yield _ + ... + >>> EXP = subgen() + >>> def delegating_gen(): + ... _i = iter(EXP) + ... try: + ... _y = next(_i) + ... except StopIteration as _e: + ... RES = _e.value + ... else: + ... while True: + ... _s = yield _y + ... try: + ... _y = _i.send(_s) + ... except StopIteration as _e: + ... RES = _e.value + ... break + ... + >>> g = delegating_gen() + >>> next(g) + 0 + >>> next(g) + 1 + >>> next(g) + 2 + + # equivalent to + >>> EXP = subgen() + >>> def delegating_gen(): + ... RES = yield from EXP + ... + >>> g = delegating_gen() + >>> next(g) + 0 + >>> next(g) + 1 + +Check generator type +-------------------- + +.. code-block:: python + + >>> from types import GeneratorType + >>> def gen_func(): + ... yield 5566 + ... + >>> g = gen_func() + >>> isinstance(g, GeneratorType) + True + >>> isinstance(123, GeneratorType) + False + +Check Generator State +--------------------- + +.. code-block:: python + + >>> import inspect + >>> def gen_func(): + ... yield 9527 + ... + >>> g = gen_func() + >>> inspect.getgeneratorstate(g) + 'GEN_CREATED' + >>> next(g) + 9527 + >>> inspect.getgeneratorstate(g) + 'GEN_SUSPENDED' + >>> g.close() + >>> inspect.getgeneratorstate(g) + 'GEN_CLOSED' + +Context manager and generator +----------------------------- + +.. code-block:: python + + >>> import contextlib + >>> @contextlib.contextmanager + ... def mylist(): + ... try: + ... l = [1,2,3,4,5] + ... yield l + ... finally: + ... print("exit scope") + ... + >>> with mylist() as l: + ... print(l) + ... + [1, 2, 3, 4, 5] + exit scope + +What @contextmanager actually doing? +------------------------------------ + +.. code-block:: python + + # ref: PyCon 2014 - David Beazley + # define a context manager class + class GeneratorCM(object): + def __init__(self,gen): + self._gen = gen + def __enter__(self): + return next(self._gen) + def __exit__(self, *exc_info): + try: + if exc_info[0] is None: + next(self._gen) + else: + self._gen.throw(*exc_info) + raise RuntimeError + except StopIteration: + return True + except: + raise + + # define a decorator + def contextmanager(func): + def run(*a, **k): + return GeneratorCM(func(*a, **k)) + return run + + # example of context manager + @contextmanager + def mylist(): + try: + l=[1,2,3,4,5] + yield l + finally: + print "exit scope" + + with mylist() as l: + print l + +output: + +.. code-block:: console + + $ python ctx.py + [1, 2, 3, 4, 5] + exit scope + +'yield from' and '__iter__' +--------------------------- + +.. code-block:: python + + >>> class FakeGen: + ... def __iter__(self): + ... n = 0 + ... while True: + ... yield n + ... n += 1 + ... def __reversed(self): + ... n = 9527 + ... while True: + ... yield n + ... n -= 1 + ... + >>> def spam(): + ... yield from FakeGen() + ... + >>> s = spam() + >>> next(s) + 0 + >>> next(s) + 1 + >>> next(s) + 2 + >>> next(s) + 3 + >>> def reversed_spam(): + ... yield from reversed(FakeGen()) + ... + >>> g = reversed_spam() + >>> next(g) + 9527 + >>> next(g) + 9526 + >>> next(g) + 9525 + +"yield from == await" expression +-------------------------------- + +.. code-block:: python + + # "await" include in pyhton3.5 + import asyncio + import socket + + # set socket and event loop + loop = asyncio.get_event_loop() + host = 'localhost' + port = 5566 + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM,0) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,1) + sock.setblocking(False) + sock.bind((host, port)) + sock.listen(10) + + @asyncio.coroutine + def echo_server(): + while True: + conn, addr = yield from loop.sock_accept(sock) + loop.create_task(handler(conn)) + + @asyncio.coroutine + def handler(conn): + while True: + msg = yield from loop.sock_recv(conn, 1024) + if not msg: + break + yield from loop.sock_sendall(conn, msg) + conn.close() + + # equal to + async def echo_server(): + while True: + conn, addr = await loop.sock_accept(sock) + loop.create_task(handler(conn)) + + async def handler(conn): + while True: + msg = await loop.sock_recv(conn, 1024) + if not msg: + break + await loop.sock_sendall(conn, msg) + conn.close() + + loop.create_task(echo_server()) + loop.run_forever() + +output: (bash 1) + +.. code-block:: console + + $ nc localhost 5566 + Hello + Hello + + +output: (bash 2) + +.. code-block:: console + + $ nc localhost 5566 + World + World + + +Closure in Python - using generator +----------------------------------- + +.. code-block:: python + + # nonlocal version + >>> def closure(): + ... x = 5566 + ... def inner_func(): + ... nonlocal x + ... x += 1 + ... return x + ... return inner_func + ... + >>> c = closure() + >>> c() + 5567 + >>> c() + 5568 + >>> c() + 5569 + + # class version + >>> class Closure: + ... def __init__(self): + ... self._x = 5566 + ... def __call__(self): + ... self._x += 1 + ... return self._x + ... + >>> c = Closure() + >>> c() + 5567 + >>> c() + 5568 + >>> c() + 5569 + + # generator version (best) + >>> def closure_gen(): + ... x = 5566 + ... while True: + ... x += 1 + ... yield x + ... + >>> g = closure_gen() + >>> next(g) + 5567 + >>> next(g) + 5568 + >>> next(g) + 5569 + + +Implement a simple scheduler +---------------------------- + +.. code-block:: python + + # idea: write an event loop(scheduler) + >>> def fib(n): + ... if n<=2: + ... return 1 + ... return fib(n-1)+fib(n-2) + ... + >>> def g_fib(n): + ... for _ in range(1,n+1): + ... yield fib(_) + ... + >>> from collections import deque + >>> t = [g_fib(3),g_fib(5)] + >>> q = deque() + >>> q.extend(t) + >>> def run(): + ... while q: + ... try: + ... t = q.popleft() + ... print(next(t)) + ... q.append(t) + ... except StopIteration: + ... print("Task done") + ... + >>> run() + 1 + 1 + 1 + 1 + 2 + 2 + Task done + 3 + 5 + Task done + +Simple round-robin with blocking +-------------------------------- + +.. code-block:: python + + # ref: PyCon 2015 - David Beazley + # skill: using task and wait queue + from collections import deque + from select import select + import socket + + tasks = deque() + w_read = {} + w_send = {} + + def run(): + while any([tasks,w_read,w_send]): + while not tasks: + # polling tasks + can_r,can_s,_ = select( + w_read,w_send,[]) + for _r in can_r: + tasks.append(w_read.pop(_r)) + for _w in can_s: + tasks.append(w_send.pop(_w)) + try: + task = tasks.popleft() + why,what = next(task) + if why == 'recv': + w_read[what] = task + elif why == 'send': + w_send[what] = task + else: + raise RuntimeError + except StopIteration: + pass + + def server(): + host = ('localhost',5566) + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(host) + sock.listen(5) + while True: + # tell scheduler want block + yield 'recv', sock + conn,addr = sock.accept() + tasks.append(client_handler(conn)) + + def client_handler(conn): + while True: + # tell scheduler want block + yield 'recv', conn + msg = conn.recv(1024) + if not msg: + break + # tell scheduler want block + yield 'send', conn + conn.send(msg) + conn.close() + + tasks.append(server()) + run() + +simple round-robin with blocking and non-blocking +------------------------------------------------- + +.. code-block:: python + + # this method will cause blocking hunger + from collections import deque + from select import select + import socket + + tasks = deque() + w_read = {} + w_send = {} + + def run(): + while any([tasks,w_read,w_send]): + while not tasks: + # polling tasks + can_r,can_s,_ = select( + w_read,w_send,[]) + for _r in can_r: + tasks.append(w_read.pop(_r)) + for _w in can_s: + tasks.append(w_send.pop(_w)) + try: + task = tasks.popleft() + why,what = next(task) + if why == 'recv': + w_read[what] = task + elif why == 'send': + w_send[what] = task + elif why == 'continue': + print what + tasks.append(task) + else: + raise RuntimeError + except StopIteration: + pass + + def fib(n): + if n<=2: + return 1 + return fib(n-1)+fib(n-2) + + def g_fib(n): + for _ in range(1,n+1): + yield 'continue', fib(_) + tasks.append(g_fib(15)) + + def server(): + host = ('localhost',5566) + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(host) + sock.listen(5) + while True: + yield 'recv', sock + conn,addr = sock.accept() + tasks.append(client_handler(conn)) + + def client_handler(conn): + while True: + yield 'recv', conn + msg = conn.recv(1024) + if not msg: + break + yield 'send', conn + conn.send(msg) + conn.close() + + tasks.append(server()) + run() diff --git a/docs/source/notes/python-socket.rst b/docs/source/notes/python-socket.rst new file mode 100644 index 00000000..a7087e09 --- /dev/null +++ b/docs/source/notes/python-socket.rst @@ -0,0 +1,437 @@ +======================== +Python socket cheatsheet +======================== + +Get Hostname +------------ + +.. code-block:: python + + >>> import socket + >>> socket.gethostname() + 'MacBookPro-4380.local' + >>> hostname = socket.gethostname() + >>> socket.gethostbyname(hostname) + '172.20.10.4' + >>> socket.gethostbyname('localhost') + '127.0.0.1' + +Simple TCP Echo Server +---------------------- + +.. code-block:: python + + import socket + + class Server(object): + def __init__(self,host,port): + self._host = host + self._port = port + def __enter__(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,1) + sock.bind((self._host,self._port)) + sock.listen(10) + self._sock = sock + return self._sock + def __exit__(self,*exc_info): + if exc_info[0]: + import traceback + traceback.print_exception(*exc_info) + self._sock.close() + + if __name__ == '__main__': + host = 'localhost' + port = 5566 + with Server(host,5566) as s: + while True: + conn, addr = s.accept() + msg = conn.recv(1024) + conn.send(msg) + conn.close() + +output: + +.. code-block:: console + + $ nc localhost 5566 + Hello World + Hello World + +Simple TCP Echo Server Via SocketServer +--------------------------------------- + +.. code-block:: python + + >>> import SocketServer + >>> bh = SocketServer.BaseRequestHandler + >>> class handler(bh): + ... def handle(self): + ... data = self.request.recv(1024) + ... print self.client_address + ... self.request.sendall(data) + ... + >>> host = ('localhost',5566) + >>> s = SocketServer.TCPServer( + ... host, handler) + >>> s.serve_forever() + + output: + +.. code-block:: console + + $ nc -u localhost 5566 + Hello World + Hello World + +Simple UDP Echo Server +---------------------- + +.. code-block:: python + + import socket + + class UDPServer(object): + def __init__(self,host,port): + self._host = host + self._port = port + + def __enter__(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.bind((self._host,self._port)) + self._sock = sock + return sock + def __exit__(self,*exc_info): + if exc_info[0]: + import traceback + traceback.print_exception(*exc_info) + self._sock.close() + + if __name__ == '__main__': + host = 'localhost' + port = 5566 + with UDPServer(host,port) as s: + while True: + msg, addr = s.recvfrom(1024) + s.sendto(msg, addr) + +output: + +.. code-block:: console + + $ nc -u localhost 5566 + Hello World + Hello World + + +Simple UDP Echo Server Via SocketServer +--------------------------------------- + +.. code-block:: python + + >>> import SocketServer + >>> bh = SocketServer.BaseRequestHandler + >>> class handler(bh): + ... def handle(self): + ... m,s = self.request + ... s.sendto(m,self.client_address) + ... print self.client_address + ... + >>> host = ('localhost',5566) + >>> s = SocketServer.UDPServer( + ... host, handler) + >>> s.serve_forever() + +output: + +.. code-block:: console + + $ nc -u localhost 5566 + Hello World + Hello World + + +Simple UDP client - Sender +-------------------------- + +.. code-block:: python + + >>> import socket + >>> import time + >>> sock = socket.socket( + ... socket.AF_INET, + ... socket.SOCK_DGRAM) + >>> host = ('localhost',5566) + >>> while True: + ... sock.sendto("Hello\n",host) + ... time.sleep(5) + ... + +output: + +.. code-block:: console + + $ nc -lu localhost 5566 + Hello + Hello + +Broadcast UDP Packets +--------------------- + +.. code-block:: python + + >>> import socket + >>> import time + >>> sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + >>> sock.bind(('',0)) + >>> sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST,1) + >>> while True: + ... m = '{0}\n'.format(time.time()) + ... sock.sendto(m,('',5566)) + ... time.sleep(5) + ... + +output: + +.. code-block:: console + + $ nc -k -w 1 -ul 5566 + 1431473025.72 + +Simple UNIX Domain Socket +------------------------- + +.. code-block:: python + + import socket + import contextlib + import os + + @contextlib.contextmanager + def DomainServer(addr): + try: + if os.path.exists(addr): + os.unlink(addr) + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.bind(addr) + sock.listen(10) + yield sock + finally: + sock.close() + if os.path.exists(addr): + os.unlink(addr) + + addr = "./domain.sock" + with DomainServer(addr) as sock: + while True: + conn, _ = sock.accept() + msg = conn.recv(1024) + conn.send(msg) + conn.close() + +output: + +.. code-block:: console + + $ nc -U ./domain.sock + Hello + Hello + +Simple Asynchronous TCP Server - Thread +--------------------------------------- + +.. code-block:: python + + >>> from threading import Thread + >>> import socket + >>> def work(conn): + ... while True: + ... msg = conn.recv(1024) + ... conn.send(msg) + ... + >>> sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + >>> sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,1) + >>> sock.bind(('localhost',5566)) + >>> sock.listen(5) + >>> while True: + ... conn,addr = sock.accept() + ... t=Thread(target=work,args=(conn,)) + ... t.daemon=True + ... t.start() + ... + +output: (bash 1) + +.. code-block:: console + + $ nc localhost 5566 + Hello + Hello + +output: (bash 2) + +.. code-block:: console + + $ nc localhost 5566 + Ker Ker + Ker Ker + +Simple Asynchronous TCP Server - select +--------------------------------------- + +.. code-block:: python + + from select import select + import socket + + host = ('localhost',5566) + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,1) + sock.bind(host) + sock.listen(5) + rl = [sock] + wl = [] + ml = {} + try: + while True: + r, w, _ = select(rl,wl,[]) + # process ready to ready + for _ in r: + if _ == sock: + conn, addr = sock.accept() + rl.append(conn) + else: + msg = _.recv(1024) + ml[_.fileno()] = msg + wl.append(_) + # process ready to write + for _ in w: + msg = ml[_.fileno()] + _.send(msg) + wl.remove(_) + del ml[_.fileno()] + except: + sock.close() + +output: (bash 1) + +.. code-block:: console + + $ nc localhost 5566 + Hello + Hello + +output: (bash 2) + +.. code-block:: console + + $ nc localhost 5566 + Ker Ker + Ker Ker + +High-Level API - selectors +-------------------------- + +.. code-block:: python + + # Pyton3.4+ only + # Reference: selectors + import selectors + import socket + import contextlib + + @contextlib.contextmanager + def Server(host,port): + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind((host,port)) + s.listen(10) + sel = selectors.DefaultSelector() + yield s, sel + except socket.error: + print("Get socket error") + raise + finally: + if s: + s.close() + + def read_handler(conn, sel): + msg = conn.recv(1024) + if msg: + conn.send(msg) + else: + sel.unregister(conn) + conn.close() + + def accept_handler(s, sel): + conn, _ = s.accept() + sel.register(conn, selectors.EVENT_READ, read_handler) + + host = 'localhost' + port = 5566 + with Server(host, port) as (s,sel): + sel.register(s, selectors.EVENT_READ, accept_handler) + while True: + events = sel.select() + for sel_key, m in events: + handler = sel_key.data + handler(sel_key.fileobj, sel) + +output: (bash 1) + +.. code-block:: console + + $ nc localhost 5566 + Hello + Hello + +output: (bash 1) + +.. code-block:: console + + $ nc localhost 5566 + Hi + Hi + +"socketpair" - Similar to PIPE +------------------------------ + +.. code-block:: python + + import socket + import os + import time + + c_s, p_s = socket.socketpair() + try: + pid = os.fork() + except OSError: + print "Fork Error" + raise + + if pid: + # parent process + c_s.close() + while True: + p_s.sendall("Hi! Child!") + msg = p_s.recv(1024) + print msg + time.sleep(3) + os.wait() + else: + # child process + p_s.close() + while True: + msg = c_s.recv(1024) + print msg + c_s.sendall("Hi! Parent!") + +.. code-block:: console + + $ python ex.py + Hi! Child! + Hi! Parent! + Hi! Child! + Hi! Parent! + ... diff --git a/docs/source/notes/python-tests.rst b/docs/source/notes/python-tests.rst new file mode 100644 index 00000000..c95b9427 --- /dev/null +++ b/docs/source/notes/python-tests.rst @@ -0,0 +1,639 @@ +====================== +Python test cheatsheet +====================== + +A simple Python unittest +------------------------ + +.. code-block:: python + + # python unittet only run the function with prefix "test" + >>> import unittest + >>> class TestFoo(unittest.TestCase): + ... def test_foo(self): + ... self.assertTrue(True) + ... def fun_not_run(self): + ... print "no run" + ... + >>> unittest.main() + . + ---------------------------------------------------------------------- + Ran 1 test in 0.000s + + OK + >>> import unittest + >>> class TestFail(unittest.TestCase): + ... def test_false(self): + ... self.assertTrue(False) + ... + >>> unittest.main() + F + ====================================================================== + FAIL: test_false (__main__.TestFail) + ---------------------------------------------------------------------- + Traceback (most recent call last): + File "", line 3, in test_false + AssertionError: False is not true + + ---------------------------------------------------------------------- + Ran 1 test in 0.000s + + FAILED (failures=1) + + +Python unittest setup & teardown hierarchy +------------------------------------------ + +.. code-block:: python + + import unittest + + def fib(n): + return 1 if n<=2 else fib(n-1)+fib(n-2) + + def setUpModule(): + print "setup module" + def tearDownModule(): + print "teardown module" + + class TestFib(unittest.TestCase): + + def setUp(self): + print "setUp" + self.n = 10 + def tearDown(self): + print "tearDown" + del self.n + @classmethod + def setUpClass(cls): + print "setUpClass" + @classmethod + def tearDownClass(cls): + print "tearDownClass" + def test_fib_assert_equal(self): + self.assertEqual(fib(self.n), 55) + def test_fib_assert_true(self): + self.assertTrue(fib(self.n) == 55) + + if __name__ == "__main__": + unittest.main() + +output: + +.. code-block:: console + + $ python test.py + setup module + setUpClass + setUp + tearDown + .setUp + tearDown + .tearDownClass + teardown module + + ---------------------------------------------------------------------- + Ran 2 tests in 0.000s + + OK + +Different module of setUp & tearDown hierarchy +---------------------------------------------- + +.. code-block:: python + + # test_module.py + import unittest + + class TestFoo(unittest.TestCase): + @classmethod + def setUpClass(self): + print "foo setUpClass" + @classmethod + def tearDownClass(self): + print "foo tearDownClass" + def setUp(self): + print "foo setUp" + def tearDown(self): + print "foo tearDown" + def test_foo(self): + self.assertTrue(True) + + class TestBar(unittest.TestCase): + def setUp(self): + print "bar setUp" + def tearDown(self): + print "bar tearDown" + def test_bar(self): + self.assertTrue(True) + + # test.py + from test_module import TestFoo + from test_module import TestBar + import test_module + import unittest + + def setUpModule(): + print "setUpModule" + + def tearDownModule(): + print "tearDownModule" + + + if __name__ == "__main__": + test_module.setUpModule = setUpModule + test_module.tearDownModule = tearDownModule + suite1 = unittest.TestLoader().loadTestsFromTestCase(TestFoo) + suite2 = unittest.TestLoader().loadTestsFromTestCase(TestBar) + suite = unittest.TestSuite([suite1,suite2]) + unittest.TextTestRunner().run(suite) + + +output: + +.. code-block:: console + + $ python test.py + setUpModule + foo setUpClass + foo setUp + foo tearDown + .foo tearDownClass + bar setUp + bar tearDown + .tearDownModule + + ---------------------------------------------------------------------- + Ran 2 tests in 0.000s + + OK + +Run tests via unittest.TextTestRunner +------------------------------------- + +.. code-block:: python + + >>> import unittest + >>> class TestFoo(unittest.TestCase): + ... def test_foo(self): + ... self.assertTrue(True) + ... def test_bar(self): + ... self.assertFalse(False) + + >>> suite = unittest.TestLoader().loadTestsFromTestCase(TestFoo) + >>> unittest.TextTestRunner(verbosity=2).run(suite) + test_bar (__main__.TestFoo) ... ok + test_foo (__main__.TestFoo) ... ok + + ---------------------------------------------------------------------- + Ran 2 tests in 0.000s + + OK + +Test raise exception +-------------------- + +.. code-block:: python + + >>> import unittest + + >>> class TestRaiseException(unittest.TestCase): + ... def test_raise_except(self): + ... with self.assertRaises(SystemError): + ... raise SystemError + >>> suite_loader = unittest.TestLoader() + >>> suite = suite_loader.loadTestsFromTestCase(TestRaiseException) + >>> unittest.TextTestRunner().run(suite) + . + ---------------------------------------------------------------------- + Ran 1 test in 0.000s + + OK + >>> class TestRaiseFail(unittest.TestCase): + ... def test_raise_fail(self): + ... with self.assertRaises(SystemError): + ... pass + >>> suite = unittest.TestLoader().loadTestsFromTestCase(TestRaiseFail) + >>> unittest.TextTestRunner(verbosity=2).run(suite) + test_raise_fail (__main__.TestRaiseFail) ... FAIL + + ====================================================================== + FAIL: test_raise_fail (__main__.TestRaiseFail) + ---------------------------------------------------------------------- + Traceback (most recent call last): + File "", line 4, in test_raise_fail + AssertionError: SystemError not raised + + ---------------------------------------------------------------------- + Ran 1 test in 0.000s + + FAILED (failures=1) + + +Pass arguments into a TestCase +------------------------------ + +.. code-block:: python + + >>> import unittest + >>> class TestArg(unittest.TestCase): + ... def __init__(self, testname, arg): + ... super(TestArg, self).__init__(testname) + ... self._arg = arg + ... def setUp(self): + ... print "setUp:", self._arg + ... def test_arg(self): + ... print "test_arg:", self._arg + ... self.assertTrue(True) + ... + >>> suite = unittest.TestSuite() + >>> suite.addTest(TestArg('test_arg', 'foo')) + >>> unittest.TextTestRunner(verbosity=2).run(suite) + test_arg (__main__.TestArg) ... setUp: foo + test_arg: foo + ok + + ---------------------------------------------------------------------- + Ran 1 test in 0.000s + + OK + +Group multiple testcases into a suite +------------------------------------- + +.. code-block:: python + + >>> import unittest + >>> class TestFooBar(unittest.TestCase): + ... def test_foo(self): + ... self.assertTrue(True) + ... def test_bar(self): + ... self.assertTrue(True) + ... + >>> class TestHelloWorld(unittest.TestCase): + ... def test_hello(self): + ... self.assertEqual("Hello", "Hello") + ... def test_world(self): + ... self.assertEqual("World", "World") + ... + >>> suite_loader = unittest.TestLoader() + >>> suite1 = suite_loader.loadTestsFromTestCase(TestFooBar) + >>> suite2 = suite_loader.loadTestsFromTestCase(TestHelloWorld) + >>> suite = unittest.TestSuite([suite1, suite2]) + >>> unittest.TextTestRunner(verbosity=2).run(suite) + test_bar (__main__.TestFooBar) ... ok + test_foo (__main__.TestFooBar) ... ok + test_hello (__main__.TestHelloWorld) ... ok + test_world (__main__.TestHelloWorld) ... ok + + ---------------------------------------------------------------------- + Ran 4 tests in 0.000s + + OK + +Group multiple tests from different TestCase +-------------------------------------------- + +.. code-block:: python + + >>> import unittest + >>> class TestFoo(unittest.TestCase): + ... def test_foo(self): + ... assert "foo" == "foo" + ... + >>> class TestBar(unittest.TestCase): + ... def test_bar(self): + ... assert "bar" == "bar" + ... + >>> suite = unittest.TestSuite() + >>> suite.addTest(TestFoo('test_foo')) + >>> suite.addTest(TestBar('test_bar')) + >>> unittest.TextTestRunner(verbosity=2).run(suite) + test_foo (__main__.TestFoo) ... ok + test_bar (__main__.TestBar) ... ok + + ---------------------------------------------------------------------- + Ran 2 tests in 0.001s + + OK + +Skip some tests in the TestCase +------------------------------- + +.. code-block:: python + + >>> import unittest + >>> RUN_FOO = False + >>> DONT_RUN_BAR = False + >>> class TestSkip(unittest.TestCase): + ... def test_always_run(self): + ... self.assertTrue(True) + ... @unittest.skip("always skip this test") + ... def test_always_skip(self): + ... raise RuntimeError + ... @unittest.skipIf(RUN_FOO == False, "demo skipIf") + ... def test_skipif(self): + ... raise RuntimeError + ... @unittest.skipUnless(DONT_RUN_BAR == True, "demo skipUnless") + ... def test_skipunless(self): + ... raise RuntimeError + ... + >>> suite = unittest.TestLoader().loadTestsFromTestCase(TestSkip) + >>> unittest.TextTestRunner(verbosity=2).run(suite) + test_always_run (__main__.TestSkip) ... ok + test_always_skip (__main__.TestSkip) ... skipped 'always skip this test' + test_skipif (__main__.TestSkip) ... skipped 'demo skipIf' + test_skipunless (__main__.TestSkip) ... skipped 'demo skipUnless' + + ---------------------------------------------------------------------- + Ran 4 tests in 0.000s + + OK (skipped=3) + +Cross-module variables to Test files +------------------------------------ + +test_foo.py + +.. code-block:: python + + import unittest + + print conf + + class TestFoo(unittest.TestCase): + def test_foo(self): + print conf + + @unittest.skipIf(conf.isskip==True, "skip test") + def test_skip(self): + raise RuntimeError + +test_bar.py + +.. code-block:: python + + import unittest + import __builtin__ + + if __name__ == "__main__": + conf = type('TestConf', (object,), {}) + conf.isskip = True + + # make a cross-module variable + __builtin__.conf = conf + module = __import__('test_foo') + loader = unittest.TestLoader() + suite = loader.loadTestsFromTestCase(module.TestFoo) + unittest.TextTestRunner(verbosity=2).run(suite) + +output: + +.. code-block:: console + + $ python test_bar.py + + test_foo (test_foo.TestFoo) ... + ok + test_skip (test_foo.TestFoo) ... skipped 'skip test' + + ---------------------------------------------------------------------- + Ran 2 tests in 0.000s + + OK (skipped=1) + + +skip setup & teardown when the test is skipped +----------------------------------------------- + +.. code-block:: python + + >>> import unittest + >>> class TestSkip(unittest.TestCase): + ... def setUp(self): + ... print "setUp" + ... def tearDown(self): + ... print "tearDown" + ... @unittest.skip("skip this test") + ... def test_skip(self): + ... raise RuntimeError + ... def test_not_skip(self): + ... self.assertTrue(True) + ... + >>> suite = unittest.TestLoader().loadTestsFromTestCase(TestSkip) + >>> unittest.TextTestRunner(verbosity=2).run(suite) + test_not_skip (__main__.TestSkip) ... setUp + tearDown + ok + test_skip (__main__.TestSkip) ... skipped 'skip this test' + + ---------------------------------------------------------------------- + Ran 2 tests in 0.000s + + OK (skipped=1) + +Re-using old test code +---------------------- + +.. code-block:: python + + >>> import unittest + >>> def old_func_test(): + ... assert "Hello" == "Hello" + ... + >>> def old_func_setup(): + ... print "setup" + ... + >>> def old_func_teardown(): + ... print "teardown" + ... + >>> testcase = unittest.FunctionTestCase(old_func_test, + ... setUp=old_func_setup, + ... tearDown=old_func_teardown) + >>> suite = unittest.TestSuite([testcase]) + >>> unittest.TextTestRunner().run(suite) + setup + teardown + . + ---------------------------------------------------------------------- + Ran 1 test in 0.000s + + OK + + +Testing your document is right +------------------------------ + +.. code-block:: python + + """ + This is an example of doctest + + >>> fib(10) + 55 + """ + + def fib(n): + """ + This function calculate fib number. + + example: + + >>> fib(10) + 55 + >>> fib(-1) + Traceback (most recent call last): + ... + ValueError + """ + if n < 0: + raise ValueError('') + return 1 if n<=2 else fib(n-1) + fib(n-2) + + if __name__ == "__main__": + import doctest + doctest.testmod() + +output: + +.. code-block:: console + + $ python demo_doctest.py -v + Trying: + fib(10) + Expecting: + 55 + ok + Trying: + fib(10) + Expecting: + 55 + ok + Trying: + fib(-1) + Expecting: + Traceback (most recent call last): + ... + ValueError + ok + 2 items passed all tests: + 1 tests in __main__ + 2 tests in __main__.fib + 3 tests in 2 items. + 3 passed and 0 failed. + Test passed. + +Re-using doctest to unittest +---------------------------- + +.. code-block:: python + + import unittest + import doctest + + """ + This is an example of doctest + + >>> fib(10) + 55 + """ + + def fib(n): + """ + This function calculate fib number. + + example: + + >>> fib(10) + 55 + >>> fib(-1) + Traceback (most recent call last): + ... + ValueError + """ + if n < 0: + raise ValueError('') + return 1 if n<=2 else fib(n-1) + fib(n-2) + + if __name__ == "__main__": + finder = doctest.DocTestFinder() + suite = doctest.DocTestSuite(test_finder=finder) + unittest.TextTestRunner(verbosity=2).run(suite) + +output: + +.. code-block:: console + + fib (__main__) + Doctest: __main__.fib ... ok + + ---------------------------------------------------------------------- + Ran 1 test in 0.023s + + OK + +Mocking Test +------------ + +without mock - test will always failed +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + import unittest + import os + + class TestFoo(unittest.TestCase): + def test_foo(self): + os.remove('!@#$%^~') + + if __name__ == "__main__": + unittest.main() + +output: + +.. code-block:: console + + $ python wo_mock_test.py + E + ====================================================================== + ERROR: test_foo (__main__.TestFoo) + ---------------------------------------------------------------------- + Traceback (most recent call last): + File "mock_test.py", line 7, in test_foo + os.remove('!@#$%^~') + OSError: [Errno 2] No such file or directory: '!@#$%^~' + + ---------------------------------------------------------------------- + Ran 1 test in 0.000s + +with mock - substitute real object to fake object +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + import mock + import unittest + import os + + def mock_os_remove(path): + pass + + class TestFoo(unittest.TestCase): + @mock.patch('os.remove', mock_os_remove) + def test_foo(self): + os.remove('!@#$%^~') + + if __name__ == "__main__": + unittest.main() + +output: + +.. code-block:: console + + $ python w_mock_test.py + . + ---------------------------------------------------------------------- + Ran 1 test in 0.000s + + OK diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..33dd5701 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,18 @@ +alabaster==0.7.7 +Babel==2.2.0 +docutils==0.12 +Flask==0.10.1 +Flask-SSLify==0.1.5 +itsdangerous==0.24 +Jinja2==2.8 +MarkupSafe==0.23 +PyAudio==0.2.9 +Pygments==2.1.1 +pytz==2015.7 +six==1.10.0 +snowballstemmer==1.2.1 +SpeechRecognition==3.2.0 +Sphinx==1.3.5 +sphinx-rtd-theme==0.1.9 +Werkzeug==0.11.3 +wheel==0.24.0