diff options
Diffstat (limited to '')
50 files changed, 8968 insertions, 0 deletions
diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..e86cbd4 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,30 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build +PYTHON ?= python3 + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) || true + +serve: + PSYCOPG_IMPL=python sphinx-autobuild . _build/html/ + +.PHONY: help serve env Makefile + +env: .venv + +.venv: + $(PYTHON) -m venv .venv + ./.venv/bin/pip install -e "../psycopg[docs]" -e ../psycopg_pool + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_static/psycopg.css b/docs/_static/psycopg.css new file mode 100644 index 0000000..de9d779 --- /dev/null +++ b/docs/_static/psycopg.css @@ -0,0 +1,11 @@ +/* style rubric in furo (too small IMO) */ +p.rubric { + font-size: 1.2rem; + font-weight: bold; +} + +/* override a silly default */ +table.align-default td, +table.align-default th { + text-align: left; +} diff --git a/docs/_static/psycopg.svg b/docs/_static/psycopg.svg new file mode 100644 index 0000000..0e9ee32 --- /dev/null +++ b/docs/_static/psycopg.svg @@ -0,0 +1 @@ +<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 228 148"><path fill="#ffc836" stroke="#000" stroke-width="7.415493" d="M142.3 67.6c.6-4.7.4-5.4 4-4.6h.8c2.7.2 6.2-.4 8.3-1.3 4.4-2.1 7-5.5 2.7-4.6-10 2-10.7-1.4-10.7-1.4 10.5-15.6 15-35.5 11.1-40.3-10.3-13.3-28.3-7-28.6-6.8h-.1c-2-.4-4.2-.7-6.7-.7-4.5 0-8 1.2-10.5 3.1 0 0-32-13.2-30.6 16.6.4 6.4 9.1 48 19.6 35.4 3.8-4.6 7.5-8.4 7.5-8.4 1.8 1.2 4 1.8 6.3 1.6l.2-.2a7 7 0 000 1.8c-2.6 3-1.8 3.5-7.2 4.7-5.4 1-2.2 3-.2 3.6 2.6.6 8.4 1.5 12.4-4l-.2.6c1.1.9 1 6.1 1.2 9.8.1 3.8.4 7.3 1.1 9.3.8 2 1.7 7.4 8.8 5.9 5.9-1.3 10.4-3.1 10.8-20.1"/><path fill="#ff0" d="M105.4 54.2a2.4 2.4 0 114.8 0c0 1.3-1.1 2.4-2.4 2.4-1.3 0-2.4-1-2.4-2.4z"/><g fill="#336791"><path stroke="#000" stroke-width="7.415493" d="M85.7 80.4c-.6 4.7-.4 5.4-4 4.6H81c-2.7-.2-6.2.4-8.3 1.3-4.4 2.1-7 5.5-2.7 4.6 10-2 10.7 1.4 10.7 1.4-10.5 15.6-15 35.5-11.1 40.3 10.3 13.3 28.3 7 28.6 6.8h.1c2 .4 4.2.7 6.7.7 4.5 0 8-1.2 10.5-3.1 0 0 32 13.2 30.6-16.6-.4-6.4-9.1-48-19.6-35.4-3.8 4.6-7.5 8.4-7.5 8.4a9.7 9.7 0 00-6.3-1.6l-.2.2a7 7 0 000-1.8c2.6-3 1.8-3.5 7.2-4.7 5.4-1 2.2-3 .2-3.6-2.6-.6-8.4-1.5-12.4 4l.2-.6c-1.1-.9-1-6.1-1.2-9.8-.1-3.8-.4-7.3-1.1-9.3-.8-2-1.7-7.4-8.8-5.9-5.9 1.3-10.4 3.1-10.8 20.1"/><path d="M70 91c10-2.1 10.6 1.3 10.6 1.3-10.5 15.6-15 35.5-11.1 40.3 10.3 13.3 28.3 7 28.6 6.8h.1c2 .4 4.2.7 6.7.7 4.5 0 8-1.2 10.5-3.1 0 0 32 13.2 30.6-16.6-.4-6.4-9.1-48-19.6-35.4-3.8 4.6-7.5 8.4-7.5 8.4a9.7 9.7 0 00-6.3-1.6l-.2.2a7 7 0 000-1.8c2.6-3 1.8-3.5 7.2-4.7 5.5-1 2.2-3 .2-3.6-2.6-.6-8.4-1.5-12.4 4l.2-.6c-1.1-.9-1.8-5.5-1.7-9.7.1-4.3.2-7.2-.6-9.4s-1.7-7.4-8.8-5.9c-5.9 1.3-9 4.6-9.4 10-.3 4-1 3.4-1 6.9l-.6 1.6c-.6 5.3 0 7-3.7 6.2h-.9c-2.7-.2-6.2.4-8.3 1.3-4.4 2.1-7 5.5-2.7 4.6z"/><g stroke="#ffc836" stroke-linecap="round" stroke-width="15.6"><g stroke-linejoin="round"><path stroke-width="2.477124" d="M107 88c.2-10-.1-19.8-1-22.2-1-2.4-3-7.1-10.2-5.6-5.9 1.3-8 3.7-9 9.1-.7 4-2 15.1-2.2 17.4M115.5 137.2s32 13 30.5-16.7c-.3-6.4-9-48-19.5-35.4-3.8 4.6-7.3 8.2-7.3 8.2M98.1 139.6c1.2-.4-17.8 6.9-28.6-6.9-3.8-4.8.6-24.7 11.2-40.3"/></g><path stroke-linejoin="bevel" stroke-width="2.477124" d="M80.7 92.4S80 89 70 91c-4.4.9-1.7-2.6 2.7-4.6 3.7-1.7 11.8-2.2 12 .2.3 6-4.3 4.2-4 5.7.3 1.3 2.4 2.7 3.7 6 1.2 2.9 16.5 25.2-4.2 21.9-.7.1 5.4 19.6 24.7 20 19.4.3 18.7-23.8 18.7-23.8"/><path stroke-linejoin="round" stroke-width="2.477124" d="M112.4 90.3c2.7-3 1.9-3.5 7.3-4.6 5.4-1.2 2.2-3.2.1-3.7-2.5-.6-8.4-1.5-12.3 4-1.2 1.7 0 4.4 1.6 5.1.8.3 2 .8 3.3-.8z"/><path stroke-linejoin="round" stroke-width="2.477124" d="M112.6 90.4c.2 1.7-.6 3.8-1.5 6.3-1.4 3.7-4.6 7.4-2 19.1 1.8 8.8 14.5 1.8 14.5.7 0-1.2-.5-6 .2-11.7 1-7.3-4.6-13.5-11.2-12.8"/></g></g><g stroke="#ffc836"><path fill="#ffc836" stroke-width=".825708" d="M115.6 116.6c0-.4-.8-1.4-1.8-1.6-1-.1-2 .7-2 1.1 0 .4.7.9 1.8 1 1 .2 2 0 2-.5z"/><path fill="#ffc836" stroke-width=".412854" d="M84 117.5c-.1-.4.7-1.5 1.7-1.7 1-.1 2 .7 2 1.1 0 .4-.7.9-1.8 1s-2 0-2-.4z"/><path fill="#336791" stroke-linecap="round" stroke-linejoin="round" stroke-width="2.477124" d="M80.2 120.3c-.2-3.2.7-5.4.8-8.7.2-5-2.3-10.6 1.4-16.2"/></g><g fill="#ffc836"><path d="M158 57c-10 2.1-10.6-1.3-10.6-1.3 10.5-15.6 15-35.5 11.1-40.3-10.3-13.3-28.3-7-28.6-6.8h-.1c-2-.4-4.2-.7-6.7-.7-4.5 0-8 1.2-10.5 3.1 0 0-32-13.2-30.6 16.6.4 6.4 9.1 48 19.6 35.4 3.8-4.6 7.5-8.5 7.5-8.5 1.8 1.3 4 1.9 6.3 1.7l.2-.2a7 7 0 000 1.8c-2.6 3-1.8 3.5-7.2 4.7-5.5 1-2.3 3-.2 3.6 2.6.6 8.4 1.5 12.4-4l-.2.6c1.1.9 1.8 5.5 1.7 9.7-.1 4.3-.2 7.2.6 9.4s1.7 7.4 8.8 5.9c5.9-1.3 9-4.6 9.4-10 .3-4 1-3.4 1-6.9l.6-1.6c.6-5.3 0-7 3.7-6.2h.9c2.7.2 6.2-.4 8.3-1.3 4.4-2.1 7-5.5 2.7-4.6z"/><path d="M142.3 67.6c.6-4.7.4-5.4 4-4.6h.8c2.7.2 6.2-.4 8.3-1.3 4.4-2.1 7-5.5 2.7-4.6-10 2-10.7-1.4-10.7-1.4 10.5-15.6 15-35.5 11.1-40.3-10.3-13.3-28.3-7-28.6-6.8h-.1c-2-.4-4.2-.7-6.7-.7-4.5 0-8 1.2-10.5 3.1 0 0-32-13.2-30.6 16.6.4 6.4 9.1 48 19.6 35.4 3.8-4.6 7.5-8.4 7.5-8.4 1.8 1.2 4 1.8 6.3 1.6l.2-.2a7 7 0 000 1.8c-2.6 3-1.8 3.5-7.2 4.7-5.4 1-2.2 3-.2 3.6 2.6.6 8.4 1.5 12.4-4l-.2.6c1.1.9 1 6.1 1.2 9.8.1 3.8.4 7.3 1.1 9.3.8 2 1.7 7.4 8.8 5.9 5.9-1.3 10.4-3.1 10.8-20.1"/><g stroke="#336791" stroke-linecap="round" stroke-width="15.6"><g stroke-linejoin="round"><path stroke-width="2.477124" d="M112.5 10.8s-32-13-30.5 16.7c.3 6.4 9 48 19.5 35.4 3.8-4.6 7.3-8.2 7.3-8.2M121 60c-.2 10 .1 19.8 1 22.2 1 2.4 3 7.1 10.2 5.6 5.9-1.3 8-3.7 9-9.2.7-4 2-15 2.1-17.3M129.9 8.4c-1.2.4 17.8-6.9 28.6 6.9 3.8 4.8-.6 24.7-11.2 40.3"/></g><path stroke-linejoin="bevel" stroke-width="2.477124" d="M147.3 55.6S148 59 158 57c4.4-.9 1.7 2.6-2.7 4.6-3.7 1.7-11.8 2.2-12-.2-.3-6 4.3-4.2 4-5.7-.3-1.3-2.4-2.7-3.7-6-1.2-3-16.5-25.2 4.2-21.9.7-.1-5.4-19.6-24.7-20-19.4-.3-18.7 23.8-18.7 23.8"/><path stroke-linejoin="round" stroke-width="2.477124" d="M115.6 57.7c-2.7 3-1.9 3.5-7.3 4.6-5.4 1.2-2.2 3.2-.1 3.7 2.5.6 8.4 1.5 12.3-4 1.2-1.7 0-4.4-1.6-5.1-.8-.3-2-.8-3.3.8z"/><path stroke-linejoin="round" stroke-width="2.477124" d="M115.4 57.6c-.2-1.7.6-3.8 1.5-6.3 1.4-3.7 4.6-7.4 2-19.1-1.8-8.8-14.5-1.9-14.5-.7s.5 6-.2 11.7c-1 7.3 4.6 13.5 11.2 12.8"/></g></g><g stroke="#336791"><path fill="#336791" stroke-width=".825708" d="M112.4 31.4c0 .4.8 1.4 1.8 1.6 1 .1 2-.7 2-1.1 0-.4-.7-.9-1.8-1-1-.2-2 0-2 .5z"/><path fill="#336791" stroke-width=".412854" d="M144 30.5c.1.4-.7 1.5-1.7 1.7-1 .1-2-.7-2-1.1 0-.4.7-.9 1.8-1s2 0 2 .4z"/><path fill="#ffc836" stroke-linecap="round" stroke-linejoin="round" stroke-width="2.477124" d="M147.8 27.7c.2 3.2-.7 5.4-.8 8.7-.2 5 2.3 10.6-1.4 16.2"/></g><path fill="#ffc836" stroke="#336791" stroke-linecap="round" stroke-linejoin="round" stroke-width=".6034019999999999" d="M103.8 51h6.6v6.4h-6.6z" color="#000"/><path fill="#336791" stroke="#ffc836" stroke-linecap="round" stroke-linejoin="round" stroke-width="2.477124" d="M107 88c.2-10-.1-19.8-1-22.2-1-2.4-3-7.1-10.2-5.6-5.9 1.3-8 3.7-9 9.1-.7 4-2 15.1-2.2 17.4"/><path fill="#336791" d="M111.7 82.9h22.1v14.4h-22.1z" color="#000"/><path fill="#ffc836" d="M95.8 56h20.1v9.2H95.8z" color="#000"/><g fill="none"><path stroke="#ffc836" stroke-width="1.5878999999999999" d="M113.7 47.6c-2.2 0-4.2.2-6 .5-5.3 1-6.3 3-6.3 6.5v4.8H114V61H96.7a7.8 7.8 0 00-7.8 6.3A23.4 23.4 0 0089 80c1 3.7 3 6.4 6.7 6.4h4.3v-5.7a8 8 0 017.8-7.8h12.5c3.5 0 6.2-2.9 6.2-6.4V54.6c0-3.4-2.8-5.9-6.2-6.5a39 39 0 00-6.5-.5z"/><path stroke="#336791" stroke-width="2.38185" d="M128 61v5.5a8 8 0 01-7.8 8h-12.5c-3.4 0-6.3 2.9-6.3 6.3v12c0 3.3 3 5.3 6.3 6.3a21 21 0 0012.5 0c3.1-1 6.2-2.8 6.2-6.4V88H114v-1.6h18.8c3.6 0 5-2.6 6.2-6.4 1.3-3.9 1.3-7.6 0-12.7-.9-3.6-2.6-6.3-6.2-6.3z"/><path stroke="#ffc836" stroke-width="2.38185" d="M113.7 47.6c-2.2 0-4.2.2-6 .5-5.3 1-6.3 3-6.3 6.5v4.8H114V61H96.7a7.8 7.8 0 00-7.8 6.3A23.4 23.4 0 0089 80c1 3.7 3 6.4 6.7 6.4h4.3v-5.7a8 8 0 017.8-7.8h12.5c3.5 0 6.2-2.9 6.2-6.4V54.6c0-3.4-2.8-5.9-6.2-6.5a39 39 0 00-6.5-.5z"/><path stroke="#336791" stroke-width="1.5878999999999999" d="M128 61v5.5a8 8 0 01-7.8 8h-12.5c-3.4 0-6.3 2.9-6.3 6.3v12c0 3.3 3 5.3 6.3 6.3a21 21 0 0012.5 0c3.1-1 6.2-2.8 6.2-6.4V88H114v-1.6h18.8c3.6 0 5-2.6 6.2-6.4 1.3-3.9 1.3-7.6 0-12.7-.9-3.6-2.6-6.3-6.2-6.3z"/></g><g><path fill="#336791" d="M113.7 47.6c-2.2 0-4.2.2-6 .5-5.3 1-6.3 3-6.3 6.5v4.8H114V61H96.7a7.8 7.8 0 00-7.8 6.3A23.4 23.4 0 0089 80c1 3.7 3 6.4 6.7 6.4h4.3v-5.7a8 8 0 017.8-7.8h12.5c3.5 0 6.2-2.9 6.2-6.4V54.6c0-3.4-2.8-5.9-6.2-6.5a39 39 0 00-6.5-.5zm-6.8 3.9c1.3 0 2.3 1 2.3 2.4 0 1.3-1 2.3-2.3 2.3-1.3 0-2.3-1-2.3-2.3 0-1.4 1-2.4 2.3-2.4z"/><path fill="#ffc836" d="M128 61v5.5a8 8 0 01-7.8 8h-12.5c-3.4 0-6.3 2.9-6.3 6.3v12c0 3.3 3 5.3 6.3 6.3a21 21 0 0012.5 0c3.1-1 6.2-2.8 6.2-6.4V88H114v-1.6h18.8c3.6 0 5-2.6 6.2-6.4 1.3-3.9 1.3-7.6 0-12.7-.9-3.6-2.6-6.3-6.2-6.3zM121 91c1.3 0 2.3 1.1 2.3 2.4 0 1.3-1 2.4-2.3 2.4-1.3 0-2.4-1-2.4-2.4 0-1.3 1-2.4 2.4-2.4z" color="#000"/><path fill="#336791" d="M127.2 59.8h.7v.6h-.7z" color="#000"/></g></svg>
\ No newline at end of file diff --git a/docs/_templates/.keep b/docs/_templates/.keep new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/docs/_templates/.keep diff --git a/docs/advanced/adapt.rst b/docs/advanced/adapt.rst new file mode 100644 index 0000000..4323b07 --- /dev/null +++ b/docs/advanced/adapt.rst @@ -0,0 +1,269 @@ +.. currentmodule:: psycopg.adapt + +.. _adaptation: + +Data adaptation configuration +============================= + +The adaptation system is at the core of Psycopg and allows to customise the +way Python objects are converted to PostgreSQL when a query is performed and +how PostgreSQL values are converted to Python objects when query results are +returned. + +.. note:: + For a high-level view of the conversion of types between Python and + PostgreSQL please look at :ref:`query-parameters`. Using the objects + described in this page is useful if you intend to *customise* the + adaptation rules. + +- Adaptation configuration is performed by changing the + `~psycopg.abc.AdaptContext.adapters` object of objects implementing the + `~psycopg.abc.AdaptContext` protocol, for instance `~psycopg.Connection` + or `~psycopg.Cursor`. + +- Every context object derived from another context inherits its adapters + mapping: cursors created from a connection inherit the connection's + configuration. + + By default, connections obtain an adapters map from the global map + exposed as `psycopg.adapters`: changing the content of this object will + affect every connection created afterwards. You may specify a different + template adapters map using the `!context` parameter on + `~psycopg.Connection.connect()`. + + .. image:: ../pictures/adapt.svg + :align: center + +- The `!adapters` attributes are `AdaptersMap` instances, and contain the + mapping from Python types and `~psycopg.abc.Dumper` classes, and from + PostgreSQL OIDs to `~psycopg.abc.Loader` classes. Changing this mapping + (e.g. writing and registering your own adapters, or using a different + configuration of builtin adapters) affects how types are converted between + Python and PostgreSQL. + + - Dumpers (objects implementing the `~psycopg.abc.Dumper` protocol) are + the objects used to perform the conversion from a Python object to a bytes + sequence in a format understood by PostgreSQL. The string returned + *shouldn't be quoted*: the value will be passed to the database using + functions such as :pq:`PQexecParams()` so quoting and quotes escaping is + not necessary. The dumper usually also suggests to the server what type to + use, via its `~psycopg.abc.Dumper.oid` attribute. + + - Loaders (objects implementing the `~psycopg.abc.Loader` protocol) are + the objects used to perform the opposite operation: reading a bytes + sequence from PostgreSQL and creating a Python object out of it. + + - Dumpers and loaders are instantiated on demand by a `~Transformer` object + when a query is executed. + +.. note:: + Changing adapters in a context only affects that context and its children + objects created *afterwards*; the objects already created are not + affected. For instance, changing the global context will only change newly + created connections, not the ones already existing. + + +.. _adapt-example-xml: + +Writing a custom adapter: XML +----------------------------- + +Psycopg doesn't provide adapters for the XML data type, because there are just +too many ways of handling XML in Python. Creating a loader to parse the +`PostgreSQL xml type`__ to `~xml.etree.ElementTree` is very simple, using the +`psycopg.adapt.Loader` base class and implementing the +`~psycopg.abc.Loader.load()` method: + +.. __: https://www.postgresql.org/docs/current/datatype-xml.html + +.. code:: python + + >>> import xml.etree.ElementTree as ET + >>> from psycopg.adapt import Loader + + >>> # Create a class implementing the `load()` method. + >>> class XmlLoader(Loader): + ... def load(self, data): + ... return ET.fromstring(data) + + >>> # Register the loader on the adapters of a context. + >>> conn.adapters.register_loader("xml", XmlLoader) + + >>> # Now just query the database returning XML data. + >>> cur = conn.execute( + ... """select XMLPARSE (DOCUMENT '<?xml version="1.0"?> + ... <book><title>Manual</title><chapter>...</chapter></book>') + ... """) + + >>> elem = cur.fetchone()[0] + >>> elem + <Element 'book' at 0x7ffb55142ef0> + +The opposite operation, converting Python objects to PostgreSQL, is performed +by dumpers. The `psycopg.adapt.Dumper` base class makes it easy to implement one: +you only need to implement the `~psycopg.abc.Dumper.dump()` method:: + + >>> from psycopg.adapt import Dumper + + >>> class XmlDumper(Dumper): + ... # Setting an OID is not necessary but can be helpful + ... oid = psycopg.adapters.types["xml"].oid + ... + ... def dump(self, elem): + ... return ET.tostring(elem) + + >>> # Register the dumper on the adapters of a context + >>> conn.adapters.register_dumper(ET.Element, XmlDumper) + + >>> # Now, in that context, it is possible to use ET.Element objects as parameters + >>> conn.execute("SELECT xpath('//title/text()', %s)", [elem]).fetchone()[0] + ['Manual'] + +Note that it is possible to use a `~psycopg.types.TypesRegistry`, exposed by +any `~psycopg.abc.AdaptContext`, to obtain information on builtin types, or +extension types if they have been registered on that context using the +`~psycopg.types.TypeInfo`\.\ `~psycopg.types.TypeInfo.register()` method. + + +.. _adapt-example-float: + +Example: PostgreSQL numeric to Python float +------------------------------------------- + +Normally PostgreSQL :sql:`numeric` values are converted to Python +`~decimal.Decimal` instances, because both the types allow fixed-precision +arithmetic and are not subject to rounding. + +Sometimes, however, you may want to perform floating-point math on +:sql:`numeric` values, and `!Decimal` may get in the way (maybe because it is +slower, or maybe because mixing `!float` and `!Decimal` values causes Python +errors). + +If you are fine with the potential loss of precision and you simply want to +receive :sql:`numeric` values as Python `!float`, you can register on +:sql:`numeric` the same `Loader` class used to load +:sql:`float4`\/:sql:`float8` values. Because the PostgreSQL textual +representation of both floats and decimal is the same, the two loaders are +compatible. + +.. code:: python + + conn = psycopg.connect() + + conn.execute("SELECT 123.45").fetchone()[0] + # Decimal('123.45') + + conn.adapters.register_loader("numeric", psycopg.types.numeric.FloatLoader) + + conn.execute("SELECT 123.45").fetchone()[0] + # 123.45 + +In this example the customised adaptation takes effect only on the connection +`!conn` and on any cursor created from it, not on other connections. + + +.. _adapt-example-inf-date: + +Example: handling infinity date +------------------------------- + +Suppose you want to work with the "infinity" date which is available in +PostgreSQL but not handled by Python: + +.. code:: python + + >>> conn.execute("SELECT 'infinity'::date").fetchone() + Traceback (most recent call last): + ... + DataError: date too large (after year 10K): 'infinity' + +One possibility would be to store Python's `datetime.date.max` as PostgreSQL +infinity. For this, let's create a subclass for the dumper and the loader and +register them in the working scope (globally or just on a connection or +cursor): + +.. code:: python + + from datetime import date + + # Subclass existing adapters so that the base case is handled normally. + from psycopg.types.datetime import DateLoader, DateDumper + + class InfDateDumper(DateDumper): + def dump(self, obj): + if obj == date.max: + return b"infinity" + elif obj == date.min: + return b"-infinity" + else: + return super().dump(obj) + + class InfDateLoader(DateLoader): + def load(self, data): + if data == b"infinity": + return date.max + elif data == b"-infinity": + return date.min + else: + return super().load(data) + + # The new classes can be registered globally, on a connection, on a cursor + cur.adapters.register_dumper(date, InfDateDumper) + cur.adapters.register_loader("date", InfDateLoader) + + cur.execute("SELECT %s::text, %s::text", [date(2020, 12, 31), date.max]).fetchone() + # ('2020-12-31', 'infinity') + cur.execute("SELECT '2020-12-31'::date, 'infinity'::date").fetchone() + # (datetime.date(2020, 12, 31), datetime.date(9999, 12, 31)) + + +Dumpers and loaders life cycle +------------------------------ + +Registering dumpers and loaders will instruct Psycopg to use them +in the queries to follow, in the context where they have been registered. + +When a query is performed on a `~psycopg.Cursor`, a +`~psycopg.adapt.Transformer` object is created as a local context to manage +adaptation during the query, instantiating the required dumpers and loaders +and dispatching the values to perform the wanted conversions from Python to +Postgres and back. + +- The `!Transformer` copies the adapters configuration from the `!Cursor`, + thus inheriting all the changes made to the global `psycopg.adapters` + configuration, the current `!Connection`, the `!Cursor`. + +- For every Python type passed as query argument, the `!Transformer` will + instantiate a `!Dumper`. Usually all the objects of the same type will be + converted by the same dumper instance. + + - According to the placeholder used (``%s``, ``%b``, ``%t``), Psycopg may + pick a binary or a text dumper. When using the ``%s`` "`~PyFormat.AUTO`" + format, if the same type has both a text and a binary dumper registered, + the last one registered by `~AdaptersMap.register_dumper()` will be used. + + - Sometimes, just looking at the Python type is not enough to decide the + best PostgreSQL type to use (for instance the PostgreSQL type of a Python + list depends on the objects it contains, whether to use an :sql:`integer` + or :sql:`bigint` depends on the number size...) In these cases the + mechanism provided by `~psycopg.abc.Dumper.get_key()` and + `~psycopg.abc.Dumper.upgrade()` is used to create more specific dumpers. + +- The query is executed. Upon successful request, the result is received as a + `~psycopg.pq.PGresult`. + +- For every OID returned by the query, the `!Transformer` will instantiate a + `!Loader`. All the values with the same OID will be converted by the same + loader instance. + +- Recursive types (e.g. Python lists, PostgreSQL arrays and composite types) + will use the same adaptation rules. + +As a consequence it is possible to perform certain choices only once per query +(e.g. looking up the connection encoding) and then call a fast-path operation +for each value to convert. + +Querying will fail if a Python object for which there isn't a `!Dumper` +registered (for the right `~psycopg.pq.Format`) is used as query parameter. +If the query returns a data type whose OID doesn't have a `!Loader`, the +value will be returned as a string (or bytes string for binary types). diff --git a/docs/advanced/async.rst b/docs/advanced/async.rst new file mode 100644 index 0000000..3620ab6 --- /dev/null +++ b/docs/advanced/async.rst @@ -0,0 +1,360 @@ +.. currentmodule:: psycopg + +.. index:: asyncio + +.. _async: + +Asynchronous operations +======================= + +Psycopg `~Connection` and `~Cursor` have counterparts `~AsyncConnection` and +`~AsyncCursor` supporting an `asyncio` interface. + +The design of the asynchronous objects is pretty much the same of the sync +ones: in order to use them you will only have to scatter the `!await` keyword +here and there. + +.. code:: python + + async with await psycopg.AsyncConnection.connect( + "dbname=test user=postgres") as aconn: + async with aconn.cursor() as acur: + await acur.execute( + "INSERT INTO test (num, data) VALUES (%s, %s)", + (100, "abc'def")) + await acur.execute("SELECT * FROM test") + await acur.fetchone() + # will return (1, 100, "abc'def") + async for record in acur: + print(record) + +.. versionchanged:: 3.1 + + `AsyncConnection.connect()` performs DNS name resolution in a non-blocking + way. + + .. warning:: + + Before version 3.1, `AsyncConnection.connect()` may still block on DNS + name resolution. To avoid that you should `set the hostaddr connection + parameter`__, or use the `~psycopg._dns.resolve_hostaddr_async()` to + do it automatically. + + .. __: https://www.postgresql.org/docs/current/libpq-connect.html + #LIBPQ-PARAMKEYWORDS + +.. warning:: + + On Windows, Psycopg is not compatible with the default + `~asyncio.ProactorEventLoop`. Please use a different loop, for instance + the `~asyncio.SelectorEventLoop`. + + For instance, you can use, early in your program: + + .. parsed-literal:: + + `asyncio.set_event_loop_policy`\ ( + `asyncio.WindowsSelectorEventLoopPolicy`\ () + ) + + + +.. index:: with + +.. _async-with: + +`!with` async connections +------------------------- + +As seen in :ref:`the basic usage <usage>`, connections and cursors can act as +context managers, so you can run: + +.. code:: python + + with psycopg.connect("dbname=test user=postgres") as conn: + with conn.cursor() as cur: + cur.execute(...) + # the cursor is closed upon leaving the context + # the transaction is committed, the connection closed + +For asynchronous connections it's *almost* what you'd expect, but +not quite. Please note that `~Connection.connect()` and `~Connection.cursor()` +*don't return a context*: they are both factory methods which return *an +object which can be used as a context*. That's because there are several use +cases where it's useful to handle the objects manually and only `!close()` them +when required. + +As a consequence you cannot use `!async with connect()`: you have to do it in +two steps instead, as in + +.. code:: python + + aconn = await psycopg.AsyncConnection.connect() + async with aconn: + async with aconn.cursor() as cur: + await cur.execute(...) + +which can be condensed into `!async with await`: + +.. code:: python + + async with await psycopg.AsyncConnection.connect() as aconn: + async with aconn.cursor() as cur: + await cur.execute(...) + +...but no less than that: you still need to do the double async thing. + +Note that the `AsyncConnection.cursor()` function is not an `!async` function +(it never performs I/O), so you don't need an `!await` on it; as a consequence +you can use the normal `async with` context manager. + + +.. index:: Ctrl-C + +.. _async-ctrl-c: + +Interrupting async operations using Ctrl-C +------------------------------------------ + +If a long running operation is interrupted by a Ctrl-C on a normal connection +running in the main thread, the operation will be cancelled and the connection +will be put in error state, from which can be recovered with a normal +`~Connection.rollback()`. + +If the query is running in an async connection, a Ctrl-C will be likely +intercepted by the async loop and interrupt the whole program. In order to +emulate what normally happens with blocking connections, you can use +`asyncio's add_signal_handler()`__, to call `Connection.cancel()`: + +.. code:: python + + import asyncio + import signal + + async with await psycopg.AsyncConnection.connect() as conn: + loop.add_signal_handler(signal.SIGINT, conn.cancel) + ... + + +.. __: https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.add_signal_handler + + +.. index:: + pair: Asynchronous; Notifications + pair: LISTEN; SQL command + pair: NOTIFY; SQL command + +.. _async-messages: + +Server messages +--------------- + +PostgreSQL can send, together with the query results, `informative messages`__ +about the operation just performed, such as warnings or debug information. +Notices may be raised even if the operations are successful and don't indicate +an error. You are probably familiar with some of them, because they are +reported by :program:`psql`:: + + $ psql + =# ROLLBACK; + WARNING: there is no transaction in progress + ROLLBACK + +.. __: https://www.postgresql.org/docs/current/runtime-config-logging.html + #RUNTIME-CONFIG-SEVERITY-LEVELS + +Messages can be also sent by the `PL/pgSQL 'RAISE' statement`__ (at a level +lower than EXCEPTION, otherwise the appropriate `DatabaseError` will be +raised). The level of the messages received can be controlled using the +client_min_messages__ setting. + +.. __: https://www.postgresql.org/docs/current/plpgsql-errors-and-messages.html +.. __: https://www.postgresql.org/docs/current/runtime-config-client.html + #GUC-CLIENT-MIN-MESSAGES + + +By default, the messages received are ignored. If you want to process them on +the client you can use the `Connection.add_notice_handler()` function to +register a function that will be invoked whenever a message is received. The +message is passed to the callback as a `~errors.Diagnostic` instance, +containing all the information passed by the server, such as the message text +and the severity. The object is the same found on the `~psycopg.Error.diag` +attribute of the errors raised by the server: + +.. code:: python + + >>> import psycopg + + >>> def log_notice(diag): + ... print(f"The server says: {diag.severity} - {diag.message_primary}") + + >>> conn = psycopg.connect(autocommit=True) + >>> conn.add_notice_handler(log_notice) + + >>> cur = conn.execute("ROLLBACK") + The server says: WARNING - there is no transaction in progress + >>> print(cur.statusmessage) + ROLLBACK + +.. warning:: + + The `!Diagnostic` object received by the callback should not be used after + the callback function terminates, because its data is deallocated after + the callbacks have been processed. If you need to use the information + later please extract the attributes requested and forward them instead of + forwarding the whole `!Diagnostic` object. + + +.. index:: + pair: Asynchronous; Notifications + pair: LISTEN; SQL command + pair: NOTIFY; SQL command + +.. _async-notify: + +Asynchronous notifications +-------------------------- + +Psycopg allows asynchronous interaction with other database sessions using the +facilities offered by PostgreSQL commands |LISTEN|_ and |NOTIFY|_. Please +refer to the PostgreSQL documentation for examples about how to use this form +of communication. + +.. |LISTEN| replace:: :sql:`LISTEN` +.. _LISTEN: https://www.postgresql.org/docs/current/sql-listen.html +.. |NOTIFY| replace:: :sql:`NOTIFY` +.. _NOTIFY: https://www.postgresql.org/docs/current/sql-notify.html + +Because of the way sessions interact with notifications (see |NOTIFY|_ +documentation), you should keep the connection in `~Connection.autocommit` +mode if you wish to receive or send notifications in a timely manner. + +Notifications are received as instances of `Notify`. If you are reserving a +connection only to receive notifications, the simplest way is to consume the +`Connection.notifies` generator. The generator can be stopped using +`!close()`. + +.. note:: + + You don't need an `AsyncConnection` to handle notifications: a normal + blocking `Connection` is perfectly valid. + +The following example will print notifications and stop when one containing +the ``"stop"`` message is received. + +.. code:: python + + import psycopg + conn = psycopg.connect("", autocommit=True) + conn.execute("LISTEN mychan") + gen = conn.notifies() + for notify in gen: + print(notify) + if notify.payload == "stop": + gen.close() + print("there, I stopped") + +If you run some :sql:`NOTIFY` in a :program:`psql` session: + +.. code:: psql + + =# NOTIFY mychan, 'hello'; + NOTIFY + =# NOTIFY mychan, 'hey'; + NOTIFY + =# NOTIFY mychan, 'stop'; + NOTIFY + +You may get output from the Python process such as:: + + Notify(channel='mychan', payload='hello', pid=961823) + Notify(channel='mychan', payload='hey', pid=961823) + Notify(channel='mychan', payload='stop', pid=961823) + there, I stopped + +Alternatively, you can use `~Connection.add_notify_handler()` to register a +callback function, which will be invoked whenever a notification is received, +during the normal query processing; you will be then able to use the +connection normally. Please note that in this case notifications will not be +received immediately, but only during a connection operation, such as a query. + +.. code:: python + + conn.add_notify_handler(lambda n: print(f"got this: {n}")) + + # meanwhile in psql... + # =# NOTIFY mychan, 'hey'; + # NOTIFY + + print(conn.execute("SELECT 1").fetchone()) + # got this: Notify(channel='mychan', payload='hey', pid=961823) + # (1,) + + +.. index:: disconnections + +.. _disconnections: + +Detecting disconnections +------------------------ + +Sometimes it is useful to detect immediately when the connection with the +database is lost. One brutal way to do so is to poll a connection in a loop +running an endless stream of :sql:`SELECT 1`... *Don't* do so: polling is *so* +out of fashion. Besides, it is inefficient (unless what you really want is a +client-server generator of ones), it generates useless traffic and will only +detect a disconnection with an average delay of half the polling time. + +A more efficient and timely way to detect a server disconnection is to create +an additional connection and wait for a notification from the OS that this +connection has something to say: only then you can run some checks. You +can dedicate a thread (or an asyncio task) to wait on this connection: such +thread will perform no activity until awaken by the OS. + +In a normal (non asyncio) program you can use the `selectors` module. Because +the `!Connection` implements a `~Connection.fileno()` method you can just +register it as a file-like object. You can run such code in a dedicated thread +(and using a dedicated connection) if the rest of the program happens to have +something else to do too. + +.. code:: python + + import selectors + + sel = selectors.DefaultSelector() + sel.register(conn, selectors.EVENT_READ) + while True: + if not sel.select(timeout=60.0): + continue # No FD activity detected in one minute + + # Activity detected. Is the connection still ok? + try: + conn.execute("SELECT 1") + except psycopg.OperationalError: + # You were disconnected: do something useful such as panicking + logger.error("we lost our database!") + sys.exit(1) + +In an `asyncio` program you can dedicate a `~asyncio.Task` instead and do +something similar using `~asyncio.loop.add_reader`: + +.. code:: python + + import asyncio + + ev = asyncio.Event() + loop = asyncio.get_event_loop() + loop.add_reader(conn.fileno(), ev.set) + + while True: + try: + await asyncio.wait_for(ev.wait(), 60.0) + except asyncio.TimeoutError: + continue # No FD activity detected in one minute + + # Activity detected. Is the connection still ok? + try: + await conn.execute("SELECT 1") + except psycopg.OperationalError: + # Guess what happened + ... diff --git a/docs/advanced/cursors.rst b/docs/advanced/cursors.rst new file mode 100644 index 0000000..954d665 --- /dev/null +++ b/docs/advanced/cursors.rst @@ -0,0 +1,192 @@ +.. currentmodule:: psycopg + +.. index:: + single: Cursor + +.. _cursor-types: + +Cursor types +============ + +Psycopg can manage kinds of "cursors" which differ in where the state of a +query being processed is stored: :ref:`client-side-cursors` and +:ref:`server-side-cursors`. + +.. index:: + double: Cursor; Client-side + +.. _client-side-cursors: + +Client-side cursors +------------------- + +Client-side cursors are what Psycopg uses in its normal querying process. +They are implemented by the `Cursor` and `AsyncCursor` classes. In such +querying pattern, after a cursor sends a query to the server (usually calling +`~Cursor.execute()`), the server replies transferring to the client the whole +set of results requested, which is stored in the state of the same cursor and +from where it can be read from Python code (using methods such as +`~Cursor.fetchone()` and siblings). + +This querying process is very scalable because, after a query result has been +transmitted to the client, the server doesn't keep any state. Because the +results are already in the client memory, iterating its rows is very quick. + +The downside of this querying method is that the entire result has to be +transmitted completely to the client (with a time proportional to its size) +and the client needs enough memory to hold it, so it is only suitable for +reasonably small result sets. + + +.. index:: + double: Cursor; Client-binding + +.. _client-side-binding-cursors: + +Client-side-binding cursors +--------------------------- + +.. versionadded:: 3.1 + +The previously described :ref:`client-side cursors <client-side-cursors>` send +the query and the parameters separately to the server. This is the most +efficient way to process parametrised queries and allows to build several +features and optimizations. However, not all types of queries can be bound +server-side; in particular no Data Definition Language query can. See +:ref:`server-side-binding` for the description of these problems. + +The `ClientCursor` (and its `AsyncClientCursor` async counterpart) merge the +query on the client and send the query and the parameters merged together to +the server. This allows to parametrize any type of PostgreSQL statement, not +only queries (:sql:`SELECT`) and Data Manipulation statements (:sql:`INSERT`, +:sql:`UPDATE`, :sql:`DELETE`). + +Using `!ClientCursor`, Psycopg 3 behaviour will be more similar to `psycopg2` +(which only implements client-side binding) and could be useful to port +Psycopg 2 programs more easily to Psycopg 3. The objects in the `sql` module +allow for greater flexibility (for instance to parametrize a table name too, +not only values); however, for simple cases, a `!ClientCursor` could be the +right object. + +In order to obtain `!ClientCursor` from a connection, you can set its +`~Connection.cursor_factory` (at init time or changing its attribute +afterwards): + +.. code:: python + + from psycopg import connect, ClientCursor + + conn = psycopg.connect(DSN, cursor_factory=ClientCursor) + cur = conn.cursor() + # <psycopg.ClientCursor [no result] [IDLE] (database=piro) at 0x7fd977ae2880> + +If you need to create a one-off client-side-binding cursor out of a normal +connection, you can just use the `~ClientCursor` class passing the connection +as argument. + +.. code:: python + + conn = psycopg.connect(DSN) + cur = psycopg.ClientCursor(conn) + +.. warning:: + + Client-side cursors don't support :ref:`binary parameters and return + values <binary-data>` and don't support :ref:`prepared statements + <prepared-statements>`. + +.. tip:: + + The best use for client-side binding cursors is probably to port large + Psycopg 2 code to Psycopg 3, especially for programs making wide use of + Data Definition Language statements. + + The `psycopg.sql` module allows for more generic client-side query + composition, to mix client- and server-side parameters binding, and allows + to parametrize tables and fields names too, or entirely generic SQL + snippets. + +.. index:: + double: Cursor; Server-side + single: Portal + double: Cursor; Named + +.. _server-side-cursors: + +Server-side cursors +------------------- + +PostgreSQL has its own concept of *cursor* too (sometimes also called +*portal*). When a database cursor is created, the query is not necessarily +completely processed: the server might be able to produce results only as they +are needed. Only the results requested are transmitted to the client: if the +query result is very large but the client only needs the first few records it +is possible to transmit only them. + +The downside is that the server needs to keep track of the partially +processed results, so it uses more memory and resources on the server. + +Psycopg allows the use of server-side cursors using the classes `ServerCursor` +and `AsyncServerCursor`. They are usually created by passing the `!name` +parameter to the `~Connection.cursor()` method (reason for which, in +`!psycopg2`, they are usually called *named cursors*). The use of these classes +is similar to their client-side counterparts: their interface is the same, but +behind the scene they send commands to control the state of the cursor on the +server (for instance when fetching new records or when moving using +`~Cursor.scroll()`). + +Using a server-side cursor it is possible to process datasets larger than what +would fit in the client's memory. However for small queries they are less +efficient because it takes more commands to receive their result, so you +should use them only if you need to process huge results or if only a partial +result is needed. + +.. seealso:: + + Server-side cursors are created and managed by `ServerCursor` using SQL + commands such as DECLARE_, FETCH_, MOVE_. The PostgreSQL documentation + gives a good idea of what is possible to do with them. + + .. _DECLARE: https://www.postgresql.org/docs/current/sql-declare.html + .. _FETCH: https://www.postgresql.org/docs/current/sql-fetch.html + .. _MOVE: https://www.postgresql.org/docs/current/sql-move.html + + +.. _cursor-steal: + +"Stealing" an existing cursor +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A Psycopg `ServerCursor` can be also used to consume a cursor which was +created in other ways than the :sql:`DECLARE` that `ServerCursor.execute()` +runs behind the scene. + +For instance if you have a `PL/pgSQL function returning a cursor`__: + +.. __: https://www.postgresql.org/docs/current/plpgsql-cursors.html + +.. code:: postgres + + CREATE FUNCTION reffunc(refcursor) RETURNS refcursor AS $$ + BEGIN + OPEN $1 FOR SELECT col FROM test; + RETURN $1; + END; + $$ LANGUAGE plpgsql; + +you can run a one-off command in the same connection to call it (e.g. using +`Connection.execute()`) in order to create the cursor on the server: + +.. code:: python + + conn.execute("SELECT reffunc('curname')") + +after which you can create a server-side cursor declared by the same name, and +directly call the fetch methods, skipping the `~ServerCursor.execute()` call: + +.. code:: python + + cur = conn.cursor('curname') + # no cur.execute() + for record in cur: # or cur.fetchone(), cur.fetchmany()... + # do something with record diff --git a/docs/advanced/index.rst b/docs/advanced/index.rst new file mode 100644 index 0000000..6920bd7 --- /dev/null +++ b/docs/advanced/index.rst @@ -0,0 +1,21 @@ +.. _advanced: + +More advanced topics +==================== + +Once you have familiarised yourself with the :ref:`Psycopg basic operations +<basic>`, you can take a look at the chapter of this section for more advanced +usages. + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + async + typing + rows + pool + cursors + adapt + prepare + pipeline diff --git a/docs/advanced/pipeline.rst b/docs/advanced/pipeline.rst new file mode 100644 index 0000000..980fea7 --- /dev/null +++ b/docs/advanced/pipeline.rst @@ -0,0 +1,324 @@ +.. currentmodule:: psycopg + +.. _pipeline-mode: + +Pipeline mode support +===================== + +.. versionadded:: 3.1 + +The *pipeline mode* allows PostgreSQL client applications to send a query +without having to read the result of the previously sent query. Taking +advantage of the pipeline mode, a client will wait less for the server, since +multiple queries/results can be sent/received in a single network roundtrip. +Pipeline mode can provide a significant performance boost to the application. + +Pipeline mode is most useful when the server is distant, i.e., network latency +(“ping time”) is high, and also when many small operations are being performed +in rapid succession. There is usually less benefit in using pipelined commands +when each query takes many multiples of the client/server round-trip time to +execute. A 100-statement operation run on a server 300 ms round-trip-time away +would take 30 seconds in network latency alone without pipelining; with +pipelining it may spend as little as 0.3 s waiting for results from the +server. + +The server executes statements, and returns results, in the order the client +sends them. The server will begin executing the commands in the pipeline +immediately, not waiting for the end of the pipeline. Note that results are +buffered on the server side; the server flushes that buffer when a +:ref:`synchronization point <pipeline-sync>` is established. + +.. seealso:: + + The PostgreSQL documentation about: + + - `pipeline mode`__ + - `extended query message flow`__ + + contains many details around when it is most useful to use the pipeline + mode and about errors management and interaction with transactions. + + .. __: https://www.postgresql.org/docs/current/libpq-pipeline-mode.html + .. __: https://www.postgresql.org/docs/current/protocol-flow.html#PROTOCOL-FLOW-EXT-QUERY + + +Client-server messages flow +--------------------------- + +In order to understand better how the pipeline mode works, we should take a +closer look at the `PostgreSQL client-server message flow`__. + +During normal querying, each statement is transmitted by the client to the +server as a stream of request messages, terminating with a **Sync** message to +tell it that it should process the messages sent so far. The server will +execute the statement and describe the results back as a stream of messages, +terminating with a **ReadyForQuery**, telling the client that it may now send a +new query. + +For example, the statement (returning no result): + +.. code:: python + + conn.execute("INSERT INTO mytable (data) VALUES (%s)", ["hello"]) + +results in the following two groups of messages: + +.. table:: + :align: left + + +---------------+-----------------------------------------------------------+ + | Direction | Message | + +===============+===========================================================+ + | Python | - Parse ``INSERT INTO ... (VALUE $1)`` (skipped if | + | | :ref:`the statement is prepared <prepared-statements>`) | + | |>| | - Bind ``'hello'`` | + | | - Describe | + | PostgreSQL | - Execute | + | | - Sync | + +---------------+-----------------------------------------------------------+ + | PostgreSQL | - ParseComplete | + | | - BindComplete | + | |<| | - NoData | + | | - CommandComplete ``INSERT 0 1`` | + | Python | - ReadyForQuery | + +---------------+-----------------------------------------------------------+ + +and the query: + +.. code:: python + + conn.execute("SELECT data FROM mytable WHERE id = %s", [1]) + +results in the two groups of messages: + +.. table:: + :align: left + + +---------------+-----------------------------------------------------------+ + | Direction | Message | + +===============+===========================================================+ + | Python | - Parse ``SELECT data FROM mytable WHERE id = $1`` | + | | - Bind ``1`` | + | |>| | - Describe | + | | - Execute | + | PostgreSQL | - Sync | + +---------------+-----------------------------------------------------------+ + | PostgreSQL | - ParseComplete | + | | - BindComplete | + | |<| | - RowDescription ``data`` | + | | - DataRow ``hello`` | + | Python | - CommandComplete ``SELECT 1`` | + | | - ReadyForQuery | + +---------------+-----------------------------------------------------------+ + +The two statements, sent consecutively, pay the communication overhead four +times, once per leg. + +The pipeline mode allows the client to combine several operations in longer +streams of messages to the server, then to receive more than one response in a +single batch. If we execute the two operations above in a pipeline: + +.. code:: python + + with conn.pipeline(): + conn.execute("INSERT INTO mytable (data) VALUES (%s)", ["hello"]) + conn.execute("SELECT data FROM mytable WHERE id = %s", [1]) + +they will result in a single roundtrip between the client and the server: + +.. table:: + :align: left + + +---------------+-----------------------------------------------------------+ + | Direction | Message | + +===============+===========================================================+ + | Python | - Parse ``INSERT INTO ... (VALUE $1)`` | + | | - Bind ``'hello'`` | + | |>| | - Describe | + | | - Execute | + | PostgreSQL | - Parse ``SELECT data FROM mytable WHERE id = $1`` | + | | - Bind ``1`` | + | | - Describe | + | | - Execute | + | | - Sync (sent only once) | + +---------------+-----------------------------------------------------------+ + | PostgreSQL | - ParseComplete | + | | - BindComplete | + | |<| | - NoData | + | | - CommandComplete ``INSERT 0 1`` | + | Python | - ParseComplete | + | | - BindComplete | + | | - RowDescription ``data`` | + | | - DataRow ``hello`` | + | | - CommandComplete ``SELECT 1`` | + | | - ReadyForQuery (sent only once) | + +---------------+-----------------------------------------------------------+ + +.. |<| unicode:: U+25C0 +.. |>| unicode:: U+25B6 + +.. __: https://www.postgresql.org/docs/current/protocol-flow.html + + +.. _pipeline-usage: + +Pipeline mode usage +------------------- + +Psycopg supports the pipeline mode via the `Connection.pipeline()` method. The +method is a context manager: entering the ``with`` block yields a `Pipeline` +object. At the end of block, the connection resumes the normal operation mode. + +Within the pipeline block, you can use normally one or more cursors to execute +several operations, using `Connection.execute()`, `Cursor.execute()` and +`~Cursor.executemany()`. + +.. code:: python + + >>> with conn.pipeline(): + ... conn.execute("INSERT INTO mytable VALUES (%s)", ["hello"]) + ... with conn.cursor() as cur: + ... cur.execute("INSERT INTO othertable VALUES (%s)", ["world"]) + ... cur.executemany( + ... "INSERT INTO elsewhere VALUES (%s)", + ... [("one",), ("two",), ("four",)]) + +Unlike in normal mode, Psycopg will not wait for the server to receive the +result of each query; the client will receive results in batches when the +server flushes it output buffer. + +When a flush (or a sync) is performed, all pending results are sent back to +the cursors which executed them. If a cursor had run more than one query, it +will receive more than one result; results after the first will be available, +in their execution order, using `~Cursor.nextset()`: + +.. code:: python + + >>> with conn.pipeline(): + ... with conn.cursor() as cur: + ... cur.execute("INSERT INTO mytable (data) VALUES (%s) RETURNING *", ["hello"]) + ... cur.execute("INSERT INTO mytable (data) VALUES (%s) RETURNING *", ["world"]) + ... while True: + ... print(cur.fetchall()) + ... if not cur.nextset(): + ... break + + [(1, 'hello')] + [(2, 'world')] + +If any statement encounters an error, the server aborts the current +transaction and will not execute any subsequent command in the queue until the +next :ref:`synchronization point <pipeline-sync>`; a `~errors.PipelineAborted` +exception is raised for each such command. Query processing resumes after the +synchronization point. + +.. warning:: + + Certain features are not available in pipeline mode, including: + + - COPY is not supported in pipeline mode by PostgreSQL. + - `Cursor.stream()` doesn't make sense in pipeline mode (its job is the + opposite of batching!) + - `ServerCursor` are currently not implemented in pipeline mode. + +.. note:: + + Starting from Psycopg 3.1, `~Cursor.executemany()` makes use internally of + the pipeline mode; as a consequence there is no need to handle a pipeline + block just to call `!executemany()` once. + + +.. _pipeline-sync: + +Synchronization points +---------------------- + +Flushing query results to the client can happen either when a synchronization +point is established by Psycopg: + +- using the `Pipeline.sync()` method; +- on `Connection.commit()` or `~Connection.rollback()`; +- at the end of a `!Pipeline` block; +- possibly when opening a nested `!Pipeline` block; +- using a fetch method such as `Cursor.fetchone()` (which only flushes the + query but doesn't issue a Sync and doesn't reset a pipeline state error). + +The server might perform a flush on its own initiative, for instance when the +output buffer is full. + +Note that, even in :ref:`autocommit <autocommit>`, the server wraps the +statements sent in pipeline mode in an implicit transaction, which will be +only committed when the Sync is received. As such, a failure in a group of +statements will probably invalidate the effect of statements executed after +the previous Sync, and will propagate to the following Sync. + +For example, in the following block: + +.. code:: python + + >>> with psycopg.connect(autocommit=True) as conn: + ... with conn.pipeline() as p, conn.cursor() as cur: + ... try: + ... cur.execute("INSERT INTO mytable (data) VALUES (%s)", ["one"]) + ... cur.execute("INSERT INTO no_such_table (data) VALUES (%s)", ["two"]) + ... conn.execute("INSERT INTO mytable (data) VALUES (%s)", ["three"]) + ... p.sync() + ... except psycopg.errors.UndefinedTable: + ... pass + ... cur.execute("INSERT INTO mytable (data) VALUES (%s)", ["four"]) + +there will be an error in the block, ``relation "no_such_table" does not +exist`` caused by the insert ``two``, but probably raised by the `!sync()` +call. At at the end of the block, the table will contain: + +.. code:: text + + =# SELECT * FROM mytable; + +----+------+ + | id | data | + +----+------+ + | 2 | four | + +----+------+ + (1 row) + +because: + +- the value 1 of the sequence is consumed by the statement ``one``, but + the record discarded because of the error in the same implicit transaction; +- the statement ``three`` is not executed because the pipeline is aborted (so + it doesn't consume a sequence item); +- the statement ``four`` is executed with + success after the Sync has terminated the failed transaction. + +.. warning:: + + The exact Python statement where an exception caused by a server error is + raised is somewhat arbitrary: it depends on when the server flushes its + buffered result. + + If you want to make sure that a group of statements is applied atomically + by the server, do make use of transaction methods such as + `~Connection.commit()` or `~Connection.transaction()`: these methods will + also sync the pipeline and raise an exception if there was any error in + the commands executed so far. + + +The fine prints +--------------- + +.. warning:: + + The Pipeline mode is an experimental feature. + + Its behaviour, especially around error conditions and concurrency, hasn't + been explored as much as the normal request-response messages pattern, and + its async nature makes it inherently more complex. + + As we gain more experience and feedback (which is welcome), we might find + bugs and shortcomings forcing us to change the current interface or + behaviour. + +The pipeline mode is available on any currently supported PostgreSQL version, +but, in order to make use of it, the client must use a libpq from PostgreSQL +14 or higher. You can use `Pipeline.is_supported()` to make sure your client +has the right library. diff --git a/docs/advanced/pool.rst b/docs/advanced/pool.rst new file mode 100644 index 0000000..adea0a7 --- /dev/null +++ b/docs/advanced/pool.rst @@ -0,0 +1,332 @@ +.. currentmodule:: psycopg_pool + +.. _connection-pools: + +Connection pools +================ + +A `connection pool`__ is an object managing a set of connections and allowing +their use in functions needing one. Because the time to establish a new +connection can be relatively long, keeping connections open can reduce latency. + +.. __: https://en.wikipedia.org/wiki/Connection_pool + +This page explains a few basic concepts of Psycopg connection pool's +behaviour. Please refer to the `ConnectionPool` object API for details about +the pool operations. + +.. note:: The connection pool objects are distributed in a package separate + from the main `psycopg` package: use ``pip install "psycopg[pool]"`` or ``pip + install psycopg_pool`` to make the `psycopg_pool` package available. See + :ref:`pool-installation`. + + +Pool life cycle +--------------- + +A simple way to use the pool is to create a single instance of it, as a +global object, and to use this object in the rest of the program, allowing +other functions, modules, threads to use it:: + + # module db.py in your program + from psycopg_pool import ConnectionPool + + pool = ConnectionPool(conninfo, **kwargs) + # the pool starts connecting immediately. + + # in another module + from .db import pool + + def my_function(): + with pool.connection() as conn: + conn.execute(...) + +Ideally you may want to call `~ConnectionPool.close()` when the use of the +pool is finished. Failing to call `!close()` at the end of the program is not +terribly bad: probably it will just result in some warnings printed on stderr. +However, if you think that it's sloppy, you could use the `atexit` module to +have `!close()` called at the end of the program. + +If you want to avoid starting to connect to the database at import time, and +want to wait for the application to be ready, you can create the pool using +`!open=False`, and call the `~ConnectionPool.open()` and +`~ConnectionPool.close()` methods when the conditions are right. Certain +frameworks provide callbacks triggered when the program is started and stopped +(for instance `FastAPI startup/shutdown events`__): they are perfect to +initiate and terminate the pool operations:: + + pool = ConnectionPool(conninfo, open=False, **kwargs) + + @app.on_event("startup") + def open_pool(): + pool.open() + + @app.on_event("shutdown") + def close_pool(): + pool.close() + +.. __: https://fastapi.tiangolo.com/advanced/events/#events-startup-shutdown + +Creating a single pool as a global variable is not the mandatory use: your +program can create more than one pool, which might be useful to connect to +more than one database, or to provide different types of connections, for +instance to provide separate read/write and read-only connections. The pool +also acts as a context manager and is open and closed, if necessary, on +entering and exiting the context block:: + + from psycopg_pool import ConnectionPool + + with ConnectionPool(conninfo, **kwargs) as pool: + run_app(pool) + + # the pool is now closed + +When the pool is open, the pool's background workers start creating the +requested `!min_size` connections, while the constructor (or the `!open()` +method) returns immediately. This allows the program some leeway to start +before the target database is up and running. However, if your application is +misconfigured, or the network is down, it means that the program will be able +to start, but the threads requesting a connection will fail with a +`PoolTimeout` only after the timeout on `~ConnectionPool.connection()` is +expired. If this behaviour is not desirable (and you prefer your program to +crash hard and fast, if the surrounding conditions are not right, because +something else will respawn it) you should call the `~ConnectionPool.wait()` +method after creating the pool, or call `!open(wait=True)`: these methods will +block until the pool is full, or will raise a `PoolTimeout` exception if the +pool isn't ready within the allocated time. + + +Connections life cycle +---------------------- + +The pool background workers create connections according to the parameters +`!conninfo`, `!kwargs`, and `!connection_class` passed to `ConnectionPool` +constructor, invoking something like :samp:`{connection_class}({conninfo}, +**{kwargs})`. Once a connection is created it is also passed to the +`!configure()` callback, if provided, after which it is put in the pool (or +passed to a client requesting it, if someone is already knocking at the door). + +If a connection expires (it passes `!max_lifetime`), or is returned to the pool +in broken state, or is found closed by `~ConnectionPool.check()`), then the +pool will dispose of it and will start a new connection attempt in the +background. + + +Using connections from the pool +------------------------------- + +The pool can be used to request connections from multiple threads or +concurrent tasks - it is hardly useful otherwise! If more connections than the +ones available in the pool are requested, the requesting threads are queued +and are served a connection as soon as one is available, either because +another client has finished using it or because the pool is allowed to grow +(when `!max_size` > `!min_size`) and a new connection is ready. + +The main way to use the pool is to obtain a connection using the +`~ConnectionPool.connection()` context, which returns a `~psycopg.Connection` +or subclass:: + + with my_pool.connection() as conn: + conn.execute("what you want") + +The `!connection()` context behaves like the `~psycopg.Connection` object +context: at the end of the block, if there is a transaction open, it will be +committed, or rolled back if the context is exited with as exception. + +At the end of the block the connection is returned to the pool and shouldn't +be used anymore by the code which obtained it. If a `!reset()` function is +specified in the pool constructor, it is called on the connection before +returning it to the pool. Note that the `!reset()` function is called in a +worker thread, so that the thread which used the connection can keep its +execution without being slowed down by it. + + +Pool connection and sizing +-------------------------- + +A pool can have a fixed size (specifying no `!max_size` or `!max_size` = +`!min_size`) or a dynamic size (when `!max_size` > `!min_size`). In both +cases, as soon as the pool is created, it will try to acquire `!min_size` +connections in the background. + +If an attempt to create a connection fails, a new attempt will be made soon +after, using an exponential backoff to increase the time between attempts, +until a maximum of `!reconnect_timeout` is reached. When that happens, the pool +will call the `!reconnect_failed()` function, if provided to the pool, and just +start a new connection attempt. You can use this function either to send +alerts or to interrupt the program and allow the rest of your infrastructure +to restart it. + +If more than `!min_size` connections are requested concurrently, new ones are +created, up to `!max_size`. Note that the connections are always created by the +background workers, not by the thread asking for the connection: if a client +requests a new connection, and a previous client terminates its job before the +new connection is ready, the waiting client will be served the existing +connection. This is especially useful in scenarios where the time to establish +a connection dominates the time for which the connection is used (see `this +analysis`__, for instance). + +.. __: https://github.com/brettwooldridge/HikariCP/blob/dev/documents/ + Welcome-To-The-Jungle.md + +If a pool grows above `!min_size`, but its usage decreases afterwards, a number +of connections are eventually closed: one every time a connection is unused +after the `!max_idle` time specified in the pool constructor. + + +What's the right size for the pool? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Big question. Who knows. However, probably not as large as you imagine. Please +take a look at `this analysis`__ for some ideas. + +.. __: https://github.com/brettwooldridge/HikariCP/wiki/About-Pool-Sizing + +Something useful you can do is probably to use the +`~ConnectionPool.get_stats()` method and monitor the behaviour of your program +to tune the configuration parameters. The size of the pool can also be changed +at runtime using the `~ConnectionPool.resize()` method. + + +.. _null-pool: + +Null connection pools +--------------------- + +.. versionadded:: 3.1 + +Sometimes you may want leave the choice of using or not using a connection +pool as a configuration parameter of your application. For instance, you might +want to use a pool if you are deploying a "large instance" of your application +and can dedicate it a handful of connections; conversely you might not want to +use it if you deploy the application in several instances, behind a load +balancer, and/or using an external connection pool process such as PgBouncer. + +Switching between using or not using a pool requires some code change, because +the `ConnectionPool` API is different from the normal `~psycopg.connect()` +function and because the pool can perform additional connection configuration +(in the `!configure` parameter) that, if the pool is removed, should be +performed in some different code path of your application. + +The `!psycopg_pool` 3.1 package introduces the `NullConnectionPool` class. +This class has the same interface, and largely the same behaviour, of the +`!ConnectionPool`, but doesn't create any connection beforehand. When a +connection is returned, unless there are other clients already waiting, it +is closed immediately and not kept in the pool state. + +A null pool is not only a configuration convenience, but can also be used to +regulate the access to the server by a client program. If `!max_size` is set to +a value greater than 0, the pool will make sure that no more than `!max_size` +connections are created at any given time. If more clients ask for further +connections, they will be queued and served a connection as soon as a previous +client has finished using it, like for the basic pool. Other mechanisms to +throttle client requests (such as `!timeout` or `!max_waiting`) are respected +too. + +.. note:: + + Queued clients will be handed an already established connection, as soon + as a previous client has finished using it (and after the pool has + returned it to idle state and called `!reset()` on it, if necessary). + +Because normally (i.e. unless queued) every client will be served a new +connection, the time to obtain the connection is paid by the waiting client; +background workers are not normally involved in obtaining new connections. + + +Connection quality +------------------ + +The state of the connection is verified when a connection is returned to the +pool: if a connection is broken during its usage it will be discarded on +return and a new connection will be created. + +.. warning:: + + The health of the connection is not checked when the pool gives it to a + client. + +Why not? Because doing so would require an extra network roundtrip: we want to +save you from its latency. Before getting too angry about it, just think that +the connection can be lost any moment while your program is using it. As your +program should already be able to cope with a loss of a connection during its +process, it should be able to tolerate to be served a broken connection: +unpleasant but not the end of the world. + +.. warning:: + + The health of the connection is not checked when the connection is in the + pool. + +Does the pool keep a watchful eye on the quality of the connections inside it? +No, it doesn't. Why not? Because you will do it for us! Your program is only +a big ruse to make sure the connections are still alive... + +Not (entirely) trolling: if you are using a connection pool, we assume that +you are using and returning connections at a good pace. If the pool had to +check for the quality of a broken connection before your program notices it, +it should be polling each connection even faster than your program uses them. +Your database server wouldn't be amused... + +Can you do something better than that? Of course you can, there is always a +better way than polling. You can use the same recipe of :ref:`disconnections`, +reserving a connection and using a thread to monitor for any activity +happening on it. If any activity is detected, you can call the pool +`~ConnectionPool.check()` method, which will run a quick check on each +connection in the pool, removing the ones found in broken state, and using the +background workers to replace them with fresh ones. + +If you set up a similar check in your program, in case the database connection +is temporarily lost, we cannot do anything for the threads which had taken +already a connection from the pool, but no other thread should be served a +broken connection, because `!check()` would empty the pool and refill it with +working connections, as soon as they are available. + +Faster than you can say poll. Or pool. + + +.. _pool-stats: + +Pool stats +---------- + +The pool can return information about its usage using the methods +`~ConnectionPool.get_stats()` or `~ConnectionPool.pop_stats()`. Both methods +return the same values, but the latter reset the counters after its use. The +values can be sent to a monitoring system such as Graphite_ or Prometheus_. + +.. _Graphite: https://graphiteapp.org/ +.. _Prometheus: https://prometheus.io/ + +The following values should be provided, but please don't consider them as a +rigid interface: it is possible that they might change in the future. Keys +whose value is 0 may not be returned. + + +======================= ===================================================== +Metric Meaning +======================= ===================================================== + ``pool_min`` Current value for `~ConnectionPool.min_size` + ``pool_max`` Current value for `~ConnectionPool.max_size` + ``pool_size`` Number of connections currently managed by the pool + (in the pool, given to clients, being prepared) + ``pool_available`` Number of connections currently idle in the pool + ``requests_waiting`` Number of requests currently waiting in a queue to + receive a connection + ``usage_ms`` Total usage time of the connections outside the pool + ``requests_num`` Number of connections requested to the pool + ``requests_queued`` Number of requests queued because a connection wasn't + immediately available in the pool + ``requests_wait_ms`` Total time in the queue for the clients waiting + ``requests_errors`` Number of connection requests resulting in an error + (timeouts, queue full...) + ``returns_bad`` Number of connections returned to the pool in a bad + state + ``connections_num`` Number of connection attempts made by the pool to the + server + ``connections_ms`` Total time spent to establish connections with the + server + ``connections_errors`` Number of failed connection attempts + ``connections_lost`` Number of connections lost identified by + `~ConnectionPool.check()` +======================= ===================================================== diff --git a/docs/advanced/prepare.rst b/docs/advanced/prepare.rst new file mode 100644 index 0000000..e41bcae --- /dev/null +++ b/docs/advanced/prepare.rst @@ -0,0 +1,57 @@ +.. currentmodule:: psycopg + +.. index:: + single: Prepared statements + +.. _prepared-statements: + +Prepared statements +=================== + +Psycopg uses an automatic system to manage *prepared statements*. When a +query is prepared, its parsing and planning is stored in the server session, +so that further executions of the same query on the same connection (even with +different parameters) are optimised. + +A query is prepared automatically after it is executed more than +`~Connection.prepare_threshold` times on a connection. `!psycopg` will make +sure that no more than `~Connection.prepared_max` statements are planned: if +further queries are executed, the least recently used ones are deallocated and +the associated resources freed. + +Statement preparation can be controlled in several ways: + +- You can decide to prepare a query immediately by passing `!prepare=True` to + `Connection.execute()` or `Cursor.execute()`. The query is prepared, if it + wasn't already, and executed as prepared from its first use. + +- Conversely, passing `!prepare=False` to `!execute()` will avoid to prepare + the query, regardless of the number of times it is executed. The default for + the parameter is `!None`, meaning that the query is prepared if the + conditions described above are met. + +- You can disable the use of prepared statements on a connection by setting + its `~Connection.prepare_threshold` attribute to `!None`. + +.. versionchanged:: 3.1 + You can set `!prepare_threshold` as a `~Connection.connect()` keyword + parameter too. + +.. seealso:: + + The `PREPARE`__ PostgreSQL documentation contains plenty of details about + prepared statements in PostgreSQL. + + Note however that Psycopg doesn't use SQL statements such as + :sql:`PREPARE` and :sql:`EXECUTE`, but protocol level commands such as the + ones exposed by :pq:`PQsendPrepare`, :pq:`PQsendQueryPrepared`. + + .. __: https://www.postgresql.org/docs/current/sql-prepare.html + +.. warning:: + + Using external connection poolers, such as PgBouncer, is not compatible + with prepared statements, because the same client connection may change + the server session it refers to. If such middleware is used you should + disable prepared statements, by setting the `Connection.prepare_threshold` + attribute to `!None`. diff --git a/docs/advanced/rows.rst b/docs/advanced/rows.rst new file mode 100644 index 0000000..c23efe5 --- /dev/null +++ b/docs/advanced/rows.rst @@ -0,0 +1,116 @@ +.. currentmodule:: psycopg + +.. index:: row factories + +.. _row-factories: + +Row factories +============= + +Cursor's `fetch*` methods, by default, return the records received from the +database as tuples. This can be changed to better suit the needs of the +programmer by using custom *row factories*. + +The module `psycopg.rows` exposes several row factories ready to be used. For +instance, if you want to return your records as dictionaries, you can use +`~psycopg.rows.dict_row`:: + + >>> from psycopg.rows import dict_row + + >>> conn = psycopg.connect(DSN, row_factory=dict_row) + + >>> conn.execute("select 'John Doe' as name, 33 as age").fetchone() + {'name': 'John Doe', 'age': 33} + +The `!row_factory` parameter is supported by the `~Connection.connect()` +method and the `~Connection.cursor()` method. Later usage of `!row_factory` +overrides a previous one. It is also possible to change the +`Connection.row_factory` or `Cursor.row_factory` attributes to change what +they return:: + + >>> cur = conn.cursor(row_factory=dict_row) + >>> cur.execute("select 'John Doe' as name, 33 as age").fetchone() + {'name': 'John Doe', 'age': 33} + + >>> from psycopg.rows import namedtuple_row + >>> cur.row_factory = namedtuple_row + >>> cur.execute("select 'John Doe' as name, 33 as age").fetchone() + Row(name='John Doe', age=33) + +If you want to return objects of your choice you can use a row factory +*generator*, for instance `~psycopg.rows.class_row` or +`~psycopg.rows.args_row`, or you can :ref:`write your own row factory +<row-factory-create>`:: + + >>> from dataclasses import dataclass + + >>> @dataclass + ... class Person: + ... name: str + ... age: int + ... weight: Optional[int] = None + + >>> from psycopg.rows import class_row + >>> cur = conn.cursor(row_factory=class_row(Person)) + >>> cur.execute("select 'John Doe' as name, 33 as age").fetchone() + Person(name='John Doe', age=33, weight=None) + + +.. index:: + single: Row Maker + single: Row Factory + +.. _row-factory-create: + +Creating new row factories +-------------------------- + +A *row factory* is a callable that accepts a `Cursor` object and returns +another callable, a *row maker*, which takes raw data (as a sequence of +values) and returns the desired object. + +The role of the row factory is to inspect a query result (it is called after a +query is executed and properties such as `~Cursor.description` and +`~Cursor.pgresult` are available on the cursor) and to prepare a callable +which is efficient to call repeatedly (because, for instance, the names of the +columns are extracted, sanitised, and stored in local variables). + +Formally, these objects are represented by the `~psycopg.rows.RowFactory` and +`~psycopg.rows.RowMaker` protocols. + +`~RowFactory` objects can be implemented as a class, for instance: + +.. code:: python + + from typing import Any, Sequence + from psycopg import Cursor + + class DictRowFactory: + def __init__(self, cursor: Cursor[Any]): + self.fields = [c.name for c in cursor.description] + + def __call__(self, values: Sequence[Any]) -> dict[str, Any]: + return dict(zip(self.fields, values)) + +or as a plain function: + +.. code:: python + + def dict_row_factory(cursor: Cursor[Any]) -> RowMaker[dict[str, Any]]: + fields = [c.name for c in cursor.description] + + def make_row(values: Sequence[Any]) -> dict[str, Any]: + return dict(zip(fields, values)) + + return make_row + +These can then be used by specifying a `row_factory` argument in +`Connection.connect()`, `Connection.cursor()`, or by setting the +`Connection.row_factory` attribute. + +.. code:: python + + conn = psycopg.connect(row_factory=DictRowFactory) + cur = conn.execute("SELECT first_name, last_name, age FROM persons") + person = cur.fetchone() + print(f"{person['first_name']} {person['last_name']}") diff --git a/docs/advanced/typing.rst b/docs/advanced/typing.rst new file mode 100644 index 0000000..71b4e41 --- /dev/null +++ b/docs/advanced/typing.rst @@ -0,0 +1,180 @@ +.. currentmodule:: psycopg + +.. _static-typing: + +Static Typing +============= + +Psycopg source code is annotated according to :pep:`0484` type hints and is +checked using the current version of Mypy_ in ``--strict`` mode. + +If your application is checked using Mypy too you can make use of Psycopg +types to validate the correct use of Psycopg objects and of the data returned +by the database. + +.. _Mypy: http://mypy-lang.org/ + + +Generic types +------------- + +Psycopg `Connection` and `Cursor` objects are `~typing.Generic` objects and +support a `!Row` parameter which is the type of the records returned. + +By default methods such as `Cursor.fetchall()` return normal tuples of unknown +size and content. As such, the `connect()` function returns an object of type +`!psycopg.Connection[Tuple[Any, ...]]` and `Connection.cursor()` returns an +object of type `!psycopg.Cursor[Tuple[Any, ...]]`. If you are writing generic +plumbing code it might be practical to use annotations such as +`!Connection[Any]` and `!Cursor[Any]`. + +.. code:: python + + conn = psycopg.connect() # type is psycopg.Connection[Tuple[Any, ...]] + + cur = conn.cursor() # type is psycopg.Cursor[Tuple[Any, ...]] + + rec = cur.fetchone() # type is Optional[Tuple[Any, ...]] + + recs = cur.fetchall() # type is List[Tuple[Any, ...]] + + +.. _row-factory-static: + +Type of rows returned +--------------------- + +If you want to use connections and cursors returning your data as different +types, for instance as dictionaries, you can use the `!row_factory` argument +of the `~Connection.connect()` and the `~Connection.cursor()` method, which +will control what type of record is returned by the fetch methods of the +cursors and annotate the returned objects accordingly. See +:ref:`row-factories` for more details. + +.. code:: python + + dconn = psycopg.connect(row_factory=dict_row) + # dconn type is psycopg.Connection[Dict[str, Any]] + + dcur = conn.cursor(row_factory=dict_row) + dcur = dconn.cursor() + # dcur type is psycopg.Cursor[Dict[str, Any]] in both cases + + drec = dcur.fetchone() + # drec type is Optional[Dict[str, Any]] + + +.. _example-pydantic: + +Example: returning records as Pydantic models +--------------------------------------------- + +Using Pydantic_ it is possible to enforce static typing at runtime. Using a +Pydantic model factory the code can be checked statically using Mypy and +querying the database will raise an exception if the rows returned is not +compatible with the model. + +.. _Pydantic: https://pydantic-docs.helpmanual.io/ + +The following example can be checked with ``mypy --strict`` without reporting +any issue. Pydantic will also raise a runtime error in case the +`!Person` is used with a query that returns incompatible data. + +.. code:: python + + from datetime import date + from typing import Optional + + import psycopg + from psycopg.rows import class_row + from pydantic import BaseModel + + class Person(BaseModel): + id: int + first_name: str + last_name: str + dob: Optional[date] + + def fetch_person(id: int) -> Person: + with psycopg.connect() as conn: + with conn.cursor(row_factory=class_row(Person)) as cur: + cur.execute( + """ + SELECT id, first_name, last_name, dob + FROM (VALUES + (1, 'John', 'Doe', '2000-01-01'::date), + (2, 'Jane', 'White', NULL) + ) AS data (id, first_name, last_name, dob) + WHERE id = %(id)s; + """, + {"id": id}, + ) + obj = cur.fetchone() + + # reveal_type(obj) would return 'Optional[Person]' here + + if not obj: + raise KeyError(f"person {id} not found") + + # reveal_type(obj) would return 'Person' here + + return obj + + for id in [1, 2]: + p = fetch_person(id) + if p.dob: + print(f"{p.first_name} was born in {p.dob.year}") + else: + print(f"Who knows when {p.first_name} was born") + + +.. _literal-string: + +Checking literal strings in queries +----------------------------------- + +The `~Cursor.execute()` method and similar should only receive a literal +string as input, according to :pep:`675`. This means that the query should +come from a literal string in your code, not from an arbitrary string +expression. + +For instance, passing an argument to the query should be done via the second +argument to `!execute()`, not by string composition: + +.. code:: python + + def get_record(conn: psycopg.Connection[Any], id: int) -> Any: + cur = conn.execute("SELECT * FROM my_table WHERE id = %s" % id) # BAD! + return cur.fetchone() + + # the function should be implemented as: + + def get_record(conn: psycopg.Connection[Any], id: int) -> Any: + cur = conn.execute("select * FROM my_table WHERE id = %s", (id,)) + return cur.fetchone() + +If you are composing a query dynamically you should use the `sql.SQL` object +and similar to escape safely table and field names. The parameter of the +`!SQL()` object should be a literal string: + +.. code:: python + + def count_records(conn: psycopg.Connection[Any], table: str) -> int: + query = "SELECT count(*) FROM %s" % table # BAD! + return conn.execute(query).fetchone()[0] + + # the function should be implemented as: + + def count_records(conn: psycopg.Connection[Any], table: str) -> int: + query = sql.SQL("SELECT count(*) FROM {}").format(sql.Identifier(table)) + return conn.execute(query).fetchone()[0] + +At the time of writing, no Python static analyzer implements this check (`mypy +doesn't implement it`__, Pyre_ does, but `doesn't work with psycopg yet`__). +Once the type checkers support will be complete, the above bad statements +should be reported as errors. + +.. __: https://github.com/python/mypy/issues/12554 +.. __: https://github.com/facebook/pyre-check/issues/636 + +.. _Pyre: https://pyre-check.org/ diff --git a/docs/api/abc.rst b/docs/api/abc.rst new file mode 100644 index 0000000..9514e9b --- /dev/null +++ b/docs/api/abc.rst @@ -0,0 +1,75 @@ +`!abc` -- Psycopg abstract classes +================================== + +The module exposes Psycopg definitions which can be used for static type +checking. + +.. module:: psycopg.abc + +.. autoclass:: Dumper(cls, context=None) + + :param cls: The type that will be managed by this dumper. + :type cls: type + :param context: The context where the transformation is performed. If not + specified the conversion might be inaccurate, for instance it will not + be possible to know the connection encoding or the server date format. + :type context: `AdaptContext` or None + + A partial implementation of this protocol (implementing everything except + `dump()`) is available as `psycopg.adapt.Dumper`. + + .. autoattribute:: format + + .. automethod:: dump + + The format returned by dump shouldn't contain quotes or escaped + values. + + .. automethod:: quote + + .. tip:: + + This method will be used by `~psycopg.sql.Literal` to convert a + value client-side. + + This method only makes sense for text dumpers; the result of calling + it on a binary dumper is undefined. It might scratch your car, or burn + your cake. Don't tell me I didn't warn you. + + .. autoattribute:: oid + + If the OID is not specified, PostgreSQL will try to infer the type + from the context, but this may fail in some contexts and may require a + cast (e.g. specifying :samp:`%s::{type}` for its placeholder). + + You can use the `psycopg.adapters`\ ``.``\ + `~psycopg.adapt.AdaptersMap.types` registry to find the OID of builtin + types, and you can use `~psycopg.types.TypeInfo` to extend the + registry to custom types. + + .. automethod:: get_key + .. automethod:: upgrade + + +.. autoclass:: Loader(oid, context=None) + + :param oid: The type that will be managed by this dumper. + :type oid: int + :param context: The context where the transformation is performed. If not + specified the conversion might be inaccurate, for instance it will not + be possible to know the connection encoding or the server date format. + :type context: `AdaptContext` or None + + A partial implementation of this protocol (implementing everything except + `load()`) is available as `psycopg.adapt.Loader`. + + .. autoattribute:: format + + .. automethod:: load + + +.. autoclass:: AdaptContext + :members: + + .. seealso:: :ref:`adaptation` for an explanation about how contexts are + connected. diff --git a/docs/api/adapt.rst b/docs/api/adapt.rst new file mode 100644 index 0000000..e47816c --- /dev/null +++ b/docs/api/adapt.rst @@ -0,0 +1,91 @@ +`adapt` -- Types adaptation +=========================== + +.. module:: psycopg.adapt + +The `!psycopg.adapt` module exposes a set of objects useful for the +configuration of *data adaptation*, which is the conversion of Python objects +to PostgreSQL data types and back. + +These objects are useful if you need to configure data adaptation, i.e. +if you need to change the default way that Psycopg converts between types or +if you want to adapt custom data types and objects. You don't need this object +in the normal use of Psycopg. + +See :ref:`adaptation` for an overview of the Psycopg adaptation system. + +.. _abstract base class: https://docs.python.org/glossary.html#term-abstract-base-class + + +Dumpers and loaders +------------------- + +.. autoclass:: Dumper(cls, context=None) + + This is an `abstract base class`_, partially implementing the + `~psycopg.abc.Dumper` protocol. Subclasses *must* at least implement the + `.dump()` method and optionally override other members. + + .. automethod:: dump + + .. attribute:: format + :type: psycopg.pq.Format + :value: TEXT + + Class attribute. Set it to `~psycopg.pq.Format.BINARY` if the class + `dump()` methods converts the object to binary format. + + .. automethod:: quote + + .. automethod:: get_key + + .. automethod:: upgrade + + +.. autoclass:: Loader(oid, context=None) + + This is an `abstract base class`_, partially implementing the + `~psycopg.abc.Loader` protocol. Subclasses *must* at least implement the + `.load()` method and optionally override other members. + + .. automethod:: load + + .. attribute:: format + :type: psycopg.pq.Format + :value: TEXT + + Class attribute. Set it to `~psycopg.pq.Format.BINARY` if the class + `load()` methods converts the object from binary format. + + +Other objects used in adaptations +--------------------------------- + +.. autoclass:: PyFormat + :members: + + +.. autoclass:: AdaptersMap + + .. seealso:: :ref:`adaptation` for an explanation about how contexts are + connected. + + .. automethod:: register_dumper + .. automethod:: register_loader + + .. attribute:: types + + The object where to look up for types information (such as the mapping + between type names and oids in the specified context). + + :type: `~psycopg.types.TypesRegistry` + + .. automethod:: get_dumper + .. automethod:: get_dumper_by_oid + .. automethod:: get_loader + + +.. autoclass:: Transformer(context=None) + + :param context: The context where the transformer should operate. + :type context: `~psycopg.abc.AdaptContext` diff --git a/docs/api/connections.rst b/docs/api/connections.rst new file mode 100644 index 0000000..db25382 --- /dev/null +++ b/docs/api/connections.rst @@ -0,0 +1,489 @@ +.. currentmodule:: psycopg + +Connection classes +================== + +The `Connection` and `AsyncConnection` classes are the main wrappers for a +PostgreSQL database session. You can imagine them similar to a :program:`psql` +session. + +One of the differences compared to :program:`psql` is that a `Connection` +usually handles a transaction automatically: other sessions will not be able +to see the changes until you have committed them, more or less explicitly. +Take a look to :ref:`transactions` for the details. + + +The `!Connection` class +----------------------- + +.. autoclass:: Connection() + + This class implements a `DBAPI-compliant interface`__. It is what you want + to use if you write a "classic", blocking program (eventually using + threads or Eventlet/gevent for concurrency). If your program uses `asyncio` + you might want to use `AsyncConnection` instead. + + .. __: https://www.python.org/dev/peps/pep-0249/#connection-objects + + Connections behave as context managers: on block exit, the current + transaction will be committed (or rolled back, in case of exception) and + the connection will be closed. + + .. automethod:: connect + + :param conninfo: The `connection string`__ (a ``postgresql://`` url or + a list of ``key=value`` pairs) to specify where and how to connect. + :param kwargs: Further parameters specifying the connection string. + They override the ones specified in `!conninfo`. + :param autocommit: If `!True` don't start transactions automatically. + See :ref:`transactions` for details. + :param row_factory: The row factory specifying what type of records + to create fetching data (default: `~psycopg.rows.tuple_row()`). See + :ref:`row-factories` for details. + :param cursor_factory: Initial value for the `cursor_factory` attribute + of the connection (new in Psycopg 3.1). + :param prepare_threshold: Initial value for the `prepare_threshold` + attribute of the connection (new in Psycopg 3.1). + + More specialized use: + + :param context: A context to copy the initial adapters configuration + from. It might be an `~psycopg.adapt.AdaptersMap` with customized + loaders and dumpers, used as a template to create several connections. + See :ref:`adaptation` for further details. + + .. __: https://www.postgresql.org/docs/current/libpq-connect.html + #LIBPQ-CONNSTRING + + This method is also aliased as `psycopg.connect()`. + + .. seealso:: + + - the list of `the accepted connection parameters`__ + - the `environment variables`__ affecting connection + + .. __: https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-PARAMKEYWORDS + .. __: https://www.postgresql.org/docs/current/libpq-envars.html + + .. versionchanged:: 3.1 + added `!prepare_threshold` and `!cursor_factory` parameters. + + .. automethod:: close + + .. note:: + + You can use:: + + with psycopg.connect() as conn: + ... + + to close the connection automatically when the block is exited. + See :ref:`with-connection`. + + .. autoattribute:: closed + .. autoattribute:: broken + + .. method:: cursor(*, binary: bool = False, \ + row_factory: Optional[RowFactory] = None) -> Cursor + .. method:: cursor(name: str, *, binary: bool = False, \ + row_factory: Optional[RowFactory] = None, \ + scrollable: Optional[bool] = None, withhold: bool = False) -> ServerCursor + :noindex: + + Return a new cursor to send commands and queries to the connection. + + :param name: If not specified create a client-side cursor, if + specified create a server-side cursor. See + :ref:`cursor-types` for details. + :param binary: If `!True` return binary values from the database. All + the types returned by the query must have a binary + loader. See :ref:`binary-data` for details. + :param row_factory: If specified override the `row_factory` set on the + connection. See :ref:`row-factories` for details. + :param scrollable: Specify the `~ServerCursor.scrollable` property of + the server-side cursor created. + :param withhold: Specify the `~ServerCursor.withhold` property of + the server-side cursor created. + :return: A cursor of the class specified by `cursor_factory` (or + `server_cursor_factory` if `!name` is specified). + + .. note:: + + You can use:: + + with conn.cursor() as cur: + ... + + to close the cursor automatically when the block is exited. + + .. autoattribute:: cursor_factory + + The type, or factory function, returned by `cursor()` and `execute()`. + + Default is `psycopg.Cursor`. + + .. autoattribute:: server_cursor_factory + + The type, or factory function, returned by `cursor()` when a name is + specified. + + Default is `psycopg.ServerCursor`. + + .. autoattribute:: row_factory + + The row factory defining the type of rows returned by + `~Cursor.fetchone()` and the other cursor fetch methods. + + The default is `~psycopg.rows.tuple_row`, which means that the fetch + methods will return simple tuples. + + .. seealso:: See :ref:`row-factories` for details about defining the + objects returned by cursors. + + .. automethod:: execute + + :param query: The query to execute. + :type query: `!str`, `!bytes`, `sql.SQL`, or `sql.Composed` + :param params: The parameters to pass to the query, if any. + :type params: Sequence or Mapping + :param prepare: Force (`!True`) or disallow (`!False`) preparation of + the query. By default (`!None`) prepare automatically. See + :ref:`prepared-statements`. + :param binary: If `!True` the cursor will return binary values from the + database. All the types returned by the query must have a binary + loader. See :ref:`binary-data` for details. + + The method simply creates a `Cursor` instance, `~Cursor.execute()` the + query requested, and returns it. + + See :ref:`query-parameters` for all the details about executing + queries. + + .. automethod:: pipeline + + The method is a context manager: you should call it using:: + + with conn.pipeline() as p: + ... + + At the end of the block, a synchronization point is established and + the connection returns in normal mode. + + You can call the method recursively from within a pipeline block. + Innermost blocks will establish a synchronization point on exit, but + pipeline mode will be kept until the outermost block exits. + + See :ref:`pipeline-mode` for details. + + .. versionadded:: 3.1 + + + .. rubric:: Transaction management methods + + For details see :ref:`transactions`. + + .. automethod:: commit + .. automethod:: rollback + .. automethod:: transaction + + .. note:: + + The method must be called with a syntax such as:: + + with conn.transaction(): + ... + + with conn.transaction() as tx: + ... + + The latter is useful if you need to interact with the + `Transaction` object. See :ref:`transaction-context` for details. + + Inside a transaction block it will not be possible to call `commit()` + or `rollback()`. + + .. autoattribute:: autocommit + + The property is writable for sync connections, read-only for async + ones: you should call `!await` `~AsyncConnection.set_autocommit` + :samp:`({value})` instead. + + The following three properties control the characteristics of new + transactions. See :ref:`transaction-characteristics` for details. + + .. autoattribute:: isolation_level + + `!None` means use the default set in the default_transaction_isolation__ + configuration parameter of the server. + + .. __: https://www.postgresql.org/docs/current/runtime-config-client.html + #GUC-DEFAULT-TRANSACTION-ISOLATION + + .. autoattribute:: read_only + + `!None` means use the default set in the default_transaction_read_only__ + configuration parameter of the server. + + .. __: https://www.postgresql.org/docs/current/runtime-config-client.html + #GUC-DEFAULT-TRANSACTION-READ-ONLY + + .. autoattribute:: deferrable + + `!None` means use the default set in the default_transaction_deferrable__ + configuration parameter of the server. + + .. __: https://www.postgresql.org/docs/current/runtime-config-client.html + #GUC-DEFAULT-TRANSACTION-DEFERRABLE + + + .. rubric:: Checking and configuring the connection state + + .. attribute:: pgconn + :type: psycopg.pq.PGconn + + The `~pq.PGconn` libpq connection wrapper underlying the `!Connection`. + + It can be used to send low level commands to PostgreSQL and access + features not currently wrapped by Psycopg. + + .. autoattribute:: info + + .. autoattribute:: prepare_threshold + + See :ref:`prepared-statements` for details. + + + .. autoattribute:: prepared_max + + If more queries need to be prepared, old ones are deallocated__. + + .. __: https://www.postgresql.org/docs/current/sql-deallocate.html + + + .. rubric:: Methods you can use to do something cool + + .. automethod:: cancel + + .. automethod:: notifies + + Notifies are received after using :sql:`LISTEN` in a connection, when + any sessions in the database generates a :sql:`NOTIFY` on one of the + listened channels. + + .. automethod:: add_notify_handler + + See :ref:`async-notify` for details. + + .. automethod:: remove_notify_handler + + .. automethod:: add_notice_handler + + See :ref:`async-messages` for details. + + .. automethod:: remove_notice_handler + + .. automethod:: fileno + + + .. _tpc-methods: + + .. rubric:: Two-Phase Commit support methods + + .. versionadded:: 3.1 + + .. seealso:: :ref:`two-phase-commit` for an introductory explanation of + these methods. + + .. automethod:: xid + + .. automethod:: tpc_begin + + :param xid: The id of the transaction + :type xid: Xid or str + + This method should be called outside of a transaction (i.e. nothing + may have executed since the last `commit()` or `rollback()` and + `~ConnectionInfo.transaction_status` is `~pq.TransactionStatus.IDLE`). + + Furthermore, it is an error to call `!commit()` or `!rollback()` + within the TPC transaction: in this case a `ProgrammingError` + is raised. + + The `!xid` may be either an object returned by the `xid()` method or a + plain string: the latter allows to create a transaction using the + provided string as PostgreSQL transaction id. See also + `tpc_recover()`. + + + .. automethod:: tpc_prepare + + A `ProgrammingError` is raised if this method is used outside of a TPC + transaction. + + After calling `!tpc_prepare()`, no statements can be executed until + `tpc_commit()` or `tpc_rollback()` will be + called. + + .. seealso:: The |PREPARE TRANSACTION|_ PostgreSQL command. + + .. |PREPARE TRANSACTION| replace:: :sql:`PREPARE TRANSACTION` + .. _PREPARE TRANSACTION: https://www.postgresql.org/docs/current/static/sql-prepare-transaction.html + + + .. automethod:: tpc_commit + + :param xid: The id of the transaction + :type xid: Xid or str + + When called with no arguments, `!tpc_commit()` commits a TPC + transaction previously prepared with `tpc_prepare()`. + + If `!tpc_commit()` is called prior to `!tpc_prepare()`, a single phase + commit is performed. A transaction manager may choose to do this if + only a single resource is participating in the global transaction. + + When called with a transaction ID `!xid`, the database commits the + given transaction. If an invalid transaction ID is provided, a + `ProgrammingError` will be raised. This form should be called outside + of a transaction, and is intended for use in recovery. + + On return, the TPC transaction is ended. + + .. seealso:: The |COMMIT PREPARED|_ PostgreSQL command. + + .. |COMMIT PREPARED| replace:: :sql:`COMMIT PREPARED` + .. _COMMIT PREPARED: https://www.postgresql.org/docs/current/static/sql-commit-prepared.html + + + .. automethod:: tpc_rollback + + :param xid: The id of the transaction + :type xid: Xid or str + + When called with no arguments, `!tpc_rollback()` rolls back a TPC + transaction. It may be called before or after `tpc_prepare()`. + + When called with a transaction ID `!xid`, it rolls back the given + transaction. If an invalid transaction ID is provided, a + `ProgrammingError` is raised. This form should be called outside of a + transaction, and is intended for use in recovery. + + On return, the TPC transaction is ended. + + .. seealso:: The |ROLLBACK PREPARED|_ PostgreSQL command. + + .. |ROLLBACK PREPARED| replace:: :sql:`ROLLBACK PREPARED` + .. _ROLLBACK PREPARED: https://www.postgresql.org/docs/current/static/sql-rollback-prepared.html + + + .. automethod:: tpc_recover + + Returns a list of `Xid` representing pending transactions, suitable + for use with `tpc_commit()` or `tpc_rollback()`. + + If a transaction was not initiated by Psycopg, the returned Xids will + have attributes `~Xid.format_id` and `~Xid.bqual` set to `!None` and + the `~Xid.gtrid` set to the PostgreSQL transaction ID: such Xids are + still usable for recovery. Psycopg uses the same algorithm of the + `PostgreSQL JDBC driver`__ to encode a XA triple in a string, so + transactions initiated by a program using such driver should be + unpacked correctly. + + .. __: https://jdbc.postgresql.org/ + + Xids returned by `!tpc_recover()` also have extra attributes + `~Xid.prepared`, `~Xid.owner`, `~Xid.database` populated with the + values read from the server. + + .. seealso:: the |pg_prepared_xacts|_ system view. + + .. |pg_prepared_xacts| replace:: `pg_prepared_xacts` + .. _pg_prepared_xacts: https://www.postgresql.org/docs/current/static/view-pg-prepared-xacts.html + + +The `!AsyncConnection` class +---------------------------- + +.. autoclass:: AsyncConnection() + + This class implements a DBAPI-inspired interface, with all the blocking + methods implemented as coroutines. Unless specified otherwise, + non-blocking methods are shared with the `Connection` class. + + The following methods have the same behaviour of the matching `!Connection` + methods, but should be called using the `await` keyword. + + .. automethod:: connect + + .. versionchanged:: 3.1 + + Automatically resolve domain names asynchronously. In previous + versions, name resolution blocks, unless the `!hostaddr` + parameter is specified, or the `~psycopg._dns.resolve_hostaddr_async()` + function is used. + + .. automethod:: close + + .. note:: You can use ``async with`` to close the connection + automatically when the block is exited, but be careful about + the async quirkness: see :ref:`async-with` for details. + + .. method:: cursor(*, binary: bool = False, \ + row_factory: Optional[RowFactory] = None) -> AsyncCursor + .. method:: cursor(name: str, *, binary: bool = False, \ + row_factory: Optional[RowFactory] = None, \ + scrollable: Optional[bool] = None, withhold: bool = False) -> AsyncServerCursor + :noindex: + + .. note:: + + You can use:: + + async with conn.cursor() as cur: + ... + + to close the cursor automatically when the block is exited. + + .. autoattribute:: cursor_factory + + Default is `psycopg.AsyncCursor`. + + .. autoattribute:: server_cursor_factory + + Default is `psycopg.AsyncServerCursor`. + + .. autoattribute:: row_factory + + .. automethod:: execute + + .. automethod:: pipeline + + .. note:: + + It must be called as:: + + async with conn.pipeline() as p: + ... + + .. automethod:: commit + .. automethod:: rollback + + .. automethod:: transaction + + .. note:: + + It must be called as:: + + async with conn.transaction() as tx: + ... + + .. automethod:: notifies + .. automethod:: set_autocommit + .. automethod:: set_isolation_level + .. automethod:: set_read_only + .. automethod:: set_deferrable + + .. automethod:: tpc_prepare + .. automethod:: tpc_commit + .. automethod:: tpc_rollback + .. automethod:: tpc_recover diff --git a/docs/api/conninfo.rst b/docs/api/conninfo.rst new file mode 100644 index 0000000..9e5b01d --- /dev/null +++ b/docs/api/conninfo.rst @@ -0,0 +1,24 @@ +.. _psycopg.conninfo: + +`conninfo` -- manipulate connection strings +=========================================== + +This module contains a few utility functions to manipulate database +connection strings. + +.. module:: psycopg.conninfo + +.. autofunction:: conninfo_to_dict + + .. code:: python + + >>> conninfo_to_dict("postgres://jeff@example.com/db", user="piro") + {'user': 'piro', 'dbname': 'db', 'host': 'example.com'} + + +.. autofunction:: make_conninfo + + .. code:: python + + >>> make_conninfo("dbname=db user=jeff", user="piro", port=5432) + 'dbname=db user=piro port=5432' diff --git a/docs/api/copy.rst b/docs/api/copy.rst new file mode 100644 index 0000000..81a96e2 --- /dev/null +++ b/docs/api/copy.rst @@ -0,0 +1,117 @@ +.. currentmodule:: psycopg + +COPY-related objects +==================== + +The main objects (`Copy`, `AsyncCopy`) present the main interface to exchange +data during a COPY operations. These objects are normally obtained by the +methods `Cursor.copy()` and `AsyncCursor.copy()`; however, they can be also +created directly, for instance to write to a destination which is not a +database (e.g. using a `~psycopg.copy.FileWriter`). + +See :ref:`copy` for details. + + +Main Copy objects +----------------- + +.. autoclass:: Copy() + + The object is normally returned by `!with` `Cursor.copy()`. + + .. automethod:: write_row + + The data in the tuple will be converted as configured on the cursor; + see :ref:`adaptation` for details. + + .. automethod:: write + .. automethod:: read + + Instead of using `!read()` you can iterate on the `!Copy` object to + read its data row by row, using ``for row in copy: ...``. + + .. automethod:: rows + + Equivalent of iterating on `read_row()` until it returns `!None` + + .. automethod:: read_row + .. automethod:: set_types + + +.. autoclass:: AsyncCopy() + + The object is normally returned by ``async with`` `AsyncCursor.copy()`. + Its methods are similar to the ones of the `Copy` object but offering an + `asyncio` interface (`await`, `async for`, `async with`). + + .. automethod:: write_row + .. automethod:: write + .. automethod:: read + + Instead of using `!read()` you can iterate on the `!AsyncCopy` object + to read its data row by row, using ``async for row in copy: ...``. + + .. automethod:: rows + + Use it as `async for record in copy.rows():` ... + + .. automethod:: read_row + + +.. _copy-writers: + +Writer objects +-------------- + +.. currentmodule:: psycopg.copy + +.. versionadded:: 3.1 + +Copy writers are helper objects to specify where to write COPY-formatted data. +By default, data is written to the database (using the `LibpqWriter`). It is +possible to write copy-data for offline use by using a `FileWriter`, or to +customize further writing by implementing your own `Writer` or `AsyncWriter` +subclass. + +Writers instances can be used passing them to the cursor +`~psycopg.Cursor.copy()` method or to the `~psycopg.Copy` constructor, as the +`!writer` argument. + +.. autoclass:: Writer + + This is an abstract base class: subclasses are required to implement their + `write()` method. + + .. automethod:: write + .. automethod:: finish + + +.. autoclass:: LibpqWriter + + This is the writer used by default if none is specified. + + +.. autoclass:: FileWriter + + This writer should be used without executing a :sql:`COPY` operation on + the database. For example, if `records` is a list of tuples containing + data to save in COPY format to a file (e.g. for later import), it can be + used as: + + .. code:: python + + with open("target-file.pgcopy", "wb") as f: + with Copy(cur, writer=FileWriter(f)) as copy: + for record in records + copy.write_row(record) + + +.. autoclass:: AsyncWriter + + This class methods have the same semantics of the ones of `Writer`, but + offer an async interface. + + .. automethod:: write + .. automethod:: finish + +.. autoclass:: AsyncLibpqWriter diff --git a/docs/api/crdb.rst b/docs/api/crdb.rst new file mode 100644 index 0000000..de8344e --- /dev/null +++ b/docs/api/crdb.rst @@ -0,0 +1,120 @@ +`crdb` -- CockroachDB support +============================= + +.. module:: psycopg.crdb + +.. versionadded:: 3.1 + +CockroachDB_ is a distributed database using the same fronted-backend protocol +of PostgreSQL. As such, Psycopg can be used to write Python programs +interacting with CockroachDB. + +.. _CockroachDB: https://www.cockroachlabs.com/ + +Opening a connection to a CRDB database using `psycopg.connect()` provides a +largely working object. However, using the `psycopg.crdb.connect()` function +instead, Psycopg will create more specialised objects and provide a types +mapping tweaked on the CockroachDB data model. + + +.. _crdb-differences: + +Main differences from PostgreSQL +-------------------------------- + +CockroachDB behaviour is `different from PostgreSQL`__: please refer to the +database documentation for details. These are some of the main differences +affecting Psycopg behaviour: + +.. __: https://www.cockroachlabs.com/docs/stable/postgresql-compatibility.html + +- `~psycopg.Connection.cancel()` doesn't work before CockroachDB 22.1. On + older versions, you can use `CANCEL QUERY`_ instead (but from a different + connection). + +- :ref:`server-side-cursors` are well supported only from CockroachDB 22.1.3. + +- `~psycopg.ConnectionInfo.backend_pid` is only populated from CockroachDB + 22.1. Note however that you cannot use the PID to terminate the session; use + `SHOW session_id`_ to find the id of a session, which you may terminate with + `CANCEL SESSION`_ in lieu of PostgreSQL's :sql:`pg_terminate_backend()`. + +- Several data types are missing or slightly different from PostgreSQL (see + `adapters` for an overview of the differences). + +- The :ref:`two-phase commit protocol <two-phase-commit>` is not supported. + +- :sql:`LISTEN` and :sql:`NOTIFY` are not supported. However the `CHANGEFEED`_ + command, in conjunction with `~psycopg.Cursor.stream()`, can provide push + notifications. + +.. _CANCEL QUERY: https://www.cockroachlabs.com/docs/stable/cancel-query.html +.. _SHOW session_id: https://www.cockroachlabs.com/docs/stable/show-vars.html +.. _CANCEL SESSION: https://www.cockroachlabs.com/docs/stable/cancel-session.html +.. _CHANGEFEED: https://www.cockroachlabs.com/docs/stable/changefeed-for.html + + +.. _crdb-objects: + +CockroachDB-specific objects +---------------------------- + +.. autofunction:: connect + + This is an alias of the class method `CrdbConnection.connect`. + + If you need an asynchronous connection use the `AsyncCrdbConnection.connect()` + method instead. + + +.. autoclass:: CrdbConnection + + `psycopg.Connection` subclass. + + .. automethod:: is_crdb + + :param conn: the connection to check + :type conn: `~psycopg.Connection`, `~psycopg.AsyncConnection`, `~psycopg.pq.PGconn` + + +.. autoclass:: AsyncCrdbConnection + + `psycopg.AsyncConnection` subclass. + + +.. autoclass:: CrdbConnectionInfo + + The object is returned by the `~psycopg.Connection.info` attribute of + `CrdbConnection` and `AsyncCrdbConnection`. + + The object behaves like `!ConnectionInfo`, with the following differences: + + .. autoattribute:: vendor + + The `CockroachDB` string. + + .. autoattribute:: server_version + + +.. data:: adapters + + The default adapters map establishing how Python and CockroachDB types are + converted into each other. + + The map is used as a template when new connections are created, using + `psycopg.crdb.connect()` (similarly to the way `psycopg.adapters` is used + as template for new PostgreSQL connections). + + This registry contains only the types and adapters supported by + CockroachDB. Several PostgreSQL types and adapters are missing or + different from PostgreSQL, among which: + + - Composite types + - :sql:`range`, :sql:`multirange` types + - The :sql:`hstore` type + - Geometric types + - Nested arrays + - Arrays of :sql:`jsonb` + - The :sql:`cidr` data type + - The :sql:`json` type is an alias for :sql:`jsonb` + - The :sql:`int` type is an alias for :sql:`int8`, not `int4`. diff --git a/docs/api/cursors.rst b/docs/api/cursors.rst new file mode 100644 index 0000000..9c5b478 --- /dev/null +++ b/docs/api/cursors.rst @@ -0,0 +1,517 @@ +.. currentmodule:: psycopg + +Cursor classes +============== + +The `Cursor` and `AsyncCursor` classes are the main objects to send commands +to a PostgreSQL database session. They are normally created by the +connection's `~Connection.cursor()` method. + +Using the `!name` parameter on `!cursor()` will create a `ServerCursor` or +`AsyncServerCursor`, which can be used to retrieve partial results from a +database. + +A `Connection` can create several cursors, but only one at time can perform +operations, so they are not the best way to achieve parallelism (you may want +to operate with several connections instead). All the cursors on the same +connection have a view of the same session, so they can see each other's +uncommitted data. + + +The `!Cursor` class +------------------- + +.. autoclass:: Cursor + + This class implements a `DBAPI-compliant interface`__. It is what the + classic `Connection.cursor()` method returns. `AsyncConnection.cursor()` + will create instead `AsyncCursor` objects, which have the same set of + method but expose an `asyncio` interface and require `!async` and + `!await` keywords to operate. + + .. __: dbapi-cursor_ + .. _dbapi-cursor: https://www.python.org/dev/peps/pep-0249/#cursor-objects + + + Cursors behave as context managers: on block exit they are closed and + further operation will not be possible. Closing a cursor will not + terminate a transaction or a session though. + + .. attribute:: connection + :type: Connection + + The connection this cursor is using. + + .. automethod:: close + + .. note:: + + You can use:: + + with conn.cursor() as cur: + ... + + to close the cursor automatically when the block is exited. See + :ref:`usage`. + + .. autoattribute:: closed + + .. rubric:: Methods to send commands + + .. automethod:: execute + + :param query: The query to execute. + :type query: `!str`, `!bytes`, `sql.SQL`, or `sql.Composed` + :param params: The parameters to pass to the query, if any. + :type params: Sequence or Mapping + :param prepare: Force (`!True`) or disallow (`!False`) preparation of + the query. By default (`!None`) prepare automatically. See + :ref:`prepared-statements`. + :param binary: Specify whether the server should return data in binary + format (`!True`) or in text format (`!False`). By default + (`!None`) return data as requested by the cursor's `~Cursor.format`. + + Return the cursor itself, so that it will be possible to chain a fetch + operation after the call. + + See :ref:`query-parameters` for all the details about executing + queries. + + .. versionchanged:: 3.1 + + The `query` argument must be a `~typing.StringLiteral`. If you + need to compose a query dynamically, please use `sql.SQL` and + related objects. + + See :pep:`675` for details. + + .. automethod:: executemany + + :param query: The query to execute + :type query: `!str`, `!bytes`, `sql.SQL`, or `sql.Composed` + :param params_seq: The parameters to pass to the query + :type params_seq: Sequence of Sequences or Mappings + :param returning: If `!True`, fetch the results of the queries executed + :type returning: `!bool` + + This is more efficient than performing separate queries, but in case of + several :sql:`INSERT` (and with some SQL creativity for massive + :sql:`UPDATE` too) you may consider using `copy()`. + + If the queries return data you want to read (e.g. when executing an + :sql:`INSERT ... RETURNING` or a :sql:`SELECT` with a side-effect), + you can specify `!returning=True`; the results will be available in + the cursor's state and can be read using `fetchone()` and similar + methods. Each input parameter will produce a separate result set: use + `nextset()` to read the results of the queries after the first one. + + See :ref:`query-parameters` for all the details about executing + queries. + + .. versionchanged:: 3.1 + + - Added `!returning` parameter to receive query results. + - Performance optimised by making use of the pipeline mode, when + using libpq 14 or newer. + + .. automethod:: copy + + :param statement: The copy operation to execute + :type statement: `!str`, `!bytes`, `sql.SQL`, or `sql.Composed` + :param params: The parameters to pass to the statement, if any. + :type params: Sequence or Mapping + + .. note:: + + The method must be called with:: + + with cursor.copy() as copy: + ... + + See :ref:`copy` for information about :sql:`COPY`. + + .. versionchanged:: 3.1 + Added parameters support. + + .. automethod:: stream + + This command is similar to execute + iter; however it supports endless + data streams. The feature is not available in PostgreSQL, but some + implementations exist: Materialize `TAIL`__ and CockroachDB + `CHANGEFEED`__ for instance. + + The feature, and the API supporting it, are still experimental. + Beware... 👀 + + .. __: https://materialize.com/docs/sql/tail/#main + .. __: https://www.cockroachlabs.com/docs/stable/changefeed-for.html + + The parameters are the same of `execute()`. + + .. warning:: + + Failing to consume the iterator entirely will result in a + connection left in `~psycopg.ConnectionInfo.transaction_status` + `~pq.TransactionStatus.ACTIVE` state: this connection will refuse + to receive further commands (with a message such as *another + command is already in progress*). + + If there is a chance that the generator is not consumed entirely, + in order to restore the connection to a working state you can call + `~generator.close` on the generator object returned by `!stream()`. The + `contextlib.closing` function might be particularly useful to make + sure that `!close()` is called: + + .. code:: + + with closing(cur.stream("select generate_series(1, 10000)")) as gen: + for rec in gen: + something(rec) # might fail + + Without calling `!close()`, in case of error, the connection will + be `!ACTIVE` and unusable. If `!close()` is called, the connection + might be `!INTRANS` or `!INERROR`, depending on whether the server + managed to send the entire resultset to the client. An autocommit + connection will be `!IDLE` instead. + + + .. attribute:: format + + The format of the data returned by the queries. It can be selected + initially e.g. specifying `Connection.cursor`\ `!(binary=True)` and + changed during the cursor's lifetime. It is also possible to override + the value for single queries, e.g. specifying `execute`\ + `!(binary=True)`. + + :type: `pq.Format` + :default: `~pq.Format.TEXT` + + .. seealso:: :ref:`binary-data` + + + .. rubric:: Methods to retrieve results + + Fetch methods are only available if the last operation produced results, + e.g. a :sql:`SELECT` or a command with :sql:`RETURNING`. They will raise + an exception if used with operations that don't return result, such as an + :sql:`INSERT` with no :sql:`RETURNING` or an :sql:`ALTER TABLE`. + + .. note:: + + Cursors are iterable objects, so just using the:: + + for record in cursor: + ... + + syntax will iterate on the records in the current recordset. + + .. autoattribute:: row_factory + + The property affects the objects returned by the `fetchone()`, + `fetchmany()`, `fetchall()` methods. The default + (`~psycopg.rows.tuple_row`) returns a tuple for each record fetched. + + See :ref:`row-factories` for details. + + .. automethod:: fetchone + .. automethod:: fetchmany + .. automethod:: fetchall + .. automethod:: nextset + .. automethod:: scroll + + .. attribute:: pgresult + :type: Optional[psycopg.pq.PGresult] + + The result returned by the last query and currently exposed by the + cursor, if available, else `!None`. + + It can be used to obtain low level info about the last query result + and to access to features not currently wrapped by Psycopg. + + + .. rubric:: Information about the data + + .. autoattribute:: description + + .. autoattribute:: statusmessage + + This is the status tag you typically see in :program:`psql` after + a successful command, such as ``CREATE TABLE`` or ``UPDATE 42``. + + .. autoattribute:: rowcount + .. autoattribute:: rownumber + + .. attribute:: _query + + An helper object used to convert queries and parameters before sending + them to PostgreSQL. + + .. note:: + This attribute is exposed because it might be helpful to debug + problems when the communication between Python and PostgreSQL + doesn't work as expected. For this reason, the attribute is + available when a query fails too. + + .. warning:: + You shouldn't consider it part of the public interface of the + object: it might change without warnings. + + Except this warning, I guess. + + If you would like to build reliable features using this object, + please get in touch so we can try and design an useful interface + for it. + + Among the properties currently exposed by this object: + + - `!query` (`!bytes`): the query effectively sent to PostgreSQL. It + will have Python placeholders (``%s``\-style) replaced with + PostgreSQL ones (``$1``, ``$2``\-style). + + - `!params` (sequence of `!bytes`): the parameters passed to + PostgreSQL, adapted to the database format. + + - `!types` (sequence of `!int`): the OID of the parameters passed to + PostgreSQL. + + - `!formats` (sequence of `pq.Format`): whether the parameter format + is text or binary. + + +The `!ClientCursor` class +------------------------- + +.. seealso:: See :ref:`client-side-binding-cursors` for details. + +.. autoclass:: ClientCursor + + This `Cursor` subclass has exactly the same interface of its parent class, + but, instead of sending query and parameters separately to the server, it + merges them on the client and sends them as a non-parametric query on the + server. This allows, for instance, to execute parametrized data definition + statements and other :ref:`problematic queries <server-side-binding>`. + + .. versionadded:: 3.1 + + .. automethod:: mogrify + + :param query: The query to execute. + :type query: `!str`, `!bytes`, `sql.SQL`, or `sql.Composed` + :param params: The parameters to pass to the query, if any. + :type params: Sequence or Mapping + + +The `!ServerCursor` class +-------------------------- + +.. seealso:: See :ref:`server-side-cursors` for details. + +.. autoclass:: ServerCursor + + This class also implements a `DBAPI-compliant interface`__. It is created + by `Connection.cursor()` specifying the `!name` parameter. Using this + object results in the creation of an equivalent PostgreSQL cursor in the + server. DBAPI-extension methods (such as `~Cursor.copy()` or + `~Cursor.stream()`) are not implemented on this object: use a normal + `Cursor` instead. + + .. __: dbapi-cursor_ + + Most attribute and methods behave exactly like in `Cursor`, here are + documented the differences: + + .. autoattribute:: name + .. autoattribute:: scrollable + + .. seealso:: The PostgreSQL DECLARE_ statement documentation + for the description of :sql:`[NO] SCROLL`. + + .. autoattribute:: withhold + + .. seealso:: The PostgreSQL DECLARE_ statement documentation + for the description of :sql:`{WITH|WITHOUT} HOLD`. + + .. _DECLARE: https://www.postgresql.org/docs/current/sql-declare.html + + + .. automethod:: close + + .. warning:: Closing a server-side cursor is more important than + closing a client-side one because it also releases the resources + on the server, which otherwise might remain allocated until the + end of the session (memory, locks). Using the pattern:: + + with conn.cursor(): + ... + + is especially useful so that the cursor is closed at the end of + the block. + + .. automethod:: execute + + :param query: The query to execute. + :type query: `!str`, `!bytes`, `sql.SQL`, or `sql.Composed` + :param params: The parameters to pass to the query, if any. + :type params: Sequence or Mapping + :param binary: Specify whether the server should return data in binary + format (`!True`) or in text format (`!False`). By default + (`!None`) return data as requested by the cursor's `~Cursor.format`. + + Create a server cursor with given `!name` and the `!query` in argument. + + If using :sql:`DECLARE` is not appropriate (for instance because the + cursor is returned by calling a stored procedure) you can avoid to use + `!execute()`, crete the cursor in other ways, and use directly the + `!fetch*()` methods instead. See :ref:`cursor-steal` for an example. + + Using `!execute()` more than once will close the previous cursor and + open a new one with the same name. + + .. automethod:: executemany + .. automethod:: fetchone + .. automethod:: fetchmany + .. automethod:: fetchall + + These methods use the FETCH_ SQL statement to retrieve some of the + records from the cursor's current position. + + .. _FETCH: https://www.postgresql.org/docs/current/sql-fetch.html + + .. note:: + + You can also iterate on the cursor to read its result one at + time with:: + + for record in cur: + ... + + In this case, the records are not fetched one at time from the + server but they are retrieved in batches of `itersize` to reduce + the number of server roundtrips. + + .. autoattribute:: itersize + + Number of records to fetch at time when iterating on the cursor. The + default is 100. + + .. automethod:: scroll + + This method uses the MOVE_ SQL statement to move the current position + in the server-side cursor, which will affect following `!fetch*()` + operations. If you need to scroll backwards you should probably + call `~Connection.cursor()` using `scrollable=True`. + + Note that PostgreSQL doesn't provide a reliable way to report when a + cursor moves out of bound, so the method might not raise `!IndexError` + when it happens, but it might rather stop at the cursor boundary. + + .. _MOVE: https://www.postgresql.org/docs/current/sql-fetch.html + + +The `!AsyncCursor` class +------------------------ + +.. autoclass:: AsyncCursor + + This class implements a DBAPI-inspired interface, with all the blocking + methods implemented as coroutines. Unless specified otherwise, + non-blocking methods are shared with the `Cursor` class. + + The following methods have the same behaviour of the matching `!Cursor` + methods, but should be called using the `await` keyword. + + .. attribute:: connection + :type: AsyncConnection + + .. automethod:: close + + .. note:: + + You can use:: + + async with conn.cursor(): + ... + + to close the cursor automatically when the block is exited. + + .. automethod:: execute + .. automethod:: executemany + .. automethod:: copy + + .. note:: + + The method must be called with:: + + async with cursor.copy() as copy: + ... + + .. automethod:: stream + + .. note:: + + The method must be called with:: + + async for record in cursor.stream(query): + ... + + .. automethod:: fetchone + .. automethod:: fetchmany + .. automethod:: fetchall + .. automethod:: scroll + + .. note:: + + You can also use:: + + async for record in cursor: + ... + + to iterate on the async cursor results. + + +The `!AsyncClientCursor` class +------------------------------ + +.. autoclass:: AsyncClientCursor + + This class is the `!async` equivalent of the `ClientCursor`. The + difference are the same shown in `AsyncCursor`. + + .. versionadded:: 3.1 + + + +The `!AsyncServerCursor` class +------------------------------ + +.. autoclass:: AsyncServerCursor + + This class implements a DBAPI-inspired interface as the `AsyncCursor` + does, but wraps a server-side cursor like the `ServerCursor` class. It is + created by `AsyncConnection.cursor()` specifying the `!name` parameter. + + The following are the methods exposing a different (async) interface from + the `ServerCursor` counterpart, but sharing the same semantics. + + .. automethod:: close + + .. note:: + You can close the cursor automatically using:: + + async with conn.cursor("name") as cursor: + ... + + .. automethod:: execute + .. automethod:: executemany + .. automethod:: fetchone + .. automethod:: fetchmany + .. automethod:: fetchall + + .. note:: + + You can also iterate on the cursor using:: + + async for record in cur: + ... + + .. automethod:: scroll diff --git a/docs/api/dns.rst b/docs/api/dns.rst new file mode 100644 index 0000000..186bde3 --- /dev/null +++ b/docs/api/dns.rst @@ -0,0 +1,145 @@ +`_dns` -- DNS resolution utilities +================================== + +.. module:: psycopg._dns + +This module contains a few experimental utilities to interact with the DNS +server before performing a connection. + +.. warning:: + This module is experimental and its interface could change in the future, + without warning or respect for the version scheme. It is provided here to + allow experimentation before making it more stable. + +.. warning:: + This module depends on the `dnspython`_ package. The package is currently + not installed automatically as a Psycopg dependency and must be installed + manually: + + .. code:: sh + + $ pip install "dnspython >= 2.1" + + .. _dnspython: https://dnspython.readthedocs.io/ + + +.. function:: resolve_srv(params) + + Apply SRV DNS lookup as defined in :RFC:`2782`. + + :param params: The input parameters, for instance as returned by + `~psycopg.conninfo.conninfo_to_dict()`. + :type params: `!dict` + :return: An updated list of connection parameters. + + For every host defined in the ``params["host"]`` list (comma-separated), + perform SRV lookup if the host is in the form ``_Service._Proto.Target``. + If lookup is successful, return a params dict with hosts and ports replaced + with the looked-up entries. + + Raise `~psycopg.OperationalError` if no lookup is successful and no host + (looked up or unchanged) could be returned. + + In addition to the rules defined by RFC 2782 about the host name pattern, + perform SRV lookup also if the the port is the string ``SRV`` (case + insensitive). + + .. warning:: + This is an experimental functionality. + + .. note:: + One possible way to use this function automatically is to subclass + `~psycopg.Connection`, extending the + `~psycopg.Connection._get_connection_params()` method:: + + import psycopg._dns # not imported automatically + + class SrvCognizantConnection(psycopg.Connection): + @classmethod + def _get_connection_params(cls, conninfo, **kwargs): + params = super()._get_connection_params(conninfo, **kwargs) + params = psycopg._dns.resolve_srv(params) + return params + + # The name will be resolved to db1.example.com + cnn = SrvCognizantConnection.connect("host=_postgres._tcp.db.psycopg.org") + + +.. function:: resolve_srv_async(params) + :async: + + Async equivalent of `resolve_srv()`. + + +.. automethod:: psycopg.Connection._get_connection_params + + .. warning:: + This is an experimental method. + + This method is a subclass hook allowing to manipulate the connection + parameters before performing the connection. Make sure to call the + `!super()` implementation before further manipulation of the arguments:: + + @classmethod + def _get_connection_params(cls, conninfo, **kwargs): + params = super()._get_connection_params(conninfo, **kwargs) + # do something with the params + return params + + +.. automethod:: psycopg.AsyncConnection._get_connection_params + + .. warning:: + This is an experimental method. + + +.. function:: resolve_hostaddr_async(params) + :async: + + Perform async DNS lookup of the hosts and return a new params dict. + + .. deprecated:: 3.1 + The use of this function is not necessary anymore, because + `psycopg.AsyncConnection.connect()` performs non-blocking name + resolution automatically. + + :param params: The input parameters, for instance as returned by + `~psycopg.conninfo.conninfo_to_dict()`. + :type params: `!dict` + + If a ``host`` param is present but not ``hostname``, resolve the host + addresses dynamically. + + The function may change the input ``host``, ``hostname``, ``port`` to allow + connecting without further DNS lookups, eventually removing hosts that are + not resolved, keeping the lists of hosts and ports consistent. + + Raise `~psycopg.OperationalError` if connection is not possible (e.g. no + host resolve, inconsistent lists length). + + See `the PostgreSQL docs`__ for explanation of how these params are used, + and how they support multiple entries. + + .. __: https://www.postgresql.org/docs/current/libpq-connect.html + #LIBPQ-PARAMKEYWORDS + + .. warning:: + Before psycopg 3.1, this function doesn't handle the ``/etc/hosts`` file. + + .. note:: + Starting from psycopg 3.1, a similar operation is performed + automatically by `!AsyncConnection._get_connection_params()`, so this + function is unneeded. + + In psycopg 3.0, one possible way to use this function automatically is + to subclass `~psycopg.AsyncConnection`, extending the + `~psycopg.AsyncConnection._get_connection_params()` method:: + + import psycopg._dns # not imported automatically + + class AsyncDnsConnection(psycopg.AsyncConnection): + @classmethod + async def _get_connection_params(cls, conninfo, **kwargs): + params = await super()._get_connection_params(conninfo, **kwargs) + params = await psycopg._dns.resolve_hostaddr_async(params) + return params diff --git a/docs/api/errors.rst b/docs/api/errors.rst new file mode 100644 index 0000000..2fca7c6 --- /dev/null +++ b/docs/api/errors.rst @@ -0,0 +1,540 @@ +`errors` -- Package exceptions +============================== + +.. module:: psycopg.errors + +.. index:: + single: Error; Class + +This module exposes objects to represent and examine database errors. + + +.. currentmodule:: psycopg + +.. index:: + single: Exceptions; DB-API + +.. _dbapi-exceptions: + +DB-API exceptions +----------------- + +In compliance with the DB-API, all the exceptions raised by Psycopg +derive from the following classes: + +.. parsed-literal:: + + `!Exception` + \|__ `Warning` + \|__ `Error` + \|__ `InterfaceError` + \|__ `DatabaseError` + \|__ `DataError` + \|__ `OperationalError` + \|__ `IntegrityError` + \|__ `InternalError` + \|__ `ProgrammingError` + \|__ `NotSupportedError` + +These classes are exposed both by this module and the root `psycopg` module. + +.. autoexception:: Error() + + .. autoattribute:: diag + .. autoattribute:: sqlstate + + The code of the error, if received from the server. + + This attribute is also available as class attribute on the + :ref:`sqlstate-exceptions` classes. + + .. autoattribute:: pgconn + + Most likely it will be in `~psycopg.pq.ConnStatus.BAD` state; + however it might be useful to verify precisely what went wrong, for + instance checking the `~psycopg.pq.PGconn.needs_password` and + `~psycopg.pq.PGconn.used_password` attributes. + + .. versionadded:: 3.1 + + .. autoattribute:: pgresult + + .. versionadded:: 3.1 + + +.. autoexception:: Warning() +.. autoexception:: InterfaceError() +.. autoexception:: DatabaseError() +.. autoexception:: DataError() +.. autoexception:: OperationalError() +.. autoexception:: IntegrityError() +.. autoexception:: InternalError() +.. autoexception:: ProgrammingError() +.. autoexception:: NotSupportedError() + + +Other Psycopg errors +^^^^^^^^^^^^^^^^^^^^ + +.. currentmodule:: psycopg.errors + + +In addition to the standard DB-API errors, Psycopg defines a few more specific +ones. + +.. autoexception:: ConnectionTimeout() +.. autoexception:: PipelineAborted() + + + +.. index:: + single: Exceptions; PostgreSQL + +Error diagnostics +----------------- + +.. autoclass:: Diagnostic() + + The object is available as the `~psycopg.Error`.\ `~psycopg.Error.diag` + attribute and is passed to the callback functions registered with + `~psycopg.Connection.add_notice_handler()`. + + All the information available from the :pq:`PQresultErrorField()` function + are exposed as attributes by the object. For instance the `!severity` + attribute returns the `!PG_DIAG_SEVERITY` code. Please refer to the + PostgreSQL documentation for the meaning of all the attributes. + + The attributes available are: + + .. attribute:: + column_name + constraint_name + context + datatype_name + internal_position + internal_query + message_detail + message_hint + message_primary + schema_name + severity + severity_nonlocalized + source_file + source_function + source_line + sqlstate + statement_position + table_name + + A string with the error field if available; `!None` if not available. + The attribute value is available only for errors sent by the server: + not all the fields are available for all the errors and for all the + server versions. + + +.. _sqlstate-exceptions: + +SQLSTATE exceptions +------------------- + +Errors coming from a database server (as opposite as ones generated +client-side, such as connection failed) usually have a 5-letters error code +called SQLSTATE (available in the `~Diagnostic.sqlstate` attribute of the +error's `~psycopg.Error.diag` attribute). + +Psycopg exposes a different class for each SQLSTATE value, allowing to +write idiomatic error handling code according to specific conditions happening +in the database: + +.. code-block:: python + + try: + cur.execute("LOCK TABLE mytable IN ACCESS EXCLUSIVE MODE NOWAIT") + except psycopg.errors.LockNotAvailable: + locked = True + +The exception names are generated from the PostgreSQL source code and includes +classes for every error defined by PostgreSQL in versions between 9.6 and 15. +Every class in the module is named after what referred as "condition name" `in +the documentation`__, converted to CamelCase: e.g. the error 22012, +``division_by_zero`` is exposed by this module as the class `!DivisionByZero`. +There is a handful of... exceptions to this rule, required for disambiguate +name clashes: please refer to the :ref:`table below <exceptions-list>` for all +the classes defined. + +.. __: https://www.postgresql.org/docs/current/errcodes-appendix.html#ERRCODES-TABLE + +Every exception class is a subclass of one of the :ref:`standard DB-API +exception <dbapi-exceptions>`, thus exposing the `~psycopg.Error` interface. + +.. versionchanged:: 3.1.4 + Added exceptions introduced in PostgreSQL 15. + +.. autofunction:: lookup + + Example: if you have code using constant names or sql codes you can use + them to look up the exception class. + + .. code-block:: python + + try: + cur.execute("LOCK TABLE mytable IN ACCESS EXCLUSIVE MODE NOWAIT") + except psycopg.errors.lookup("UNDEFINED_TABLE"): + missing = True + except psycopg.errors.lookup("55P03"): + locked = True + + +.. _exceptions-list: + +List of known exceptions +^^^^^^^^^^^^^^^^^^^^^^^^ + +The following are all the SQLSTATE-related error classed defined by this +module, together with the base DBAPI exception they derive from. + +.. autogenerated: start + +========= ================================================== ==================== +SQLSTATE Exception Base exception +========= ================================================== ==================== +**Class 02** - No Data (this is also a warning class per the SQL standard) +--------------------------------------------------------------------------------- +``02000`` `!NoData` `!DatabaseError` +``02001`` `!NoAdditionalDynamicResultSetsReturned` `!DatabaseError` +**Class 03** - SQL Statement Not Yet Complete +--------------------------------------------------------------------------------- +``03000`` `!SqlStatementNotYetComplete` `!DatabaseError` +**Class 08** - Connection Exception +--------------------------------------------------------------------------------- +``08000`` `!ConnectionException` `!OperationalError` +``08001`` `!SqlclientUnableToEstablishSqlconnection` `!OperationalError` +``08003`` `!ConnectionDoesNotExist` `!OperationalError` +``08004`` `!SqlserverRejectedEstablishmentOfSqlconnection` `!OperationalError` +``08006`` `!ConnectionFailure` `!OperationalError` +``08007`` `!TransactionResolutionUnknown` `!OperationalError` +``08P01`` `!ProtocolViolation` `!OperationalError` +**Class 09** - Triggered Action Exception +--------------------------------------------------------------------------------- +``09000`` `!TriggeredActionException` `!DatabaseError` +**Class 0A** - Feature Not Supported +--------------------------------------------------------------------------------- +``0A000`` `!FeatureNotSupported` `!NotSupportedError` +**Class 0B** - Invalid Transaction Initiation +--------------------------------------------------------------------------------- +``0B000`` `!InvalidTransactionInitiation` `!DatabaseError` +**Class 0F** - Locator Exception +--------------------------------------------------------------------------------- +``0F000`` `!LocatorException` `!DatabaseError` +``0F001`` `!InvalidLocatorSpecification` `!DatabaseError` +**Class 0L** - Invalid Grantor +--------------------------------------------------------------------------------- +``0L000`` `!InvalidGrantor` `!DatabaseError` +``0LP01`` `!InvalidGrantOperation` `!DatabaseError` +**Class 0P** - Invalid Role Specification +--------------------------------------------------------------------------------- +``0P000`` `!InvalidRoleSpecification` `!DatabaseError` +**Class 0Z** - Diagnostics Exception +--------------------------------------------------------------------------------- +``0Z000`` `!DiagnosticsException` `!DatabaseError` +``0Z002`` `!StackedDiagnosticsAccessedWithoutActiveHandler` `!DatabaseError` +**Class 20** - Case Not Found +--------------------------------------------------------------------------------- +``20000`` `!CaseNotFound` `!ProgrammingError` +**Class 21** - Cardinality Violation +--------------------------------------------------------------------------------- +``21000`` `!CardinalityViolation` `!ProgrammingError` +**Class 22** - Data Exception +--------------------------------------------------------------------------------- +``22000`` `!DataException` `!DataError` +``22001`` `!StringDataRightTruncation` `!DataError` +``22002`` `!NullValueNoIndicatorParameter` `!DataError` +``22003`` `!NumericValueOutOfRange` `!DataError` +``22004`` `!NullValueNotAllowed` `!DataError` +``22005`` `!ErrorInAssignment` `!DataError` +``22007`` `!InvalidDatetimeFormat` `!DataError` +``22008`` `!DatetimeFieldOverflow` `!DataError` +``22009`` `!InvalidTimeZoneDisplacementValue` `!DataError` +``2200B`` `!EscapeCharacterConflict` `!DataError` +``2200C`` `!InvalidUseOfEscapeCharacter` `!DataError` +``2200D`` `!InvalidEscapeOctet` `!DataError` +``2200F`` `!ZeroLengthCharacterString` `!DataError` +``2200G`` `!MostSpecificTypeMismatch` `!DataError` +``2200H`` `!SequenceGeneratorLimitExceeded` `!DataError` +``2200L`` `!NotAnXmlDocument` `!DataError` +``2200M`` `!InvalidXmlDocument` `!DataError` +``2200N`` `!InvalidXmlContent` `!DataError` +``2200S`` `!InvalidXmlComment` `!DataError` +``2200T`` `!InvalidXmlProcessingInstruction` `!DataError` +``22010`` `!InvalidIndicatorParameterValue` `!DataError` +``22011`` `!SubstringError` `!DataError` +``22012`` `!DivisionByZero` `!DataError` +``22013`` `!InvalidPrecedingOrFollowingSize` `!DataError` +``22014`` `!InvalidArgumentForNtileFunction` `!DataError` +``22015`` `!IntervalFieldOverflow` `!DataError` +``22016`` `!InvalidArgumentForNthValueFunction` `!DataError` +``22018`` `!InvalidCharacterValueForCast` `!DataError` +``22019`` `!InvalidEscapeCharacter` `!DataError` +``2201B`` `!InvalidRegularExpression` `!DataError` +``2201E`` `!InvalidArgumentForLogarithm` `!DataError` +``2201F`` `!InvalidArgumentForPowerFunction` `!DataError` +``2201G`` `!InvalidArgumentForWidthBucketFunction` `!DataError` +``2201W`` `!InvalidRowCountInLimitClause` `!DataError` +``2201X`` `!InvalidRowCountInResultOffsetClause` `!DataError` +``22021`` `!CharacterNotInRepertoire` `!DataError` +``22022`` `!IndicatorOverflow` `!DataError` +``22023`` `!InvalidParameterValue` `!DataError` +``22024`` `!UnterminatedCString` `!DataError` +``22025`` `!InvalidEscapeSequence` `!DataError` +``22026`` `!StringDataLengthMismatch` `!DataError` +``22027`` `!TrimError` `!DataError` +``2202E`` `!ArraySubscriptError` `!DataError` +``2202G`` `!InvalidTablesampleRepeat` `!DataError` +``2202H`` `!InvalidTablesampleArgument` `!DataError` +``22030`` `!DuplicateJsonObjectKeyValue` `!DataError` +``22031`` `!InvalidArgumentForSqlJsonDatetimeFunction` `!DataError` +``22032`` `!InvalidJsonText` `!DataError` +``22033`` `!InvalidSqlJsonSubscript` `!DataError` +``22034`` `!MoreThanOneSqlJsonItem` `!DataError` +``22035`` `!NoSqlJsonItem` `!DataError` +``22036`` `!NonNumericSqlJsonItem` `!DataError` +``22037`` `!NonUniqueKeysInAJsonObject` `!DataError` +``22038`` `!SingletonSqlJsonItemRequired` `!DataError` +``22039`` `!SqlJsonArrayNotFound` `!DataError` +``2203A`` `!SqlJsonMemberNotFound` `!DataError` +``2203B`` `!SqlJsonNumberNotFound` `!DataError` +``2203C`` `!SqlJsonObjectNotFound` `!DataError` +``2203D`` `!TooManyJsonArrayElements` `!DataError` +``2203E`` `!TooManyJsonObjectMembers` `!DataError` +``2203F`` `!SqlJsonScalarRequired` `!DataError` +``2203G`` `!SqlJsonItemCannotBeCastToTargetType` `!DataError` +``22P01`` `!FloatingPointException` `!DataError` +``22P02`` `!InvalidTextRepresentation` `!DataError` +``22P03`` `!InvalidBinaryRepresentation` `!DataError` +``22P04`` `!BadCopyFileFormat` `!DataError` +``22P05`` `!UntranslatableCharacter` `!DataError` +``22P06`` `!NonstandardUseOfEscapeCharacter` `!DataError` +**Class 23** - Integrity Constraint Violation +--------------------------------------------------------------------------------- +``23000`` `!IntegrityConstraintViolation` `!IntegrityError` +``23001`` `!RestrictViolation` `!IntegrityError` +``23502`` `!NotNullViolation` `!IntegrityError` +``23503`` `!ForeignKeyViolation` `!IntegrityError` +``23505`` `!UniqueViolation` `!IntegrityError` +``23514`` `!CheckViolation` `!IntegrityError` +``23P01`` `!ExclusionViolation` `!IntegrityError` +**Class 24** - Invalid Cursor State +--------------------------------------------------------------------------------- +``24000`` `!InvalidCursorState` `!InternalError` +**Class 25** - Invalid Transaction State +--------------------------------------------------------------------------------- +``25000`` `!InvalidTransactionState` `!InternalError` +``25001`` `!ActiveSqlTransaction` `!InternalError` +``25002`` `!BranchTransactionAlreadyActive` `!InternalError` +``25003`` `!InappropriateAccessModeForBranchTransaction` `!InternalError` +``25004`` `!InappropriateIsolationLevelForBranchTransaction` `!InternalError` +``25005`` `!NoActiveSqlTransactionForBranchTransaction` `!InternalError` +``25006`` `!ReadOnlySqlTransaction` `!InternalError` +``25007`` `!SchemaAndDataStatementMixingNotSupported` `!InternalError` +``25008`` `!HeldCursorRequiresSameIsolationLevel` `!InternalError` +``25P01`` `!NoActiveSqlTransaction` `!InternalError` +``25P02`` `!InFailedSqlTransaction` `!InternalError` +``25P03`` `!IdleInTransactionSessionTimeout` `!InternalError` +**Class 26** - Invalid SQL Statement Name +--------------------------------------------------------------------------------- +``26000`` `!InvalidSqlStatementName` `!ProgrammingError` +**Class 27** - Triggered Data Change Violation +--------------------------------------------------------------------------------- +``27000`` `!TriggeredDataChangeViolation` `!OperationalError` +**Class 28** - Invalid Authorization Specification +--------------------------------------------------------------------------------- +``28000`` `!InvalidAuthorizationSpecification` `!OperationalError` +``28P01`` `!InvalidPassword` `!OperationalError` +**Class 2B** - Dependent Privilege Descriptors Still Exist +--------------------------------------------------------------------------------- +``2B000`` `!DependentPrivilegeDescriptorsStillExist` `!InternalError` +``2BP01`` `!DependentObjectsStillExist` `!InternalError` +**Class 2D** - Invalid Transaction Termination +--------------------------------------------------------------------------------- +``2D000`` `!InvalidTransactionTermination` `!InternalError` +**Class 2F** - SQL Routine Exception +--------------------------------------------------------------------------------- +``2F000`` `!SqlRoutineException` `!OperationalError` +``2F002`` `!ModifyingSqlDataNotPermitted` `!OperationalError` +``2F003`` `!ProhibitedSqlStatementAttempted` `!OperationalError` +``2F004`` `!ReadingSqlDataNotPermitted` `!OperationalError` +``2F005`` `!FunctionExecutedNoReturnStatement` `!OperationalError` +**Class 34** - Invalid Cursor Name +--------------------------------------------------------------------------------- +``34000`` `!InvalidCursorName` `!ProgrammingError` +**Class 38** - External Routine Exception +--------------------------------------------------------------------------------- +``38000`` `!ExternalRoutineException` `!OperationalError` +``38001`` `!ContainingSqlNotPermitted` `!OperationalError` +``38002`` `!ModifyingSqlDataNotPermittedExt` `!OperationalError` +``38003`` `!ProhibitedSqlStatementAttemptedExt` `!OperationalError` +``38004`` `!ReadingSqlDataNotPermittedExt` `!OperationalError` +**Class 39** - External Routine Invocation Exception +--------------------------------------------------------------------------------- +``39000`` `!ExternalRoutineInvocationException` `!OperationalError` +``39001`` `!InvalidSqlstateReturned` `!OperationalError` +``39004`` `!NullValueNotAllowedExt` `!OperationalError` +``39P01`` `!TriggerProtocolViolated` `!OperationalError` +``39P02`` `!SrfProtocolViolated` `!OperationalError` +``39P03`` `!EventTriggerProtocolViolated` `!OperationalError` +**Class 3B** - Savepoint Exception +--------------------------------------------------------------------------------- +``3B000`` `!SavepointException` `!OperationalError` +``3B001`` `!InvalidSavepointSpecification` `!OperationalError` +**Class 3D** - Invalid Catalog Name +--------------------------------------------------------------------------------- +``3D000`` `!InvalidCatalogName` `!ProgrammingError` +**Class 3F** - Invalid Schema Name +--------------------------------------------------------------------------------- +``3F000`` `!InvalidSchemaName` `!ProgrammingError` +**Class 40** - Transaction Rollback +--------------------------------------------------------------------------------- +``40000`` `!TransactionRollback` `!OperationalError` +``40001`` `!SerializationFailure` `!OperationalError` +``40002`` `!TransactionIntegrityConstraintViolation` `!OperationalError` +``40003`` `!StatementCompletionUnknown` `!OperationalError` +``40P01`` `!DeadlockDetected` `!OperationalError` +**Class 42** - Syntax Error or Access Rule Violation +--------------------------------------------------------------------------------- +``42000`` `!SyntaxErrorOrAccessRuleViolation` `!ProgrammingError` +``42501`` `!InsufficientPrivilege` `!ProgrammingError` +``42601`` `!SyntaxError` `!ProgrammingError` +``42602`` `!InvalidName` `!ProgrammingError` +``42611`` `!InvalidColumnDefinition` `!ProgrammingError` +``42622`` `!NameTooLong` `!ProgrammingError` +``42701`` `!DuplicateColumn` `!ProgrammingError` +``42702`` `!AmbiguousColumn` `!ProgrammingError` +``42703`` `!UndefinedColumn` `!ProgrammingError` +``42704`` `!UndefinedObject` `!ProgrammingError` +``42710`` `!DuplicateObject` `!ProgrammingError` +``42712`` `!DuplicateAlias` `!ProgrammingError` +``42723`` `!DuplicateFunction` `!ProgrammingError` +``42725`` `!AmbiguousFunction` `!ProgrammingError` +``42803`` `!GroupingError` `!ProgrammingError` +``42804`` `!DatatypeMismatch` `!ProgrammingError` +``42809`` `!WrongObjectType` `!ProgrammingError` +``42830`` `!InvalidForeignKey` `!ProgrammingError` +``42846`` `!CannotCoerce` `!ProgrammingError` +``42883`` `!UndefinedFunction` `!ProgrammingError` +``428C9`` `!GeneratedAlways` `!ProgrammingError` +``42939`` `!ReservedName` `!ProgrammingError` +``42P01`` `!UndefinedTable` `!ProgrammingError` +``42P02`` `!UndefinedParameter` `!ProgrammingError` +``42P03`` `!DuplicateCursor` `!ProgrammingError` +``42P04`` `!DuplicateDatabase` `!ProgrammingError` +``42P05`` `!DuplicatePreparedStatement` `!ProgrammingError` +``42P06`` `!DuplicateSchema` `!ProgrammingError` +``42P07`` `!DuplicateTable` `!ProgrammingError` +``42P08`` `!AmbiguousParameter` `!ProgrammingError` +``42P09`` `!AmbiguousAlias` `!ProgrammingError` +``42P10`` `!InvalidColumnReference` `!ProgrammingError` +``42P11`` `!InvalidCursorDefinition` `!ProgrammingError` +``42P12`` `!InvalidDatabaseDefinition` `!ProgrammingError` +``42P13`` `!InvalidFunctionDefinition` `!ProgrammingError` +``42P14`` `!InvalidPreparedStatementDefinition` `!ProgrammingError` +``42P15`` `!InvalidSchemaDefinition` `!ProgrammingError` +``42P16`` `!InvalidTableDefinition` `!ProgrammingError` +``42P17`` `!InvalidObjectDefinition` `!ProgrammingError` +``42P18`` `!IndeterminateDatatype` `!ProgrammingError` +``42P19`` `!InvalidRecursion` `!ProgrammingError` +``42P20`` `!WindowingError` `!ProgrammingError` +``42P21`` `!CollationMismatch` `!ProgrammingError` +``42P22`` `!IndeterminateCollation` `!ProgrammingError` +**Class 44** - WITH CHECK OPTION Violation +--------------------------------------------------------------------------------- +``44000`` `!WithCheckOptionViolation` `!ProgrammingError` +**Class 53** - Insufficient Resources +--------------------------------------------------------------------------------- +``53000`` `!InsufficientResources` `!OperationalError` +``53100`` `!DiskFull` `!OperationalError` +``53200`` `!OutOfMemory` `!OperationalError` +``53300`` `!TooManyConnections` `!OperationalError` +``53400`` `!ConfigurationLimitExceeded` `!OperationalError` +**Class 54** - Program Limit Exceeded +--------------------------------------------------------------------------------- +``54000`` `!ProgramLimitExceeded` `!OperationalError` +``54001`` `!StatementTooComplex` `!OperationalError` +``54011`` `!TooManyColumns` `!OperationalError` +``54023`` `!TooManyArguments` `!OperationalError` +**Class 55** - Object Not In Prerequisite State +--------------------------------------------------------------------------------- +``55000`` `!ObjectNotInPrerequisiteState` `!OperationalError` +``55006`` `!ObjectInUse` `!OperationalError` +``55P02`` `!CantChangeRuntimeParam` `!OperationalError` +``55P03`` `!LockNotAvailable` `!OperationalError` +``55P04`` `!UnsafeNewEnumValueUsage` `!OperationalError` +**Class 57** - Operator Intervention +--------------------------------------------------------------------------------- +``57000`` `!OperatorIntervention` `!OperationalError` +``57014`` `!QueryCanceled` `!OperationalError` +``57P01`` `!AdminShutdown` `!OperationalError` +``57P02`` `!CrashShutdown` `!OperationalError` +``57P03`` `!CannotConnectNow` `!OperationalError` +``57P04`` `!DatabaseDropped` `!OperationalError` +``57P05`` `!IdleSessionTimeout` `!OperationalError` +**Class 58** - System Error (errors external to PostgreSQL itself) +--------------------------------------------------------------------------------- +``58000`` `!SystemError` `!OperationalError` +``58030`` `!IoError` `!OperationalError` +``58P01`` `!UndefinedFile` `!OperationalError` +``58P02`` `!DuplicateFile` `!OperationalError` +**Class 72** - Snapshot Failure +--------------------------------------------------------------------------------- +``72000`` `!SnapshotTooOld` `!DatabaseError` +**Class F0** - Configuration File Error +--------------------------------------------------------------------------------- +``F0000`` `!ConfigFileError` `!OperationalError` +``F0001`` `!LockFileExists` `!OperationalError` +**Class HV** - Foreign Data Wrapper Error (SQL/MED) +--------------------------------------------------------------------------------- +``HV000`` `!FdwError` `!OperationalError` +``HV001`` `!FdwOutOfMemory` `!OperationalError` +``HV002`` `!FdwDynamicParameterValueNeeded` `!OperationalError` +``HV004`` `!FdwInvalidDataType` `!OperationalError` +``HV005`` `!FdwColumnNameNotFound` `!OperationalError` +``HV006`` `!FdwInvalidDataTypeDescriptors` `!OperationalError` +``HV007`` `!FdwInvalidColumnName` `!OperationalError` +``HV008`` `!FdwInvalidColumnNumber` `!OperationalError` +``HV009`` `!FdwInvalidUseOfNullPointer` `!OperationalError` +``HV00A`` `!FdwInvalidStringFormat` `!OperationalError` +``HV00B`` `!FdwInvalidHandle` `!OperationalError` +``HV00C`` `!FdwInvalidOptionIndex` `!OperationalError` +``HV00D`` `!FdwInvalidOptionName` `!OperationalError` +``HV00J`` `!FdwOptionNameNotFound` `!OperationalError` +``HV00K`` `!FdwReplyHandle` `!OperationalError` +``HV00L`` `!FdwUnableToCreateExecution` `!OperationalError` +``HV00M`` `!FdwUnableToCreateReply` `!OperationalError` +``HV00N`` `!FdwUnableToEstablishConnection` `!OperationalError` +``HV00P`` `!FdwNoSchemas` `!OperationalError` +``HV00Q`` `!FdwSchemaNotFound` `!OperationalError` +``HV00R`` `!FdwTableNotFound` `!OperationalError` +``HV010`` `!FdwFunctionSequenceError` `!OperationalError` +``HV014`` `!FdwTooManyHandles` `!OperationalError` +``HV021`` `!FdwInconsistentDescriptorInformation` `!OperationalError` +``HV024`` `!FdwInvalidAttributeValue` `!OperationalError` +``HV090`` `!FdwInvalidStringLengthOrBufferLength` `!OperationalError` +``HV091`` `!FdwInvalidDescriptorFieldIdentifier` `!OperationalError` +**Class P0** - PL/pgSQL Error +--------------------------------------------------------------------------------- +``P0000`` `!PlpgsqlError` `!ProgrammingError` +``P0001`` `!RaiseException` `!ProgrammingError` +``P0002`` `!NoDataFound` `!ProgrammingError` +``P0003`` `!TooManyRows` `!ProgrammingError` +``P0004`` `!AssertFailure` `!ProgrammingError` +**Class XX** - Internal Error +--------------------------------------------------------------------------------- +``XX000`` `!InternalError_` `!InternalError` +``XX001`` `!DataCorrupted` `!InternalError` +``XX002`` `!IndexCorrupted` `!InternalError` +========= ================================================== ==================== + +.. autogenerated: end + +.. versionadded:: 3.1.4 + Exception `!SqlJsonItemCannotBeCastToTargetType`, introduced in PostgreSQL + 15. diff --git a/docs/api/index.rst b/docs/api/index.rst new file mode 100644 index 0000000..b99550d --- /dev/null +++ b/docs/api/index.rst @@ -0,0 +1,29 @@ +Psycopg 3 API +============= + +.. _api: + +This sections is a reference for all the public objects exposed by the +`psycopg` module. For a more conceptual description you can take a look at +:ref:`basic` and :ref:`advanced`. + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + module + connections + cursors + copy + objects + sql + rows + errors + pool + conninfo + adapt + types + abc + pq + crdb + dns diff --git a/docs/api/module.rst b/docs/api/module.rst new file mode 100644 index 0000000..3c3d3c4 --- /dev/null +++ b/docs/api/module.rst @@ -0,0 +1,59 @@ +The `!psycopg` module +===================== + +Psycopg implements the `Python Database DB API 2.0 specification`__. As such +it also exposes the `module-level objects`__ required by the specifications. + +.. __: https://www.python.org/dev/peps/pep-0249/ +.. __: https://www.python.org/dev/peps/pep-0249/#module-interface + +.. module:: psycopg + +.. autofunction:: connect + + This is an alias of the class method `Connection.connect`: see its + documentation for details. + + If you need an asynchronous connection use `AsyncConnection.connect` + instead. + + +.. rubric:: Exceptions + +The standard `DBAPI exceptions`__ are exposed both by the `!psycopg` module +and by the `psycopg.errors` module. The latter also exposes more specific +exceptions, mapping to the database error states (see +:ref:`sqlstate-exceptions`). + +.. __: https://www.python.org/dev/peps/pep-0249/#exceptions + +.. parsed-literal:: + + `!Exception` + \|__ `Warning` + \|__ `Error` + \|__ `InterfaceError` + \|__ `DatabaseError` + \|__ `DataError` + \|__ `OperationalError` + \|__ `IntegrityError` + \|__ `InternalError` + \|__ `ProgrammingError` + \|__ `NotSupportedError` + + +.. data:: adapters + + The default adapters map establishing how Python and PostgreSQL types are + converted into each other. + + This map is used as a template when new connections are created, using + `psycopg.connect()`. Its `~psycopg.adapt.AdaptersMap.types` attribute is a + `~psycopg.types.TypesRegistry` containing information about every + PostgreSQL builtin type, useful for adaptation customisation (see + :ref:`adaptation`):: + + >>> psycopg.adapters.types["int4"] + <TypeInfo: int4 (oid: 23, array oid: 1007)> + + :type: `~psycopg.adapt.AdaptersMap` diff --git a/docs/api/objects.rst b/docs/api/objects.rst new file mode 100644 index 0000000..f085ed9 --- /dev/null +++ b/docs/api/objects.rst @@ -0,0 +1,256 @@ +.. currentmodule:: psycopg + +Other top-level objects +======================= + +Connection information +---------------------- + +.. autoclass:: ConnectionInfo() + + The object is usually returned by `Connection.info`. + + .. autoattribute:: dsn + + .. note:: The `get_parameters()` method returns the same information + as a dict. + + .. autoattribute:: status + + The status can be one of a number of values. However, only two of + these are seen outside of an asynchronous connection procedure: + `~pq.ConnStatus.OK` and `~pq.ConnStatus.BAD`. A good connection to the + database has the status `!OK`. Ordinarily, an `!OK` status will remain + so until `Connection.close()`, but a communications failure might + result in the status changing to `!BAD` prematurely. + + .. autoattribute:: transaction_status + + The status can be `~pq.TransactionStatus.IDLE` (currently idle), + `~pq.TransactionStatus.ACTIVE` (a command is in progress), + `~pq.TransactionStatus.INTRANS` (idle, in a valid transaction block), + or `~pq.TransactionStatus.INERROR` (idle, in a failed transaction + block). `~pq.TransactionStatus.UNKNOWN` is reported if the connection + is bad. `!ACTIVE` is reported only when a query has been sent to the + server and not yet completed. + + .. autoattribute:: pipeline_status + + .. autoattribute:: backend_pid + .. autoattribute:: vendor + + Normally it is `PostgreSQL`; it may be different if connected to + a different database. + + .. versionadded:: 3.1 + + .. autoattribute:: server_version + + The number is formed by converting the major, minor, and revision + numbers into two-decimal-digit numbers and appending them together. + Starting from PostgreSQL 10 the minor version was dropped, so the + second group of digits is always 00. For example, version 9.3.5 is + returned as 90305, version 10.2 as 100002. + + .. autoattribute:: error_message + + .. automethod:: get_parameters + + .. note:: The `dsn` attribute returns the same information in the form + as a string. + + .. autoattribute:: timezone + + .. code:: pycon + + >>> conn.info.timezone + zoneinfo.ZoneInfo(key='Europe/Rome') + + .. autoattribute:: host + + This can be a host name, an IP address, or a directory path if the + connection is via Unix socket. (The path case can be distinguished + because it will always be an absolute path, beginning with ``/``.) + + .. autoattribute:: hostaddr + + Only available if the libpq used is at least from PostgreSQL 12. + Raise `~psycopg.NotSupportedError` otherwise. + + .. autoattribute:: port + .. autoattribute:: dbname + .. autoattribute:: user + .. autoattribute:: password + .. autoattribute:: options + .. automethod:: parameter_status + + Example of parameters are ``server_version``, + ``standard_conforming_strings``... See :pq:`PQparameterStatus()` for + all the available parameters. + + .. autoattribute:: encoding + + The value returned is always normalized to the Python codec + `~codecs.CodecInfo.name`:: + + conn.execute("SET client_encoding TO LATIN9") + conn.info.encoding + 'iso8859-15' + + A few PostgreSQL encodings are not available in Python and cannot be + selected (currently ``EUC_TW``, ``MULE_INTERNAL``). The PostgreSQL + ``SQL_ASCII`` encoding has the special meaning of "no encoding": see + :ref:`adapt-string` for details. + + .. seealso:: + + The `PostgreSQL supported encodings`__. + + .. __: https://www.postgresql.org/docs/current/multibyte.html + + +The description `Column` object +------------------------------- + +.. autoclass:: Column() + + An object describing a column of data from a database result, `as described + by the DBAPI`__, so it can also be unpacked as a 7-items tuple. + + The object is returned by `Cursor.description`. + + .. __: https://www.python.org/dev/peps/pep-0249/#description + + .. autoattribute:: name + .. autoattribute:: type_code + .. autoattribute:: display_size + .. autoattribute:: internal_size + .. autoattribute:: precision + .. autoattribute:: scale + + +Notifications +------------- + +.. autoclass:: Notify() + + The object is usually returned by `Connection.notifies()`. + + .. attribute:: channel + :type: str + + The name of the channel on which the notification was received. + + .. attribute:: payload + :type: str + + The message attached to the notification. + + .. attribute:: pid + :type: int + + The PID of the backend process which sent the notification. + + +Pipeline-related objects +------------------------ + +See :ref:`pipeline-mode` for details. + +.. autoclass:: Pipeline + + This objects is returned by `Connection.pipeline()`. + + .. automethod:: sync + .. automethod:: is_supported + + +.. autoclass:: AsyncPipeline + + This objects is returned by `AsyncConnection.pipeline()`. + + .. automethod:: sync + + +Transaction-related objects +--------------------------- + +See :ref:`transactions` for details about these objects. + +.. autoclass:: IsolationLevel + :members: + + The value is usually used with the `Connection.isolation_level` property. + + Check the PostgreSQL documentation for a description of the effects of the + different `levels of transaction isolation`__. + + .. __: https://www.postgresql.org/docs/current/transaction-iso.html + + +.. autoclass:: Transaction() + + .. autoattribute:: savepoint_name + .. autoattribute:: connection + + +.. autoclass:: AsyncTransaction() + + .. autoattribute:: connection + + +.. autoexception:: Rollback + + It can be used as: + + - ``raise Rollback``: roll back the operation that happened in the current + transaction block and continue the program after the block. + + - ``raise Rollback()``: same effect as above + + - :samp:`raise Rollback({tx})`: roll back any operation that happened in + the `Transaction` `!tx` (returned by a statement such as :samp:`with + conn.transaction() as {tx}:` and all the blocks nested within. The + program will continue after the `!tx` block. + + +Two-Phase Commit related objects +-------------------------------- + +.. autoclass:: Xid() + + See :ref:`two-phase-commit` for details. + + .. autoattribute:: format_id + + Format Identifier of the two-phase transaction. + + .. autoattribute:: gtrid + + Global Transaction Identifier of the two-phase transaction. + + If the Xid doesn't follow the XA standard, it will be the PostgreSQL + ID of the transaction (in which case `format_id` and `bqual` will be + `!None`). + + .. autoattribute:: bqual + + Branch Qualifier of the two-phase transaction. + + .. autoattribute:: prepared + + Timestamp at which the transaction was prepared for commit. + + Only available on transactions recovered by `~Connection.tpc_recover()`. + + .. autoattribute:: owner + + Named of the user that executed the transaction. + + Only available on recovered transactions. + + .. autoattribute:: database + + Named of the database in which the transaction was executed. + + Only available on recovered transactions. diff --git a/docs/api/pool.rst b/docs/api/pool.rst new file mode 100644 index 0000000..76ccc74 --- /dev/null +++ b/docs/api/pool.rst @@ -0,0 +1,331 @@ +`!psycopg_pool` -- Connection pool implementations +================================================== + +.. index:: + double: Connection; Pool + +.. module:: psycopg_pool + +A connection pool is an object used to create and maintain a limited amount of +PostgreSQL connections, reducing the time requested by the program to obtain a +working connection and allowing an arbitrary large number of concurrent +threads or tasks to use a controlled amount of resources on the server. See +:ref:`connection-pools` for more details and usage pattern. + +This package exposes a few connection pool classes: + +- `ConnectionPool` is a synchronous connection pool yielding + `~psycopg.Connection` objects and can be used by multithread applications. + +- `AsyncConnectionPool` has an interface similar to `!ConnectionPool`, but + with `asyncio` functions replacing blocking functions, and yields + `~psycopg.AsyncConnection` instances. + +- `NullConnectionPool` is a `!ConnectionPool` subclass exposing the same + interface of its parent, but not keeping any unused connection in its state. + See :ref:`null-pool` for details about related use cases. + +- `AsyncNullConnectionPool` has the same behaviour of the + `!NullConnectionPool`, but with the same async interface of the + `!AsyncConnectionPool`. + +.. note:: The `!psycopg_pool` package is distributed separately from the main + `psycopg` package: use ``pip install "psycopg[pool]"``, or ``pip install + psycopg_pool``, to make it available. See :ref:`pool-installation`. + + The version numbers indicated in this page refer to the `!psycopg_pool` + package, not to `psycopg`. + + +The `!ConnectionPool` class +--------------------------- + +.. autoclass:: ConnectionPool + + This class implements a connection pool serving `~psycopg.Connection` + instances (or subclasses). The constructor has *alot* of arguments, but + only `!conninfo` and `!min_size` are the fundamental ones, all the other + arguments have meaningful defaults and can probably be tweaked later, if + required. + + :param conninfo: The connection string. See + `~psycopg.Connection.connect()` for details. + :type conninfo: `!str` + + :param min_size: The minimum number of connection the pool will hold. The + pool will actively try to create new connections if some + are lost (closed, broken) and will try to never go below + `!min_size`. + :type min_size: `!int`, default: 4 + + :param max_size: The maximum number of connections the pool will hold. If + `!None`, or equal to `!min_size`, the pool will not grow or + shrink. If larger than `!min_size`, the pool can grow if + more than `!min_size` connections are requested at the same + time and will shrink back after the extra connections have + been unused for more than `!max_idle` seconds. + :type max_size: `!int`, default: `!None` + + :param kwargs: Extra arguments to pass to `!connect()`. Note that this is + *one dict argument* of the pool constructor, which is + expanded as `connect()` keyword parameters. + + :type kwargs: `!dict` + + :param connection_class: The class of the connections to serve. It should + be a `!Connection` subclass. + :type connection_class: `!type`, default: `~psycopg.Connection` + + :param open: If `!True`, open the pool, creating the required connections, + on init. If `!False`, open the pool when `!open()` is called or + when the pool context is entered. See the `open()` method + documentation for more details. + :type open: `!bool`, default: `!True` + + :param configure: A callback to configure a connection after creation. + Useful, for instance, to configure its adapters. If the + connection is used to run internal queries (to inspect the + database) make sure to close an eventual transaction + before leaving the function. + :type configure: `Callable[[Connection], None]` + + :param reset: A callback to reset a function after it has been returned to + the pool. The connection is guaranteed to be passed to the + `!reset()` function in "idle" state (no transaction). When + leaving the `!reset()` function the connection must be left in + *idle* state, otherwise it is discarded. + :type reset: `Callable[[Connection], None]` + + :param name: An optional name to give to the pool, useful, for instance, to + identify it in the logs if more than one pool is used. if not + specified pick a sequential name such as ``pool-1``, + ``pool-2``, etc. + :type name: `!str` + + :param timeout: The default maximum time in seconds that a client can wait + to receive a connection from the pool (using `connection()` + or `getconn()`). Note that these methods allow to override + the `!timeout` default. + :type timeout: `!float`, default: 30 seconds + + :param max_waiting: Maximum number of requests that can be queued to the + pool, after which new requests will fail, raising + `TooManyRequests`. 0 means no queue limit. + :type max_waiting: `!int`, default: 0 + + :param max_lifetime: The maximum lifetime of a connection in the pool, in + seconds. Connections used for longer get closed and + replaced by a new one. The amount is reduced by a + random 10% to avoid mass eviction. + :type max_lifetime: `!float`, default: 1 hour + + :param max_idle: Maximum time, in seconds, that a connection can stay unused + in the pool before being closed, and the pool shrunk. This + only happens to connections more than `!min_size`, if + `!max_size` allowed the pool to grow. + :type max_idle: `!float`, default: 10 minutes + + :param reconnect_timeout: Maximum time, in seconds, the pool will try to + create a connection. If a connection attempt + fails, the pool will try to reconnect a few + times, using an exponential backoff and some + random factor to avoid mass attempts. If repeated + attempts fail, after `!reconnect_timeout` second + the connection attempt is aborted and the + `!reconnect_failed()` callback invoked. + :type reconnect_timeout: `!float`, default: 5 minutes + + :param reconnect_failed: Callback invoked if an attempt to create a new + connection fails for more than `!reconnect_timeout` + seconds. The user may decide, for instance, to + terminate the program (executing `sys.exit()`). + By default don't do anything: restart a new + connection attempt (if the number of connection + fell below `!min_size`). + :type reconnect_failed: ``Callable[[ConnectionPool], None]`` + + :param num_workers: Number of background worker threads used to maintain the + pool state. Background workers are used for example to + create new connections and to clean up connections when + they are returned to the pool. + :type num_workers: `!int`, default: 3 + + .. versionchanged:: 3.1 + + added `!open` parameter to init method. + + .. note:: In a future version, the default value for the `!open` parameter + might be changed to `!False`. If you rely on this behaviour (e.g. if + you don't use the pool as a context manager) you might want to specify + this parameter explicitly. + + .. automethod:: connection + + .. code:: python + + with my_pool.connection() as conn: + conn.execute(...) + + # the connection is now back in the pool + + .. automethod:: open + + .. versionadded:: 3.1 + + + .. automethod:: close + + .. note:: + + The pool can be also used as a context manager, in which case it will + be opened (if necessary) on entering the block and closed on exiting it: + + .. code:: python + + with ConnectionPool(...) as pool: + # code using the pool + + .. automethod:: wait + + .. attribute:: name + :type: str + + The name of the pool set on creation, or automatically generated if not + set. + + .. autoattribute:: min_size + .. autoattribute:: max_size + + The current minimum and maximum size of the pool. Use `resize()` to + change them at runtime. + + .. automethod:: resize + .. automethod:: check + .. automethod:: get_stats + .. automethod:: pop_stats + + See :ref:`pool-stats` for the metrics returned. + + .. rubric:: Functionalities you may not need + + .. automethod:: getconn + .. automethod:: putconn + + +Pool exceptions +--------------- + +.. autoclass:: PoolTimeout() + + Subclass of `~psycopg.OperationalError` + +.. autoclass:: PoolClosed() + + Subclass of `~psycopg.OperationalError` + +.. autoclass:: TooManyRequests() + + Subclass of `~psycopg.OperationalError` + + +The `!AsyncConnectionPool` class +-------------------------------- + +`!AsyncConnectionPool` has a very similar interface to the `ConnectionPool` +class but its blocking methods are implemented as `!async` coroutines. It +returns instances of `~psycopg.AsyncConnection`, or of its subclass if +specified so in the `!connection_class` parameter. + +Only the functions with different signature from `!ConnectionPool` are +listed here. + +.. autoclass:: AsyncConnectionPool + + :param connection_class: The class of the connections to serve. It should + be an `!AsyncConnection` subclass. + :type connection_class: `!type`, default: `~psycopg.AsyncConnection` + + :param configure: A callback to configure a connection after creation. + :type configure: `async Callable[[AsyncConnection], None]` + + :param reset: A callback to reset a function after it has been returned to + the pool. + :type reset: `async Callable[[AsyncConnection], None]` + + .. automethod:: connection + + .. code:: python + + async with my_pool.connection() as conn: + await conn.execute(...) + + # the connection is now back in the pool + + .. automethod:: open + .. automethod:: close + + .. note:: + + The pool can be also used as an async context manager, in which case it + will be opened (if necessary) on entering the block and closed on + exiting it: + + .. code:: python + + async with AsyncConnectionPool(...) as pool: + # code using the pool + + All the other constructor parameters are the same of `!ConnectionPool`. + + .. automethod:: wait + .. automethod:: resize + .. automethod:: check + .. automethod:: getconn + .. automethod:: putconn + + +Null connection pools +--------------------- + +.. versionadded:: 3.1 + +The `NullConnectionPool` is a `ConnectionPool` subclass which doesn't create +connections preemptively and doesn't keep unused connections in its state. See +:ref:`null-pool` for further details. + +The interface of the object is entirely compatible with its parent class. Its +behaviour is similar, with the following differences: + +.. autoclass:: NullConnectionPool + + All the other constructor parameters are the same as in `ConnectionPool`. + + :param min_size: Always 0, cannot be changed. + :type min_size: `!int`, default: 0 + + :param max_size: If None or 0, create a new connection at every request, + without a maximum. If greater than 0, don't create more + than `!max_size` connections and queue the waiting clients. + :type max_size: `!int`, default: None + + :param reset: It is only called when there are waiting clients in the + queue, before giving them a connection already open. If no + client is waiting, the connection is closed and discarded + without a fuss. + :type reset: `Callable[[Connection], None]` + + :param max_idle: Ignored, as null pools don't leave idle connections + sitting around. + + .. automethod:: wait + .. automethod:: resize + .. automethod:: check + + +The `AsyncNullConnectionPool` is, similarly, an `AsyncConnectionPool` subclass +with the same behaviour of the `NullConnectionPool`. + +.. autoclass:: AsyncNullConnectionPool + + The interface is the same of its parent class `AsyncConnectionPool`. The + behaviour is different in the same way described for `NullConnectionPool`. diff --git a/docs/api/pq.rst b/docs/api/pq.rst new file mode 100644 index 0000000..3d9c033 --- /dev/null +++ b/docs/api/pq.rst @@ -0,0 +1,218 @@ +.. _psycopg.pq: + +`pq` -- libpq wrapper module +============================ + +.. index:: + single: libpq + +.. module:: psycopg.pq + +Psycopg is built around the libpq_, the PostgreSQL client library, which +performs most of the network communications and returns query results in C +structures. + +.. _libpq: https://www.postgresql.org/docs/current/libpq.html + +The low-level functions of the library are exposed by the objects in the +`!psycopg.pq` module. + + +.. _pq-impl: + +``pq`` module implementations +----------------------------- + +There are actually several implementations of the module, all offering the +same interface. Current implementations are: + +- ``python``: a pure-python implementation, implemented using the `ctypes` + module. It is less performing than the others, but it doesn't need a C + compiler to install. It requires the libpq installed in the system. + +- ``c``: a C implementation of the libpq wrapper (more precisely, implemented + in Cython_). It is much better performing than the ``python`` + implementation, however it requires development packages installed on the + client machine. It can be installed using the ``c`` extra, i.e. running + ``pip install "psycopg[c]"``. + +- ``binary``: a pre-compiled C implementation, bundled with all the required + libraries. It is the easiest option to deal with, fast to install and it + should require no development tool or client library, however it may be not + available for every platform. You can install it using the ``binary`` extra, + i.e. running ``pip install "psycopg[binary]"``. + +.. _Cython: https://cython.org/ + +The implementation currently used is available in the `~psycopg.pq.__impl__` +module constant. + +At import time, Psycopg 3 will try to use the best implementation available +and will fail if none is usable. You can force the use of a specific +implementation by exporting the env var :envvar:`PSYCOPG_IMPL`: importing the +library will fail if the requested implementation is not available:: + + $ PSYCOPG_IMPL=c python -c "import psycopg" + Traceback (most recent call last): + ... + ImportError: couldn't import requested psycopg 'c' implementation: No module named 'psycopg_c' + + +Module content +-------------- + +.. autodata:: __impl__ + + The choice of implementation is automatic but can be forced setting the + :envvar:`PSYCOPG_IMPL` env var. + + +.. autofunction:: version + + .. seealso:: the :pq:`PQlibVersion()` function + + +.. autodata:: __build_version__ + +.. autofunction:: error_message + + +Objects wrapping libpq structures and functions +----------------------------------------------- + +.. admonition:: TODO + + finish documentation + +.. autoclass:: PGconn() + + .. autoattribute:: pgconn_ptr + .. automethod:: get_cancel + .. autoattribute:: needs_password + .. autoattribute:: used_password + + .. automethod:: encrypt_password + + .. code:: python + + >>> enc = conn.info.encoding + >>> encrypted = conn.pgconn.encrypt_password(password.encode(enc), rolename.encode(enc)) + b'SCRAM-SHA-256$4096:... + + .. automethod:: trace + .. automethod:: set_trace_flags + .. automethod:: untrace + + .. code:: python + + >>> conn.pgconn.trace(sys.stderr.fileno()) + >>> conn.pgconn.set_trace_flags(pq.Trace.SUPPRESS_TIMESTAMPS | pq.Trace.REGRESS_MODE) + >>> conn.execute("select now()") + F 13 Parse "" "BEGIN" 0 + F 14 Bind "" "" 0 0 1 0 + F 6 Describe P "" + F 9 Execute "" 0 + F 4 Sync + B 4 ParseComplete + B 4 BindComplete + B 4 NoData + B 10 CommandComplete "BEGIN" + B 5 ReadyForQuery T + F 17 Query "select now()" + B 28 RowDescription 1 "now" NNNN 0 NNNN 8 -1 0 + B 39 DataRow 1 29 '2022-09-14 14:12:16.648035+02' + B 13 CommandComplete "SELECT 1" + B 5 ReadyForQuery T + <psycopg.Cursor [TUPLES_OK] [INTRANS] (database=postgres) at 0x7f18a18ba040> + >>> conn.pgconn.untrace() + + +.. autoclass:: PGresult() + + .. autoattribute:: pgresult_ptr + + +.. autoclass:: Conninfo +.. autoclass:: Escaping + +.. autoclass:: PGcancel() + :members: + + +Enumerations +------------ + +.. autoclass:: ConnStatus + :members: + + There are other values in this enum, but only `OK` and `BAD` are seen + after a connection has been established. Other statuses might only be seen + during the connection phase and are considered internal. + + .. seealso:: :pq:`PQstatus()` returns this value. + + +.. autoclass:: PollingStatus + :members: + + .. seealso:: :pq:`PQconnectPoll` for a description of these states. + + +.. autoclass:: TransactionStatus + :members: + + .. seealso:: :pq:`PQtransactionStatus` for a description of these states. + + +.. autoclass:: ExecStatus + :members: + + .. seealso:: :pq:`PQresultStatus` for a description of these states. + + +.. autoclass:: PipelineStatus + :members: + + .. seealso:: :pq:`PQpipelineStatus` for a description of these states. + + +.. autoclass:: Format + :members: + + +.. autoclass:: DiagnosticField + + Available attributes: + + .. attribute:: + SEVERITY + SEVERITY_NONLOCALIZED + SQLSTATE + MESSAGE_PRIMARY + MESSAGE_DETAIL + MESSAGE_HINT + STATEMENT_POSITION + INTERNAL_POSITION + INTERNAL_QUERY + CONTEXT + SCHEMA_NAME + TABLE_NAME + COLUMN_NAME + DATATYPE_NAME + CONSTRAINT_NAME + SOURCE_FILE + SOURCE_LINE + SOURCE_FUNCTION + + .. seealso:: :pq:`PQresultErrorField` for a description of these values. + + +.. autoclass:: Ping + :members: + + .. seealso:: :pq:`PQpingParams` for a description of these values. + +.. autoclass:: Trace + :members: + + .. seealso:: :pq:`PQsetTraceFlags` for a description of these values. diff --git a/docs/api/rows.rst b/docs/api/rows.rst new file mode 100644 index 0000000..204f1ea --- /dev/null +++ b/docs/api/rows.rst @@ -0,0 +1,74 @@ +.. _psycopg.rows: + +`rows` -- row factory implementations +===================================== + +.. module:: psycopg.rows + +The module exposes a few generic `~psycopg.RowFactory` implementation, which +can be used to retrieve data from the database in more complex structures than +the basic tuples. + +Check out :ref:`row-factories` for information about how to use these objects. + +.. autofunction:: tuple_row +.. autofunction:: dict_row +.. autofunction:: namedtuple_row +.. autofunction:: class_row + + This is not a row factory, but rather a factory of row factories. + Specifying `!row_factory=class_row(MyClass)` will create connections and + cursors returning `!MyClass` objects on fetch. + + Example:: + + from dataclasses import dataclass + import psycopg + from psycopg.rows import class_row + + @dataclass + class Person: + first_name: str + last_name: str + age: int = None + + conn = psycopg.connect() + cur = conn.cursor(row_factory=class_row(Person)) + + cur.execute("select 'John' as first_name, 'Smith' as last_name").fetchone() + # Person(first_name='John', last_name='Smith', age=None) + +.. autofunction:: args_row +.. autofunction:: kwargs_row + + +Formal rows protocols +--------------------- + +These objects can be used to describe your own rows adapter for static typing +checks, such as mypy_. + +.. _mypy: https://mypy.readthedocs.io/ + + +.. autoclass:: psycopg.rows.RowMaker() + + .. method:: __call__(values: Sequence[Any]) -> Row + + Convert a sequence of values from the database to a finished object. + + +.. autoclass:: psycopg.rows.RowFactory() + + .. method:: __call__(cursor: Cursor[Row]) -> RowMaker[Row] + + Inspect the result on a cursor and return a `RowMaker` to convert rows. + +.. autoclass:: psycopg.rows.AsyncRowFactory() + +.. autoclass:: psycopg.rows.BaseRowFactory() + +Note that it's easy to implement an object implementing both `!RowFactory` and +`!AsyncRowFactory`: usually, everything you need to implement a row factory is +to access the cursor's `~psycopg.Cursor.description`, which is provided by +both the cursor flavours. diff --git a/docs/api/sql.rst b/docs/api/sql.rst new file mode 100644 index 0000000..6959fee --- /dev/null +++ b/docs/api/sql.rst @@ -0,0 +1,151 @@ +`sql` -- SQL string composition +=============================== + +.. index:: + double: Binding; Client-Side + +.. module:: psycopg.sql + +The module contains objects and functions useful to generate SQL dynamically, +in a convenient and safe way. SQL identifiers (e.g. names of tables and +fields) cannot be passed to the `~psycopg.Cursor.execute()` method like query +arguments:: + + # This will not work + table_name = 'my_table' + cur.execute("INSERT INTO %s VALUES (%s, %s)", [table_name, 10, 20]) + +The SQL query should be composed before the arguments are merged, for +instance:: + + # This works, but it is not optimal + table_name = 'my_table' + cur.execute( + "INSERT INTO %s VALUES (%%s, %%s)" % table_name, + [10, 20]) + +This sort of works, but it is an accident waiting to happen: the table name +may be an invalid SQL literal and need quoting; even more serious is the +security problem in case the table name comes from an untrusted source. The +name should be escaped using `~psycopg.pq.Escaping.escape_identifier()`:: + + from psycopg.pq import Escaping + + # This works, but it is not optimal + table_name = 'my_table' + cur.execute( + "INSERT INTO %s VALUES (%%s, %%s)" % Escaping.escape_identifier(table_name), + [10, 20]) + +This is now safe, but it somewhat ad-hoc. In case, for some reason, it is +necessary to include a value in the query string (as opposite as in a value) +the merging rule is still different. It is also still relatively dangerous: if +`!escape_identifier()` is forgotten somewhere, the program will usually work, +but will eventually crash in the presence of a table or field name with +containing characters to escape, or will present a potentially exploitable +weakness. + +The objects exposed by the `!psycopg.sql` module allow generating SQL +statements on the fly, separating clearly the variable parts of the statement +from the query parameters:: + + from psycopg import sql + + cur.execute( + sql.SQL("INSERT INTO {} VALUES (%s, %s)") + .format(sql.Identifier('my_table')), + [10, 20]) + + +Module usage +------------ + +Usually you should express the template of your query as an `SQL` instance +with ``{}``\-style placeholders and use `~SQL.format()` to merge the variable +parts into them, all of which must be `Composable` subclasses. You can still +have ``%s``\-style placeholders in your query and pass values to +`~psycopg.Cursor.execute()`: such value placeholders will be untouched by +`!format()`:: + + query = sql.SQL("SELECT {field} FROM {table} WHERE {pkey} = %s").format( + field=sql.Identifier('my_name'), + table=sql.Identifier('some_table'), + pkey=sql.Identifier('id')) + +The resulting object is meant to be passed directly to cursor methods such as +`~psycopg.Cursor.execute()`, `~psycopg.Cursor.executemany()`, +`~psycopg.Cursor.copy()`, but can also be used to compose a query as a Python +string, using the `~Composable.as_string()` method:: + + cur.execute(query, (42,)) + full_query = query.as_string(cur) + +If part of your query is a variable sequence of arguments, such as a +comma-separated list of field names, you can use the `SQL.join()` method to +pass them to the query:: + + query = sql.SQL("SELECT {fields} FROM {table}").format( + fields=sql.SQL(',').join([ + sql.Identifier('field1'), + sql.Identifier('field2'), + sql.Identifier('field3'), + ]), + table=sql.Identifier('some_table')) + + +`!sql` objects +-------------- + +The `!sql` objects are in the following inheritance hierarchy: + +| `Composable`: the base class exposing the common interface +| ``|__`` `SQL`: a literal snippet of an SQL query +| ``|__`` `Identifier`: a PostgreSQL identifier or dot-separated sequence of identifiers +| ``|__`` `Literal`: a value hardcoded into a query +| ``|__`` `Placeholder`: a `%s`\ -style placeholder whose value will be added later e.g. by `~psycopg.Cursor.execute()` +| ``|__`` `Composed`: a sequence of `!Composable` instances. + + +.. autoclass:: Composable() + + .. automethod:: as_bytes + .. automethod:: as_string + + +.. autoclass:: SQL + + .. versionchanged:: 3.1 + + The input object should be a `~typing.LiteralString`. See :pep:`675` + for details. + + .. automethod:: format + + .. automethod:: join + + +.. autoclass:: Identifier + +.. autoclass:: Literal + + .. versionchanged:: 3.1 + Add a type cast to the representation if useful in ambiguous context + (e.g. ``'2000-01-01'::date``) + +.. autoclass:: Placeholder + +.. autoclass:: Composed + + .. automethod:: join + + +Utility functions +----------------- + +.. autofunction:: quote + +.. data:: + NULL + DEFAULT + + `sql.SQL` objects often useful in queries. diff --git a/docs/api/types.rst b/docs/api/types.rst new file mode 100644 index 0000000..f04659e --- /dev/null +++ b/docs/api/types.rst @@ -0,0 +1,168 @@ +.. currentmodule:: psycopg.types + +.. _psycopg.types: + +`!types` -- Types information and adapters +========================================== + +.. module:: psycopg.types + +The `!psycopg.types` package exposes: + +- objects to describe PostgreSQL types, such as `TypeInfo`, `TypesRegistry`, + to help or :ref:`customise the types conversion <adaptation>`; + +- concrete implementations of `~psycopg.abc.Loader` and `~psycopg.abc.Dumper` + protocols to :ref:`handle builtin data types <types-adaptation>`; + +- helper objects to represent PostgreSQL data types which :ref:`don't have a + straightforward Python representation <extra-adaptation>`, such as + `~range.Range`. + + +Types information +----------------- + +The `TypeInfo` object describes simple information about a PostgreSQL data +type, such as its name, oid and array oid. `!TypeInfo` subclasses may hold more +information, for instance the components of a composite type. + +You can use `TypeInfo.fetch()` to query information from a database catalog, +which is then used by helper functions, such as +`~psycopg.types.hstore.register_hstore()`, to register adapters on types whose +OID is not known upfront or to create more specialised adapters. + +The `!TypeInfo` object doesn't instruct Psycopg to convert a PostgreSQL type +into a Python type: this is the role of a `~psycopg.abc.Loader`. However it +can extend the behaviour of other adapters: if you create a loader for +`!MyType`, using the `TypeInfo` information, Psycopg will be able to manage +seamlessly arrays of `!MyType` or ranges and composite types using `!MyType` +as a subtype. + +.. seealso:: :ref:`adaptation` describes how to convert from Python objects to + PostgreSQL types and back. + +.. code:: python + + from psycopg.adapt import Loader + from psycopg.types import TypeInfo + + t = TypeInfo.fetch(conn, "mytype") + t.register(conn) + + for record in conn.execute("SELECT mytypearray FROM mytable"): + # records will return lists of "mytype" as string + + class MyTypeLoader(Loader): + def load(self, data): + # parse the data and return a MyType instance + + conn.adapters.register_loader("mytype", MyTypeLoader) + + for record in conn.execute("SELECT mytypearray FROM mytable"): + # records will return lists of MyType instances + + +.. autoclass:: TypeInfo + + .. method:: fetch(conn, name) + :classmethod: + + .. method:: fetch(aconn, name) + :classmethod: + :async: + :noindex: + + Query a system catalog to read information about a type. + + :param conn: the connection to query + :type conn: ~psycopg.Connection or ~psycopg.AsyncConnection + :param name: the name of the type to query. It can include a schema + name. + :type name: `!str` or `~psycopg.sql.Identifier` + :return: a `!TypeInfo` object (or subclass) populated with the type + information, `!None` if not found. + + If the connection is async, `!fetch()` will behave as a coroutine and + the caller will need to `!await` on it to get the result:: + + t = await TypeInfo.fetch(aconn, "mytype") + + .. automethod:: register + + :param context: the context where the type is registered, for instance + a `~psycopg.Connection` or `~psycopg.Cursor`. `!None` registers + the `!TypeInfo` globally. + :type context: Optional[~psycopg.abc.AdaptContext] + + Registering the `TypeInfo` in a context allows the adapters of that + context to look up type information: for instance it allows to + recognise automatically arrays of that type and load them from the + database as a list of the base type. + + +In order to get information about dynamic PostgreSQL types, Psycopg offers a +few `!TypeInfo` subclasses, whose `!fetch()` method can extract more complete +information about the type, such as `~psycopg.types.composite.CompositeInfo`, +`~psycopg.types.range.RangeInfo`, `~psycopg.types.multirange.MultirangeInfo`, +`~psycopg.types.enum.EnumInfo`. + +`!TypeInfo` objects are collected in `TypesRegistry` instances, which help type +information lookup. Every `~psycopg.adapt.AdaptersMap` exposes its type map on +its `~psycopg.adapt.AdaptersMap.types` attribute. + +.. autoclass:: TypesRegistry + + `!TypeRegistry` instances are typically exposed by + `~psycopg.adapt.AdaptersMap` objects in adapt contexts such as + `~psycopg.Connection` or `~psycopg.Cursor` (e.g. `!conn.adapters.types`). + + The global registry, from which the others inherit from, is available as + `psycopg.adapters`\ `!.types`. + + .. automethod:: __getitem__ + + .. code:: python + + >>> import psycopg + + >>> psycopg.adapters.types["text"] + <TypeInfo: text (oid: 25, array oid: 1009)> + + >>> psycopg.adapters.types[23] + <TypeInfo: int4 (oid: 23, array oid: 1007)> + + .. automethod:: get + + .. automethod:: get_oid + + .. code:: python + + >>> psycopg.adapters.types.get_oid("text[]") + 1009 + + .. automethod:: get_by_subtype + + +.. _json-adapters: + +JSON adapters +------------- + +See :ref:`adapt-json` for details. + +.. currentmodule:: psycopg.types.json + +.. autoclass:: Json +.. autoclass:: Jsonb + +Wrappers to signal to convert `!obj` to a json or jsonb PostgreSQL value. + +Any object supported by the underlying `!dumps()` function can be wrapped. + +If a `!dumps` function is passed to the wrapper, use it to dump the wrapped +object. Otherwise use the function specified by `set_json_dumps()`. + + +.. autofunction:: set_json_dumps +.. autofunction:: set_json_loads diff --git a/docs/basic/adapt.rst b/docs/basic/adapt.rst new file mode 100644 index 0000000..1538327 --- /dev/null +++ b/docs/basic/adapt.rst @@ -0,0 +1,522 @@ +.. currentmodule:: psycopg + +.. index:: + single: Adaptation + pair: Objects; Adaptation + single: Data types; Adaptation + +.. _types-adaptation: + +Adapting basic Python types +=========================== + +Many standard Python types are adapted into SQL and returned as Python +objects when a query is executed. + +Converting the following data types between Python and PostgreSQL works +out-of-the-box and doesn't require any configuration. In case you need to +customise the conversion you should take a look at :ref:`adaptation`. + + +.. index:: + pair: Boolean; Adaptation + +.. _adapt-bool: + +Booleans adaptation +------------------- + +Python `bool` values `!True` and `!False` are converted to the equivalent +`PostgreSQL boolean type`__:: + + >>> cur.execute("SELECT %s, %s", (True, False)) + # equivalent to "SELECT true, false" + +.. __: https://www.postgresql.org/docs/current/datatype-boolean.html + + +.. index:: + single: Adaptation; numbers + single: Integer; Adaptation + single: Float; Adaptation + single: Decimal; Adaptation + +.. _adapt-numbers: + +Numbers adaptation +------------------ + +.. seealso:: + + - `PostgreSQL numeric types + <https://www.postgresql.org/docs/current/static/datatype-numeric.html>`__ + +- Python `int` values can be converted to PostgreSQL :sql:`smallint`, + :sql:`integer`, :sql:`bigint`, or :sql:`numeric`, according to their numeric + value. Psycopg will choose the smallest data type available, because + PostgreSQL can automatically cast a type up (e.g. passing a `smallint` where + PostgreSQL expect an `integer` is gladly accepted) but will not cast down + automatically (e.g. if a function has an :sql:`integer` argument, passing it + a :sql:`bigint` value will fail, even if the value is 1). + +- Python `float` values are converted to PostgreSQL :sql:`float8`. + +- Python `~decimal.Decimal` values are converted to PostgreSQL :sql:`numeric`. + +On the way back, smaller types (:sql:`int2`, :sql:`int4`, :sql:`float4`) are +promoted to the larger Python counterpart. + +.. note:: + + Sometimes you may prefer to receive :sql:`numeric` data as `!float` + instead, for performance reason or ease of manipulation: you can configure + an adapter to :ref:`cast PostgreSQL numeric to Python float + <adapt-example-float>`. This of course may imply a loss of precision. + + +.. index:: + pair: Strings; Adaptation + single: Unicode; Adaptation + pair: Encoding; SQL_ASCII + +.. _adapt-string: + +Strings adaptation +------------------ + +.. seealso:: + + - `PostgreSQL character types + <https://www.postgresql.org/docs/current/datatype-character.html>`__ + +Python `str` are converted to PostgreSQL string syntax, and PostgreSQL types +such as :sql:`text` and :sql:`varchar` are converted back to Python `!str`: + +.. code:: python + + conn = psycopg.connect() + conn.execute( + "INSERT INTO menu (id, entry) VALUES (%s, %s)", + (1, "Crème Brûlée at 4.99€")) + conn.execute("SELECT entry FROM menu WHERE id = 1").fetchone()[0] + 'Crème Brûlée at 4.99€' + +PostgreSQL databases `have an encoding`__, and `the session has an encoding`__ +too, exposed in the `!Connection.info.`\ `~ConnectionInfo.encoding` +attribute. If your database and connection are in UTF-8 encoding you will +likely have no problem, otherwise you will have to make sure that your +application only deals with the non-ASCII chars that the database can handle; +failing to do so may result in encoding/decoding errors: + +.. __: https://www.postgresql.org/docs/current/sql-createdatabase.html +.. __: https://www.postgresql.org/docs/current/multibyte.html + +.. code:: python + + # The encoding is set at connection time according to the db configuration + conn.info.encoding + 'utf-8' + + # The Latin-9 encoding can manage some European accented letters + # and the Euro symbol + conn.execute("SET client_encoding TO LATIN9") + conn.execute("SELECT entry FROM menu WHERE id = 1").fetchone()[0] + 'Crème Brûlée at 4.99€' + + # The Latin-1 encoding doesn't have a representation for the Euro symbol + conn.execute("SET client_encoding TO LATIN1") + conn.execute("SELECT entry FROM menu WHERE id = 1").fetchone()[0] + # Traceback (most recent call last) + # ... + # UntranslatableCharacter: character with byte sequence 0xe2 0x82 0xac + # in encoding "UTF8" has no equivalent in encoding "LATIN1" + +In rare cases you may have strings with unexpected encodings in the database. +Using the ``SQL_ASCII`` client encoding will disable decoding of the data +coming from the database, which will be returned as `bytes`: + +.. code:: python + + conn.execute("SET client_encoding TO SQL_ASCII") + conn.execute("SELECT entry FROM menu WHERE id = 1").fetchone()[0] + b'Cr\xc3\xa8me Br\xc3\xbbl\xc3\xa9e at 4.99\xe2\x82\xac' + +Alternatively you can cast the unknown encoding data to :sql:`bytea` to +retrieve it as bytes, leaving other strings unaltered: see :ref:`adapt-binary` + +Note that PostgreSQL text cannot contain the ``0x00`` byte. If you need to +store Python strings that may contain binary zeros you should use a +:sql:`bytea` field. + + +.. index:: + single: bytea; Adaptation + single: bytes; Adaptation + single: bytearray; Adaptation + single: memoryview; Adaptation + single: Binary string + +.. _adapt-binary: + +Binary adaptation +----------------- + +Python types representing binary objects (`bytes`, `bytearray`, `memoryview`) +are converted by default to :sql:`bytea` fields. By default data received is +returned as `!bytes`. + +If you are storing large binary data in bytea fields (such as binary documents +or images) you should probably use the binary format to pass and return +values, otherwise binary data will undergo `ASCII escaping`__, taking some CPU +time and more bandwidth. See :ref:`binary-data` for details. + +.. __: https://www.postgresql.org/docs/current/datatype-binary.html + + +.. _adapt-date: + +Date/time types adaptation +-------------------------- + +.. seealso:: + + - `PostgreSQL date/time types + <https://www.postgresql.org/docs/current/datatype-datetime.html>`__ + +- Python `~datetime.date` objects are converted to PostgreSQL :sql:`date`. +- Python `~datetime.datetime` objects are converted to PostgreSQL + :sql:`timestamp` (if they don't have a `!tzinfo` set) or :sql:`timestamptz` + (if they do). +- Python `~datetime.time` objects are converted to PostgreSQL :sql:`time` + (if they don't have a `!tzinfo` set) or :sql:`timetz` (if they do). +- Python `~datetime.timedelta` objects are converted to PostgreSQL + :sql:`interval`. + +PostgreSQL :sql:`timestamptz` values are returned with a timezone set to the +`connection TimeZone setting`__, which is available as a Python +`~zoneinfo.ZoneInfo` object in the `!Connection.info`.\ `~ConnectionInfo.timezone` +attribute:: + + >>> conn.info.timezone + zoneinfo.ZoneInfo(key='Europe/London') + + >>> conn.execute("select '2048-07-08 12:00'::timestamptz").fetchone()[0] + datetime.datetime(2048, 7, 8, 12, 0, tzinfo=zoneinfo.ZoneInfo(key='Europe/London')) + +.. note:: + PostgreSQL :sql:`timestamptz` doesn't store "a timestamp with a timezone + attached": it stores a timestamp always in UTC, which is converted, on + output, to the connection TimeZone setting:: + + >>> conn.execute("SET TIMEZONE to 'Europe/Rome'") # UTC+2 in summer + + >>> conn.execute("SELECT '2042-07-01 12:00Z'::timestamptz").fetchone()[0] # UTC input + datetime.datetime(2042, 7, 1, 14, 0, tzinfo=zoneinfo.ZoneInfo(key='Europe/Rome')) + + Check out the `PostgreSQL documentation about timezones`__ for all the + details. + + .. __: https://www.postgresql.org/docs/current/datatype-datetime.html + #DATATYPE-TIMEZONES + +.. __: https://www.postgresql.org/docs/current/runtime-config-client.html#GUC-TIMEZONE + + +.. _adapt-json: + +JSON adaptation +--------------- + +Psycopg can map between Python objects and PostgreSQL `json/jsonb +types`__, allowing to customise the load and dump function used. + +.. __: https://www.postgresql.org/docs/current/datatype-json.html + +Because several Python objects could be considered JSON (dicts, lists, +scalars, even date/time if using a dumps function customised to use them), +Psycopg requires you to wrap the object to dump as JSON into a wrapper: +either `psycopg.types.json.Json` or `~psycopg.types.json.Jsonb`. + +.. code:: python + + from psycopg.types.json import Jsonb + + thing = {"foo": ["bar", 42]} + conn.execute("INSERT INTO mytable VALUES (%s)", [Jsonb(thing)]) + +By default Psycopg uses the standard library `json.dumps` and `json.loads` +functions to serialize and de-serialize Python objects to JSON. If you want to +customise how serialization happens, for instance changing serialization +parameters or using a different JSON library, you can specify your own +functions using the `psycopg.types.json.set_json_dumps()` and +`~psycopg.types.json.set_json_loads()` functions, to apply either globally or +to a specific context (connection or cursor). + +.. code:: python + + from functools import partial + from psycopg.types.json import Jsonb, set_json_dumps, set_json_loads + import ujson + + # Use a faster dump function + set_json_dumps(ujson.dumps) + + # Return floating point values as Decimal, just in one connection + set_json_loads(partial(json.loads, parse_float=Decimal), conn) + + conn.execute("SELECT %s", [Jsonb({"value": 123.45})]).fetchone()[0] + # {'value': Decimal('123.45')} + +If you need an even more specific dump customisation only for certain objects +(including different configurations in the same query) you can specify a +`!dumps` parameter in the +`~psycopg.types.json.Json`/`~psycopg.types.json.Jsonb` wrapper, which will +take precedence over what is specified by `!set_json_dumps()`. + +.. code:: python + + from uuid import UUID, uuid4 + + class UUIDEncoder(json.JSONEncoder): + """A JSON encoder which can dump UUID.""" + def default(self, obj): + if isinstance(obj, UUID): + return str(obj) + return json.JSONEncoder.default(self, obj) + + uuid_dumps = partial(json.dumps, cls=UUIDEncoder) + obj = {"uuid": uuid4()} + cnn.execute("INSERT INTO objs VALUES %s", [Json(obj, dumps=uuid_dumps)]) + # will insert: {'uuid': '0a40799d-3980-4c65-8315-2956b18ab0e1'} + + +.. _adapt-list: + +Lists adaptation +---------------- + +Python `list` objects are adapted to `PostgreSQL arrays`__ and back. Only +lists containing objects of the same type can be dumped to PostgreSQL (but the +list may contain `!None` elements). + +.. __: https://www.postgresql.org/docs/current/arrays.html + +.. note:: + + If you have a list of values which you want to use with the :sql:`IN` + operator... don't. It won't work (neither with a list nor with a tuple):: + + >>> conn.execute("SELECT * FROM mytable WHERE id IN %s", [[10,20,30]]) + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + psycopg.errors.SyntaxError: syntax error at or near "$1" + LINE 1: SELECT * FROM mytable WHERE id IN $1 + ^ + + What you want to do instead is to use the `'= ANY()' expression`__ and pass + the values as a list (not a tuple). + + >>> conn.execute("SELECT * FROM mytable WHERE id = ANY(%s)", [[10,20,30]]) + + This has also the advantage of working with an empty list, whereas ``IN + ()`` is not valid SQL. + + .. __: https://www.postgresql.org/docs/current/functions-comparisons.html + #id-1.5.8.30.16 + + +.. _adapt-uuid: + +UUID adaptation +--------------- + +Python `uuid.UUID` objects are adapted to PostgreSQL `UUID type`__ and back:: + + >>> conn.execute("select gen_random_uuid()").fetchone()[0] + UUID('97f0dd62-3bd2-459e-89b8-a5e36ea3c16c') + + >>> from uuid import uuid4 + >>> conn.execute("select gen_random_uuid() = %s", [uuid4()]).fetchone()[0] + False # long shot + +.. __: https://www.postgresql.org/docs/current/datatype-uuid.html + + +.. _adapt-network: + +Network data types adaptation +----------------------------- + +Objects from the `ipaddress` module are converted to PostgreSQL `network +address types`__: + +- `~ipaddress.IPv4Address`, `~ipaddress.IPv4Interface` objects are converted + to the PostgreSQL :sql:`inet` type. On the way back, :sql:`inet` values + indicating a single address are converted to `!IPv4Address`, otherwise they + are converted to `!IPv4Interface` + +- `~ipaddress.IPv4Network` objects are converted to the :sql:`cidr` type and + back. + +- `~ipaddress.IPv6Address`, `~ipaddress.IPv6Interface`, + `~ipaddress.IPv6Network` objects follow the same rules, with IPv6 + :sql:`inet` and :sql:`cidr` values. + +.. __: https://www.postgresql.org/docs/current/datatype-net-types.html#DATATYPE-CIDR + +.. code:: python + + >>> conn.execute("select '192.168.0.1'::inet, '192.168.0.1/24'::inet").fetchone() + (IPv4Address('192.168.0.1'), IPv4Interface('192.168.0.1/24')) + + >>> conn.execute("select '::ffff:1.2.3.0/120'::cidr").fetchone()[0] + IPv6Network('::ffff:102:300/120') + + +.. _adapt-enum: + +Enum adaptation +--------------- + +.. versionadded:: 3.1 + +Psycopg can adapt Python `~enum.Enum` subclasses into PostgreSQL enum types +(created with the |CREATE TYPE AS ENUM|_ command). + +.. |CREATE TYPE AS ENUM| replace:: :sql:`CREATE TYPE ... AS ENUM (...)` +.. _CREATE TYPE AS ENUM: https://www.postgresql.org/docs/current/static/datatype-enum.html + +In order to set up a bidirectional enum mapping, you should get information +about the PostgreSQL enum using the `~types.enum.EnumInfo` class and +register it using `~types.enum.register_enum()`. The behaviour of unregistered +and registered enums is different. + +- If the enum is not registered with `register_enum()`: + + - Pure `!Enum` classes are dumped as normal strings, using their member + names as value. The unknown oid is used, so PostgreSQL should be able to + use this string in most contexts (such as an enum or a text field). + + .. versionchanged:: 3.1 + In previous version dumping pure enums is not supported and raise a + "cannot adapt" error. + + - Mix-in enums are dumped according to their mix-in type (because a `class + MyIntEnum(int, Enum)` is more specifically an `!int` than an `!Enum`, so + it's dumped by default according to `!int` rules). + + - PostgreSQL enums are loaded as Python strings. If you want to load arrays + of such enums you will have to find their OIDs using `types.TypeInfo.fetch()` + and register them using `~types.TypeInfo.register()`. + +- If the enum is registered (using `~types.enum.EnumInfo`\ `!.fetch()` and + `~types.enum.register_enum()`): + + - Enums classes, both pure and mixed-in, are dumped by name. + + - The registered PostgreSQL enum is loaded back as the registered Python + enum members. + +.. autoclass:: psycopg.types.enum.EnumInfo + + `!EnumInfo` is a subclass of `~psycopg.types.TypeInfo`: refer to the + latter's documentation for generic usage, especially the + `~psycopg.types.TypeInfo.fetch()` method. + + .. attribute:: labels + + After `~psycopg.types.TypeInfo.fetch()`, it contains the labels defined + in the PostgreSQL enum type. + + .. attribute:: enum + + After `register_enum()` is called, it will contain the Python type + mapping to the registered enum. + +.. autofunction:: psycopg.types.enum.register_enum + + After registering, fetching data of the registered enum will cast + PostgreSQL enum labels into corresponding Python enum members. + + If no `!enum` is specified, a new `Enum` is created based on + PostgreSQL enum labels. + +Example:: + + >>> from enum import Enum, auto + >>> from psycopg.types.enum import EnumInfo, register_enum + + >>> class UserRole(Enum): + ... ADMIN = auto() + ... EDITOR = auto() + ... GUEST = auto() + + >>> conn.execute("CREATE TYPE user_role AS ENUM ('ADMIN', 'EDITOR', 'GUEST')") + + >>> info = EnumInfo.fetch(conn, "user_role") + >>> register_enum(info, conn, UserRole) + + >>> some_editor = info.enum.EDITOR + >>> some_editor + <UserRole.EDITOR: 2> + + >>> conn.execute( + ... "SELECT pg_typeof(%(editor)s), %(editor)s", + ... {"editor": some_editor} + ... ).fetchone() + ('user_role', <UserRole.EDITOR: 2>) + + >>> conn.execute( + ... "SELECT ARRAY[%s, %s]", + ... [UserRole.ADMIN, UserRole.GUEST] + ... ).fetchone() + [<UserRole.ADMIN: 1>, <UserRole.GUEST: 3>] + +If the Python and the PostgreSQL enum don't match 1:1 (for instance if members +have a different name, or if more than one Python enum should map to the same +PostgreSQL enum, or vice versa), you can specify the exceptions using the +`!mapping` parameter. + +`!mapping` should be a dictionary with Python enum members as keys and the +matching PostgreSQL enum labels as values, or a list of `(member, label)` +pairs with the same meaning (useful when some members are repeated). Order +matters: if an element on either side is specified more than once, the last +pair in the sequence will take precedence:: + + # Legacy roles, defined in medieval times. + >>> conn.execute( + ... "CREATE TYPE abbey_role AS ENUM ('ABBOT', 'SCRIBE', 'MONK', 'GUEST')") + + >>> info = EnumInfo.fetch(conn, "abbey_role") + >>> register_enum(info, conn, UserRole, mapping=[ + ... (UserRole.ADMIN, "ABBOT"), + ... (UserRole.EDITOR, "SCRIBE"), + ... (UserRole.EDITOR, "MONK")]) + + >>> conn.execute("SELECT '{ABBOT,SCRIBE,MONK,GUEST}'::abbey_role[]").fetchone()[0] + [<UserRole.ADMIN: 1>, + <UserRole.EDITOR: 2>, + <UserRole.EDITOR: 2>, + <UserRole.GUEST: 3>] + + >>> conn.execute("SELECT %s::text[]", [list(UserRole)]).fetchone()[0] + ['ABBOT', 'MONK', 'GUEST'] + +A particularly useful case is when the PostgreSQL labels match the *values* of +a `!str`\-based Enum. In this case it is possible to use something like ``{m: +m.value for m in enum}`` as mapping:: + + >>> class LowercaseRole(str, Enum): + ... ADMIN = "admin" + ... EDITOR = "editor" + ... GUEST = "guest" + + >>> conn.execute( + ... "CREATE TYPE lowercase_role AS ENUM ('admin', 'editor', 'guest')") + + >>> info = EnumInfo.fetch(conn, "lowercase_role") + >>> register_enum( + ... info, conn, LowercaseRole, mapping={m: m.value for m in LowercaseRole}) + + >>> conn.execute("SELECT 'editor'::lowercase_role").fetchone()[0] + <LowercaseRole.EDITOR: 'editor'> diff --git a/docs/basic/copy.rst b/docs/basic/copy.rst new file mode 100644 index 0000000..2bb4498 --- /dev/null +++ b/docs/basic/copy.rst @@ -0,0 +1,212 @@ +.. currentmodule:: psycopg + +.. index:: + pair: COPY; SQL command + +.. _copy: + +Using COPY TO and COPY FROM +=========================== + +Psycopg allows to operate with `PostgreSQL COPY protocol`__. :sql:`COPY` is +one of the most efficient ways to load data into the database (and to modify +it, with some SQL creativity). + +.. __: https://www.postgresql.org/docs/current/sql-copy.html + +Copy is supported using the `Cursor.copy()` method, passing it a query of the +form :sql:`COPY ... FROM STDIN` or :sql:`COPY ... TO STDOUT`, and managing the +resulting `Copy` object in a `!with` block: + +.. code:: python + + with cursor.copy("COPY table_name (col1, col2) FROM STDIN") as copy: + # pass data to the 'copy' object using write()/write_row() + +You can compose a COPY statement dynamically by using objects from the +`psycopg.sql` module: + +.. code:: python + + with cursor.copy( + sql.SQL("COPY {} TO STDOUT").format(sql.Identifier("table_name")) + ) as copy: + # read data from the 'copy' object using read()/read_row() + +.. versionchanged:: 3.1 + + You can also pass parameters to `!copy()`, like in `~Cursor.execute()`: + + .. code:: python + + with cur.copy("COPY (SELECT * FROM table_name LIMIT %s) TO STDOUT", (3,)) as copy: + # expect no more than three records + +The connection is subject to the usual transaction behaviour, so, unless the +connection is in autocommit, at the end of the COPY operation you will still +have to commit the pending changes and you can still roll them back. See +:ref:`transactions` for details. + + +.. _copy-in-row: + +Writing data row-by-row +----------------------- + +Using a copy operation you can load data into the database from any Python +iterable (a list of tuples, or any iterable of sequences): the Python values +are adapted as they would be in normal querying. To perform such operation use +a :sql:`COPY ... FROM STDIN` with `Cursor.copy()` and use `~Copy.write_row()` +on the resulting object in a `!with` block. On exiting the block the +operation will be concluded: + +.. code:: python + + records = [(10, 20, "hello"), (40, None, "world")] + + with cursor.copy("COPY sample (col1, col2, col3) FROM STDIN") as copy: + for record in records: + copy.write_row(record) + +If an exception is raised inside the block, the operation is interrupted and +the records inserted so far are discarded. + +In order to read or write from `!Copy` row-by-row you must not specify +:sql:`COPY` options such as :sql:`FORMAT CSV`, :sql:`DELIMITER`, :sql:`NULL`: +please leave these details alone, thank you :) + + +.. _copy-out-row: + +Reading data row-by-row +----------------------- + +You can also do the opposite, reading rows out of a :sql:`COPY ... TO STDOUT` +operation, by iterating on `~Copy.rows()`. However this is not something you +may want to do normally: usually the normal query process will be easier to +use. + +PostgreSQL, currently, doesn't give complete type information on :sql:`COPY +TO`, so the rows returned will have unparsed data, as strings or bytes, +according to the format. + +.. code:: python + + with cur.copy("COPY (VALUES (10::int, current_date)) TO STDOUT") as copy: + for row in copy.rows(): + print(row) # return unparsed data: ('10', '2046-12-24') + +You can improve the results by using `~Copy.set_types()` before reading, but +you have to specify them yourself. + +.. code:: python + + with cur.copy("COPY (VALUES (10::int, current_date)) TO STDOUT") as copy: + copy.set_types(["int4", "date"]) + for row in copy.rows(): + print(row) # (10, datetime.date(2046, 12, 24)) + + +.. _copy-block: + +Copying block-by-block +---------------------- + +If data is already formatted in a way suitable for copy (for instance because +it is coming from a file resulting from a previous `COPY TO` operation) it can +be loaded into the database using `Copy.write()` instead. + +.. code:: python + + with open("data", "r") as f: + with cursor.copy("COPY data FROM STDIN") as copy: + while data := f.read(BLOCK_SIZE): + copy.write(data) + +In this case you can use any :sql:`COPY` option and format, as long as the +input data is compatible with what the operation in `!copy()` expects. Data +can be passed as `!str`, if the copy is in :sql:`FORMAT TEXT`, or as `!bytes`, +which works with both :sql:`FORMAT TEXT` and :sql:`FORMAT BINARY`. + +In order to produce data in :sql:`COPY` format you can use a :sql:`COPY ... TO +STDOUT` statement and iterate over the resulting `Copy` object, which will +produce a stream of `!bytes` objects: + +.. code:: python + + with open("data.out", "wb") as f: + with cursor.copy("COPY table_name TO STDOUT") as copy: + for data in copy: + f.write(data) + + +.. _copy-binary: + +Binary copy +----------- + +Binary copy is supported by specifying :sql:`FORMAT BINARY` in the :sql:`COPY` +statement. In order to import binary data using `~Copy.write_row()`, all the +types passed to the database must have a binary dumper registered; this is not +necessary if the data is copied :ref:`block-by-block <copy-block>` using +`~Copy.write()`. + +.. warning:: + + PostgreSQL is particularly finicky when loading data in binary mode and + will apply **no cast rules**. This means, for example, that passing the + value 100 to an `integer` column **will fail**, because Psycopg will pass + it as a `smallint` value, and the server will reject it because its size + doesn't match what expected. + + You can work around the problem using the `~Copy.set_types()` method of + the `!Copy` object and specifying carefully the types to load. + +.. seealso:: See :ref:`binary-data` for further info about binary querying. + + +.. _copy-async: + +Asynchronous copy support +------------------------- + +Asynchronous operations are supported using the same patterns as above, using +the objects obtained by an `AsyncConnection`. For instance, if `!f` is an +object supporting an asynchronous `!read()` method returning :sql:`COPY` data, +a fully-async copy operation could be: + +.. code:: python + + async with cursor.copy("COPY data FROM STDIN") as copy: + while data := await f.read(): + await copy.write(data) + +The `AsyncCopy` object documentation describes the signature of the +asynchronous methods and the differences from its sync `Copy` counterpart. + +.. seealso:: See :ref:`async` for further info about using async objects. + + +Example: copying a table across servers +--------------------------------------- + +In order to copy a table, or a portion of a table, across servers, you can use +two COPY operations on two different connections, reading from the first and +writing to the second. + +.. code:: python + + with psycopg.connect(dsn_src) as conn1, psycopg.connect(dsn_tgt) as conn2: + with conn1.cursor().copy("COPY src TO STDOUT (FORMAT BINARY)") as copy1: + with conn2.cursor().copy("COPY tgt FROM STDIN (FORMAT BINARY)") as copy2: + for data in copy1: + copy2.write(data) + +Using :sql:`FORMAT BINARY` usually gives a performance boost, but it only +works if the source and target schema are *perfectly identical*. If the tables +are only *compatible* (for example, if you are copying an :sql:`integer` field +into a :sql:`bigint` destination field) you should omit the `BINARY` option and +perform a text-based copy. See :ref:`copy-binary` for details. + +The same pattern can be adapted to use :ref:`async objects <async>` in order +to perform an :ref:`async copy <copy-async>`. diff --git a/docs/basic/from_pg2.rst b/docs/basic/from_pg2.rst new file mode 100644 index 0000000..0692049 --- /dev/null +++ b/docs/basic/from_pg2.rst @@ -0,0 +1,359 @@ +.. index:: + pair: psycopg2; Differences + +.. currentmodule:: psycopg + +.. _from-psycopg2: + + +Differences from `!psycopg2` +============================ + +Psycopg 3 uses the common DBAPI structure of many other database adapters and +tries to behave as close as possible to `!psycopg2`. There are however a few +differences to be aware of. + +.. tip:: + Most of the times, the workarounds suggested here will work with both + Psycopg 2 and 3, which could be useful if you are porting a program or + writing a program that should work with both Psycopg 2 and 3. + + +.. _server-side-binding: + +Server-side binding +------------------- + +Psycopg 3 sends the query and the parameters to the server separately, instead +of merging them on the client side. Server-side binding works for normal +:sql:`SELECT` and data manipulation statements (:sql:`INSERT`, :sql:`UPDATE`, +:sql:`DELETE`), but it doesn't work with many other statements. For instance, +it doesn't work with :sql:`SET` or with :sql:`NOTIFY`:: + + >>> conn.execute("SET TimeZone TO %s", ["UTC"]) + Traceback (most recent call last): + ... + psycopg.errors.SyntaxError: syntax error at or near "$1" + LINE 1: SET TimeZone TO $1 + ^ + + >>> conn.execute("NOTIFY %s, %s", ["chan", 42]) + Traceback (most recent call last): + ... + psycopg.errors.SyntaxError: syntax error at or near "$1" + LINE 1: NOTIFY $1, $2 + ^ + +and with any data definition statement:: + + >>> conn.execute("CREATE TABLE foo (id int DEFAULT %s)", [42]) + Traceback (most recent call last): + ... + psycopg.errors.UndefinedParameter: there is no parameter $1 + LINE 1: CREATE TABLE foo (id int DEFAULT $1) + ^ + +Sometimes, PostgreSQL offers an alternative: for instance the `set_config()`__ +function can be used instead of the :sql:`SET` statement, the `pg_notify()`__ +function can be used instead of :sql:`NOTIFY`:: + + >>> conn.execute("SELECT set_config('TimeZone', %s, false)", ["UTC"]) + + >>> conn.execute("SELECT pg_notify(%s, %s)", ["chan", "42"]) + +.. __: https://www.postgresql.org/docs/current/functions-admin.html + #FUNCTIONS-ADMIN-SET + +.. __: https://www.postgresql.org/docs/current/sql-notify.html + #id-1.9.3.157.7.5 + +If this is not possible, you must merge the query and the parameter on the +client side. You can do so using the `psycopg.sql` objects:: + + >>> from psycopg import sql + + >>> cur.execute(sql.SQL("CREATE TABLE foo (id int DEFAULT {})").format(42)) + +or creating a :ref:`client-side binding cursor <client-side-binding-cursors>` +such as `ClientCursor`:: + + >>> cur = ClientCursor(conn) + >>> cur.execute("CREATE TABLE foo (id int DEFAULT %s)", [42]) + +If you need `!ClientCursor` often, you can set the `Connection.cursor_factory` +to have them created by default by `Connection.cursor()`. This way, Psycopg 3 +will behave largely the same way of Psycopg 2. + +Note that, both server-side and client-side, you can only specify **values** +as parameters (i.e. *the strings that go in single quotes*). If you need to +parametrize different parts of a statement (such as a table name), you must +use the `psycopg.sql` module:: + + >>> from psycopg import sql + + # This will quote the user and the password using the right quotes + # e.g.: ALTER USER "foo" SET PASSWORD 'bar' + >>> conn.execute( + ... sql.SQL("ALTER USER {} SET PASSWORD {}") + ... .format(sql.Identifier(username), password)) + + +.. _multi-statements: + +Multiple statements in the same query +------------------------------------- + +As a consequence of using :ref:`server-side bindings <server-side-binding>`, +when parameters are used, it is not possible to execute several statements in +the same `!execute()` call, separating them by semicolon:: + + >>> conn.execute( + ... "INSERT INTO foo VALUES (%s); INSERT INTO foo VALUES (%s)", + ... (10, 20)) + Traceback (most recent call last): + ... + psycopg.errors.SyntaxError: cannot insert multiple commands into a prepared statement + +One obvious way to work around the problem is to use several `!execute()` +calls. + +**There is no such limitation if no parameters are used**. As a consequence, you +can compose a multiple query on the client side and run them all in the same +`!execute()` call, using the `psycopg.sql` objects:: + + >>> from psycopg import sql + >>> conn.execute( + ... sql.SQL("INSERT INTO foo VALUES ({}); INSERT INTO foo values ({})" + ... .format(10, 20)) + +or a :ref:`client-side binding cursor <client-side-binding-cursors>`:: + + >>> cur = psycopg.ClientCursor(conn) + >>> cur.execute( + ... "INSERT INTO foo VALUES (%s); INSERT INTO foo VALUES (%s)", + ... (10, 20)) + +.. warning:: + + If a statements must be executed outside a transaction (such as + :sql:`CREATE DATABASE`), it cannot be executed in batch with other + statements, even if the connection is in autocommit mode:: + + >>> conn.autocommit = True + >>> conn.execute("CREATE DATABASE foo; SELECT 1") + Traceback (most recent call last): + ... + psycopg.errors.ActiveSqlTransaction: CREATE DATABASE cannot run inside a transaction block + + This happens because PostgreSQL itself will wrap multiple statements in a + transaction. Note that your will experience a different behaviour in + :program:`psql` (:program:`psql` will split the queries on semicolons and + send them to the server separately). + + This is not new in Psycopg 3: the same limitation is present in + `!psycopg2` too. + + +.. _multi-results: + +Multiple results returned from multiple statements +-------------------------------------------------- + +If more than one statement returning results is executed in psycopg2, only the +result of the last statement is returned:: + + >>> cur_pg2.execute("SELECT 1; SELECT 2") + >>> cur_pg2.fetchone() + (2,) + +In Psycopg 3 instead, all the results are available. After running the query, +the first result will be readily available in the cursor and can be consumed +using the usual `!fetch*()` methods. In order to access the following +results, you can use the `Cursor.nextset()` method:: + + >>> cur_pg3.execute("SELECT 1; SELECT 2") + >>> cur_pg3.fetchone() + (1,) + + >>> cur_pg3.nextset() + True + >>> cur_pg3.fetchone() + (2,) + + >>> cur_pg3.nextset() + None # no more results + +Remember though that you cannot use server-side bindings to :ref:`execute more +than one statement in the same query <multi-statements>`. + + +.. _difference-cast-rules: + +Different cast rules +-------------------- + +In rare cases, especially around variadic functions, PostgreSQL might fail to +find a function candidate for the given data types:: + + >>> conn.execute("SELECT json_build_array(%s, %s)", ["foo", "bar"]) + Traceback (most recent call last): + ... + psycopg.errors.IndeterminateDatatype: could not determine data type of parameter $1 + +This can be worked around specifying the argument types explicitly via a cast:: + + >>> conn.execute("SELECT json_build_array(%s::text, %s::text)", ["foo", "bar"]) + + +.. _in-and-tuple: + +You cannot use ``IN %s`` with a tuple +------------------------------------- + +``IN`` cannot be used with a tuple as single parameter, as was possible with +``psycopg2``:: + + >>> conn.execute("SELECT * FROM foo WHERE id IN %s", [(10,20,30)]) + Traceback (most recent call last): + ... + psycopg.errors.SyntaxError: syntax error at or near "$1" + LINE 1: SELECT * FROM foo WHERE id IN $1 + ^ + +What you can do is to use the `= ANY()`__ construct and pass the candidate +values as a list instead of a tuple, which will be adapted to a PostgreSQL +array:: + + >>> conn.execute("SELECT * FROM foo WHERE id = ANY(%s)", [[10,20,30]]) + +Note that `ANY()` can be used with `!psycopg2` too, and has the advantage of +accepting an empty list of values too as argument, which is not supported by +the :sql:`IN` operator instead. + +.. __: https://www.postgresql.org/docs/current/functions-comparisons.html + #id-1.5.8.30.16 + + +.. _diff-adapt: + +Different adaptation system +--------------------------- + +The adaptation system has been completely rewritten, in order to address +server-side parameters adaptation, but also to consider performance, +flexibility, ease of customization. + +The default behaviour with builtin data should be :ref:`what you would expect +<types-adaptation>`. If you have customised the way to adapt data, or if you +are managing your own extension types, you should look at the :ref:`new +adaptation system <adaptation>`. + +.. seealso:: + + - :ref:`types-adaptation` for the basic behaviour. + - :ref:`adaptation` for more advanced use. + + +.. _diff-copy: + +Copy is no longer file-based +---------------------------- + +`!psycopg2` exposes :ref:`a few copy methods <pg2:copy>` to interact with +PostgreSQL :sql:`COPY`. Their file-based interface doesn't make it easy to load +dynamically-generated data into a database. + +There is now a single `~Cursor.copy()` method, which is similar to +`!psycopg2` `!copy_expert()` in accepting a free-form :sql:`COPY` command and +returns an object to read/write data, block-wise or record-wise. The different +usage pattern also enables :sql:`COPY` to be used in async interactions. + +.. seealso:: See :ref:`copy` for the details. + + +.. _diff-with: + +`!with` connection +------------------ + +In `!psycopg2`, using the syntax :ref:`with connection <pg2:with>`, +only the transaction is closed, not the connection. This behaviour is +surprising for people used to several other Python classes wrapping resources, +such as files. + +In Psycopg 3, using :ref:`with connection <with-connection>` will close the +connection at the end of the `!with` block, making handling the connection +resources more familiar. + +In order to manage transactions as blocks you can use the +`Connection.transaction()` method, which allows for finer control, for +instance to use nested transactions. + +.. seealso:: See :ref:`transaction-context` for details. + + +.. _diff-callproc: + +`!callproc()` is gone +--------------------- + +`cursor.callproc()` is not implemented. The method has a simplistic semantic +which doesn't account for PostgreSQL positional parameters, procedures, +set-returning functions... Use a normal `~Cursor.execute()` with :sql:`SELECT +function_name(...)` or :sql:`CALL procedure_name(...)` instead. + + +.. _diff-client-encoding: + +`!client_encoding` is gone +-------------------------- + +Psycopg automatically uses the database client encoding to decode data to +Unicode strings. Use `ConnectionInfo.encoding` if you need to read the +encoding. You can select an encoding at connection time using the +`!client_encoding` connection parameter and you can change the encoding of a +connection by running a :sql:`SET client_encoding` statement... But why would +you? + + +.. _infinity-datetime: + +No default infinity dates handling +---------------------------------- + +PostgreSQL can represent a much wider range of dates and timestamps than +Python. While Python dates are limited to the years between 1 and 9999 +(represented by constants such as `datetime.date.min` and +`~datetime.date.max`), PostgreSQL dates extend to BC dates and past the year +10K. Furthermore PostgreSQL can also represent symbolic dates "infinity", in +both directions. + +In psycopg2, by default, `infinity dates and timestamps map to 'date.max'`__ +and similar constants. This has the problem of creating a non-bijective +mapping (two Postgres dates, infinity and 9999-12-31, both map to the same +Python date). There is also the perversity that valid Postgres dates, greater +than Python `!date.max` but arguably lesser than infinity, will still +overflow. + +In Psycopg 3, every date greater than year 9999 will overflow, including +infinity. If you would like to customize this mapping (for instance flattening +every date past Y10K on `!date.max`) you can subclass and adapt the +appropriate loaders: take a look at :ref:`this example +<adapt-example-inf-date>` to see how. + +.. __: https://www.psycopg.org/docs/usage.html#infinite-dates-handling + + +.. _whats-new: + +What's new in Psycopg 3 +----------------------- + +- :ref:`Asynchronous support <async>` +- :ref:`Server-side parameters binding <server-side-binding>` +- :ref:`Prepared statements <prepared-statements>` +- :ref:`Binary communication <binary-data>` +- :ref:`Python-based COPY support <copy>` +- :ref:`Support for static typing <static-typing>` +- :ref:`A redesigned connection pool <connection-pools>` +- :ref:`Direct access to the libpq functionalities <psycopg.pq>` diff --git a/docs/basic/index.rst b/docs/basic/index.rst new file mode 100644 index 0000000..bf9e27d --- /dev/null +++ b/docs/basic/index.rst @@ -0,0 +1,26 @@ +.. _basic: + +Getting started with Psycopg 3 +============================== + +This section of the documentation will explain :ref:`how to install Psycopg +<installation>` and how to perform normal activities such as :ref:`querying +the database <usage>` or :ref:`loading data using COPY <copy>`. + +.. important:: + + If you are familiar with psycopg2 please take a look at + :ref:`from-psycopg2` to see what is changed. + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + install + usage + params + adapt + pgtypes + transactions + copy + from_pg2 diff --git a/docs/basic/install.rst b/docs/basic/install.rst new file mode 100644 index 0000000..8e1dc6d --- /dev/null +++ b/docs/basic/install.rst @@ -0,0 +1,172 @@ +.. _installation: + +Installation +============ + +In short, if you use a :ref:`supported system<supported-systems>`:: + + pip install --upgrade pip # upgrade pip to at least 20.3 + pip install "psycopg[binary]" + +and you should be :ref:`ready to start <module-usage>`. Read further for +alternative ways to install. + + +.. _supported-systems: + +Supported systems +----------------- + +The Psycopg version documented here has *official and tested* support for: + +- Python: from version 3.7 to 3.11 + + - Python 3.6 supported before Psycopg 3.1 + +- PostgreSQL: from version 10 to 15 +- OS: Linux, macOS, Windows + +The tests to verify the supported systems run in `Github workflows`__: +anything that is not tested there is not officially supported. This includes: + +.. __: https://github.com/psycopg/psycopg/actions + +- Unofficial Python distributions such as Conda; +- Alternative PostgreSQL implementation; +- macOS hardware and releases not available on Github workflows. + +If you use an unsupported system, things might work (because, for instance, the +database may use the same wire protocol as PostgreSQL) but we cannot guarantee +the correct working or a smooth ride. + + +.. _binary-install: + +Binary installation +------------------- + +The quickest way to start developing with Psycopg 3 is to install the binary +packages by running:: + + pip install "psycopg[binary]" + +This will install a self-contained package with all the libraries needed. +**You will need pip 20.3 at least**: please run ``pip install --upgrade pip`` +to update it beforehand. + +The above package should work in most situations. It **will not work** in +some cases though. + +If your platform is not supported you should proceed to a :ref:`local +installation <local-installation>` or a :ref:`pure Python installation +<pure-python-installation>`. + +.. seealso:: + + Did Psycopg 3 install ok? Great! You can now move on to the :ref:`basic + module usage <module-usage>` to learn how it works. + + Keep on reading if the above method didn't work and you need a different + way to install Psycopg 3. + + For further information about the differences between the packages see + :ref:`pq-impl`. + + +.. _local-installation: + +Local installation +------------------ + +A "Local installation" results in a performing and maintainable library. The +library will include the speed-up C module and will be linked to the system +libraries (``libpq``, ``libssl``...) so that system upgrade of libraries will +upgrade the libraries used by Psycopg 3 too. This is the preferred way to +install Psycopg for a production site. + +In order to perform a local installation you need some prerequisites: + +- a C compiler, +- Python development headers (e.g. the ``python3-dev`` package). +- PostgreSQL client development headers (e.g. the ``libpq-dev`` package). +- The :program:`pg_config` program available in the :envvar:`PATH`. + +You **must be able** to troubleshoot an extension build, for instance you must +be able to read your compiler's error message. If you are not, please don't +try this and follow the `binary installation`_ instead. + +If your build prerequisites are in place you can run:: + + pip install "psycopg[c]" + + +.. _pure-python-installation: + +Pure Python installation +------------------------ + +If you simply install:: + + pip install psycopg + +without ``[c]`` or ``[binary]`` extras you will obtain a pure Python +implementation. This is particularly handy to debug and hack, but it still +requires the system libpq to operate (which will be imported dynamically via +`ctypes`). + +In order to use the pure Python installation you will need the ``libpq`` +installed in the system: for instance on Debian system you will probably +need:: + + sudo apt install libpq5 + +.. note:: + + The ``libpq`` is the client library used by :program:`psql`, the + PostgreSQL command line client, to connect to the database. On most + systems, installing :program:`psql` will install the ``libpq`` too as a + dependency. + +If you are not able to fulfill this requirement please follow the `binary +installation`_. + + +.. _pool-installation: + +Installing the connection pool +------------------------------ + +The :ref:`Psycopg connection pools <connection-pools>` are distributed in a +separate package from the `!psycopg` package itself, in order to allow a +different release cycle. + +In order to use the pool you must install the ``pool`` extra, using ``pip +install "psycopg[pool]"``, or install the `psycopg_pool` package separately, +which would allow to specify the release to install more precisely. + + +Handling dependencies +--------------------- + +If you need to specify your project dependencies (for instance in a +``requirements.txt`` file, ``setup.py``, ``pyproject.toml`` dependencies...) +you should probably specify one of the following: + +- If your project is a library, add a dependency on ``psycopg``. This will + make sure that your library will have the ``psycopg`` package with the right + interface and leaves the possibility of choosing a specific implementation + to the end user of your library. + +- If your project is a final application (e.g. a service running on a server) + you can require a specific implementation, for instance ``psycopg[c]``, + after you have made sure that the prerequisites are met (e.g. the depending + libraries and tools are installed in the host machine). + +In both cases you can specify which version of Psycopg to use using +`requirement specifiers`__. + +.. __: https://pip.pypa.io/en/stable/cli/pip_install/#requirement-specifiers + +If you want to make sure that a specific implementation is used you can +specify the :envvar:`PSYCOPG_IMPL` environment variable: importing the library +will fail if the implementation specified is not available. See :ref:`pq-impl`. diff --git a/docs/basic/params.rst b/docs/basic/params.rst new file mode 100644 index 0000000..a733f07 --- /dev/null +++ b/docs/basic/params.rst @@ -0,0 +1,242 @@ +.. currentmodule:: psycopg + +.. index:: + pair: Query; Parameters + +.. _query-parameters: + +Passing parameters to SQL queries +================================= + +Most of the times, writing a program you will have to mix bits of SQL +statements with values provided by the rest of the program: + +.. code:: + + SELECT some, fields FROM some_table WHERE id = ... + +:sql:`id` equals what? Probably you will have a Python value you are looking +for. + + +`!execute()` arguments +---------------------- + +Passing parameters to a SQL statement happens in functions such as +`Cursor.execute()` by using ``%s`` placeholders in the SQL statement, and +passing a sequence of values as the second argument of the function. For +example the Python function call: + +.. code:: python + + cur.execute(""" + INSERT INTO some_table (id, created_at, last_name) + VALUES (%s, %s, %s); + """, + (10, datetime.date(2020, 11, 18), "O'Reilly")) + +is *roughly* equivalent to the SQL command: + +.. code-block:: sql + + INSERT INTO some_table (id, created_at, last_name) + VALUES (10, '2020-11-18', 'O''Reilly'); + +Note that the parameters will not be really merged to the query: query and the +parameters are sent to the server separately: see :ref:`server-side-binding` +for details. + +Named arguments are supported too using :samp:`%({name})s` placeholders in the +query and specifying the values into a mapping. Using named arguments allows +to specify the values in any order and to repeat the same value in several +places in the query:: + + cur.execute(""" + INSERT INTO some_table (id, created_at, updated_at, last_name) + VALUES (%(id)s, %(created)s, %(created)s, %(name)s); + """, + {'id': 10, 'name': "O'Reilly", 'created': datetime.date(2020, 11, 18)}) + +Using characters ``%``, ``(``, ``)`` in the argument names is not supported. + +When parameters are used, in order to include a literal ``%`` in the query you +can use the ``%%`` string:: + + cur.execute("SELECT (%s % 2) = 0 AS even", (10,)) # WRONG + cur.execute("SELECT (%s %% 2) = 0 AS even", (10,)) # correct + +While the mechanism resembles regular Python strings manipulation, there are a +few subtle differences you should care about when passing parameters to a +query. + +- The Python string operator ``%`` *must not be used*: the `~cursor.execute()` + method accepts a tuple or dictionary of values as second parameter. + |sql-warn|__: + + .. |sql-warn| replace:: **Never** use ``%`` or ``+`` to merge values + into queries + + .. code:: python + + cur.execute("INSERT INTO numbers VALUES (%s, %s)" % (10, 20)) # WRONG + cur.execute("INSERT INTO numbers VALUES (%s, %s)", (10, 20)) # correct + + .. __: sql-injection_ + +- For positional variables binding, *the second argument must always be a + sequence*, even if it contains a single variable (remember that Python + requires a comma to create a single element tuple):: + + cur.execute("INSERT INTO foo VALUES (%s)", "bar") # WRONG + cur.execute("INSERT INTO foo VALUES (%s)", ("bar")) # WRONG + cur.execute("INSERT INTO foo VALUES (%s)", ("bar",)) # correct + cur.execute("INSERT INTO foo VALUES (%s)", ["bar"]) # correct + +- The placeholder *must not be quoted*:: + + cur.execute("INSERT INTO numbers VALUES ('%s')", ("Hello",)) # WRONG + cur.execute("INSERT INTO numbers VALUES (%s)", ("Hello",)) # correct + +- The variables placeholder *must always be a* ``%s``, even if a different + placeholder (such as a ``%d`` for integers or ``%f`` for floats) may look + more appropriate for the type. You may find other placeholders used in + Psycopg queries (``%b`` and ``%t``) but they are not related to the + type of the argument: see :ref:`binary-data` if you want to read more:: + + cur.execute("INSERT INTO numbers VALUES (%d)", (10,)) # WRONG + cur.execute("INSERT INTO numbers VALUES (%s)", (10,)) # correct + +- Only query values should be bound via this method: it shouldn't be used to + merge table or field names to the query. If you need to generate SQL queries + dynamically (for instance choosing a table name at runtime) you can use the + functionalities provided in the `psycopg.sql` module:: + + cur.execute("INSERT INTO %s VALUES (%s)", ('numbers', 10)) # WRONG + cur.execute( # correct + SQL("INSERT INTO {} VALUES (%s)").format(Identifier('numbers')), + (10,)) + + +.. index:: Security, SQL injection + +.. _sql-injection: + +Danger: SQL injection +--------------------- + +The SQL representation of many data types is often different from their Python +string representation. The typical example is with single quotes in strings: +in SQL single quotes are used as string literal delimiters, so the ones +appearing inside the string itself must be escaped, whereas in Python single +quotes can be left unescaped if the string is delimited by double quotes. + +Because of the difference, sometimes subtle, between the data types +representations, a naïve approach to query strings composition, such as using +Python strings concatenation, is a recipe for *terrible* problems:: + + SQL = "INSERT INTO authors (name) VALUES ('%s')" # NEVER DO THIS + data = ("O'Reilly", ) + cur.execute(SQL % data) # THIS WILL FAIL MISERABLY + # SyntaxError: syntax error at or near "Reilly" + +If the variables containing the data to send to the database come from an +untrusted source (such as data coming from a form on a web site) an attacker +could easily craft a malformed string, either gaining access to unauthorized +data or performing destructive operations on the database. This form of attack +is called `SQL injection`_ and is known to be one of the most widespread forms +of attack on database systems. Before continuing, please print `this page`__ +as a memo and hang it onto your desk. + +.. _SQL injection: https://en.wikipedia.org/wiki/SQL_injection +.. __: https://xkcd.com/327/ + +Psycopg can :ref:`automatically convert Python objects to SQL +values<types-adaptation>`: using this feature your code will be more robust +and reliable. We must stress this point: + +.. warning:: + + - Don't manually merge values to a query: hackers from a foreign country + will break into your computer and steal not only your disks, but also + your cds, leaving you only with the three most embarrassing records you + ever bought. On cassette tapes. + + - If you use the ``%`` operator to merge values to a query, con artists + will seduce your cat, who will run away taking your credit card + and your sunglasses with them. + + - If you use ``+`` to merge a textual value to a string, bad guys in + balaclava will find their way to your fridge, drink all your beer, and + leave your toilet seat up and your toilet paper in the wrong orientation. + + - You don't want to manually merge values to a query: :ref:`use the + provided methods <query-parameters>` instead. + +The correct way to pass variables in a SQL command is using the second +argument of the `Cursor.execute()` method:: + + SQL = "INSERT INTO authors (name) VALUES (%s)" # Note: no quotes + data = ("O'Reilly", ) + cur.execute(SQL, data) # Note: no % operator + +.. note:: + + Python static code checkers are not quite there yet, but, in the future, + it will be possible to check your code for improper use of string + expressions in queries. See :ref:`literal-string` for details. + +.. seealso:: + + Now that you know how to pass parameters to queries, you can take a look + at :ref:`how Psycopg converts data types <types-adaptation>`. + + +.. index:: + pair: Binary; Parameters + +.. _binary-data: + +Binary parameters and results +----------------------------- + +PostgreSQL has two different ways to transmit data between client and server: +`~psycopg.pq.Format.TEXT`, always available, and `~psycopg.pq.Format.BINARY`, +available most of the times but not always. Usually the binary format is more +efficient to use. + +Psycopg can support both formats for each data type. Whenever a value +is passed to a query using the normal ``%s`` placeholder, the best format +available is chosen (often, but not always, the binary format is picked as the +best choice). + +If you have a reason to select explicitly the binary format or the text format +for a value you can use respectively a ``%b`` placeholder or a ``%t`` +placeholder instead of the normal ``%s``. `~Cursor.execute()` will fail if a +`~psycopg.adapt.Dumper` for the right data type and format is not available. + +The same two formats, text or binary, are used by PostgreSQL to return data +from a query to the client. Unlike with parameters, where you can choose the +format value-by-value, all the columns returned by a query will have the same +format. Every type returned by the query should have a `~psycopg.adapt.Loader` +configured, otherwise the data will be returned as unparsed `!str` (for text +results) or buffer (for binary results). + +.. note:: + The `pg_type`_ table defines which format is supported for each PostgreSQL + data type. Text input/output is managed by the functions declared in the + ``typinput`` and ``typoutput`` fields (always present), binary + input/output is managed by the ``typsend`` and ``typreceive`` (which are + optional). + + .. _pg_type: https://www.postgresql.org/docs/current/catalog-pg-type.html + +Because not every PostgreSQL type supports binary output, by default, the data +will be returned in text format. In order to return data in binary format you +can create the cursor using `Connection.cursor`\ `!(binary=True)` or execute +the query using `Cursor.execute`\ `!(binary=True)`. A case in which +requesting binary results is a clear winner is when you have large binary data +in the database, such as images:: + + cur.execute( + "SELECT image_data FROM images WHERE id = %s", [image_id], binary=True) + data = cur.fetchone()[0] diff --git a/docs/basic/pgtypes.rst b/docs/basic/pgtypes.rst new file mode 100644 index 0000000..14ee5be --- /dev/null +++ b/docs/basic/pgtypes.rst @@ -0,0 +1,389 @@ +.. currentmodule:: psycopg + +.. index:: + single: Adaptation + pair: Objects; Adaptation + single: Data types; Adaptation + +.. _extra-adaptation: + +Adapting other PostgreSQL types +=============================== + +PostgreSQL offers other data types which don't map to native Python types. +Psycopg offers wrappers and conversion functions to allow their use. + + +.. index:: + pair: Composite types; Data types + pair: tuple; Adaptation + pair: namedtuple; Adaptation + +.. _adapt-composite: + +Composite types casting +----------------------- + +Psycopg can adapt PostgreSQL composite types (either created with the |CREATE +TYPE|_ command or implicitly defined after a table row type) to and from +Python tuples, `~collections.namedtuple`, or any other suitable object +configured. + +.. |CREATE TYPE| replace:: :sql:`CREATE TYPE` +.. _CREATE TYPE: https://www.postgresql.org/docs/current/static/sql-createtype.html + +Before using a composite type it is necessary to get information about it +using the `~psycopg.types.composite.CompositeInfo` class and to register it +using `~psycopg.types.composite.register_composite()`. + +.. autoclass:: psycopg.types.composite.CompositeInfo + + `!CompositeInfo` is a `~psycopg.types.TypeInfo` subclass: check its + documentation for the generic usage, especially the + `~psycopg.types.TypeInfo.fetch()` method. + + .. attribute:: python_type + + After `register_composite()` is called, it will contain the python type + mapping to the registered composite. + +.. autofunction:: psycopg.types.composite.register_composite + + After registering, fetching data of the registered composite will invoke + `!factory` to create corresponding Python objects. + + If no factory is specified, a `~collection.namedtuple` is created and used + to return data. + + If the `!factory` is a type (and not a generic callable), then dumpers for + that type are created and registered too, so that passing objects of that + type to a query will adapt them to the registered type. + +Example:: + + >>> from psycopg.types.composite import CompositeInfo, register_composite + + >>> conn.execute("CREATE TYPE card AS (value int, suit text)") + + >>> info = CompositeInfo.fetch(conn, "card") + >>> register_composite(info, conn) + + >>> my_card = info.python_type(8, "hearts") + >>> my_card + card(value=8, suit='hearts') + + >>> conn.execute( + ... "SELECT pg_typeof(%(card)s), (%(card)s).suit", {"card": my_card} + ... ).fetchone() + ('card', 'hearts') + + >>> conn.execute("SELECT (%s, %s)::card", [1, "spades"]).fetchone()[0] + card(value=1, suit='spades') + + +Nested composite types are handled as expected, provided that the type of the +composite components are registered as well:: + + >>> conn.execute("CREATE TYPE card_back AS (face card, back text)") + + >>> info2 = CompositeInfo.fetch(conn, "card_back") + >>> register_composite(info2, conn) + + >>> conn.execute("SELECT ((8, 'hearts'), 'blue')::card_back").fetchone()[0] + card_back(face=card(value=8, suit='hearts'), back='blue') + + +.. index:: + pair: range; Data types + +.. _adapt-range: + +Range adaptation +---------------- + +PostgreSQL `range types`__ are a family of data types representing a range of +values between two elements. The type of the element is called the range +*subtype*. PostgreSQL offers a few built-in range types and allows the +definition of custom ones. + +.. __: https://www.postgresql.org/docs/current/rangetypes.html + +All the PostgreSQL range types are loaded as the `~psycopg.types.range.Range` +Python type, which is a `~typing.Generic` type and can hold bounds of +different types. + +.. autoclass:: psycopg.types.range.Range + + This Python type is only used to pass and retrieve range values to and + from PostgreSQL and doesn't attempt to replicate the PostgreSQL range + features: it doesn't perform normalization and doesn't implement all the + operators__ supported by the database. + + PostgreSQL will perform normalisation on `!Range` objects used as query + parameters, so, when they are fetched back, they will be found in the + normal form (for instance ranges on integers will have `[)` bounds). + + .. __: https://www.postgresql.org/docs/current/static/functions-range.html#RANGE-OPERATORS-TABLE + + `!Range` objects are immutable, hashable, and support the `!in` operator + (checking if an element is within the range). They can be tested for + equivalence. Empty ranges evaluate to `!False` in a boolean context, + nonempty ones evaluate to `!True`. + + `!Range` objects have the following attributes: + + .. autoattribute:: isempty + .. autoattribute:: lower + .. autoattribute:: upper + .. autoattribute:: lower_inc + .. autoattribute:: upper_inc + .. autoattribute:: lower_inf + .. autoattribute:: upper_inf + +The built-in range objects are adapted automatically: if a `!Range` objects +contains `~datetime.date` bounds, it is dumped using the :sql:`daterange` OID, +and of course :sql:`daterange` values are loaded back as `!Range[date]`. + +If you create your own range type you can use `~psycopg.types.range.RangeInfo` +and `~psycopg.types.range.register_range()` to associate the range type with +its subtype and make it work like the builtin ones. + +.. autoclass:: psycopg.types.range.RangeInfo + + `!RangeInfo` is a `~psycopg.types.TypeInfo` subclass: check its + documentation for generic details, especially the + `~psycopg.types.TypeInfo.fetch()` method. + +.. autofunction:: psycopg.types.range.register_range + +Example:: + + >>> from psycopg.types.range import Range, RangeInfo, register_range + + >>> conn.execute("CREATE TYPE strrange AS RANGE (SUBTYPE = text)") + >>> info = RangeInfo.fetch(conn, "strrange") + >>> register_range(info, conn) + + >>> conn.execute("SELECT pg_typeof(%s)", [Range("a", "z")]).fetchone()[0] + 'strrange' + + >>> conn.execute("SELECT '[a,z]'::strrange").fetchone()[0] + Range('a', 'z', '[]') + + +.. index:: + pair: range; Data types + +.. _adapt-multirange: + +Multirange adaptation +--------------------- + +Since PostgreSQL 14, every range type is associated with a multirange__, a +type representing a disjoint set of ranges. A multirange is +automatically available for every range, built-in and user-defined. + +.. __: https://www.postgresql.org/docs/current/rangetypes.html + +All the PostgreSQL range types are loaded as the +`~psycopg.types.multirange.Multirange` Python type, which is a mutable +sequence of `~psycopg.types.range.Range` elements. + +.. autoclass:: psycopg.types.multirange.Multirange + + This Python type is only used to pass and retrieve multirange values to + and from PostgreSQL and doesn't attempt to replicate the PostgreSQL + multirange features: overlapping items are not merged, empty ranges are + not discarded, the items are not ordered, the behaviour of `multirange + operators`__ is not replicated in Python. + + PostgreSQL will perform normalisation on `!Multirange` objects used as + query parameters, so, when they are fetched back, they will be found + ordered, with overlapping ranges merged, etc. + + .. __: https://www.postgresql.org/docs/current/static/functions-range.html#MULTIRANGE-OPERATORS-TABLE + + `!Multirange` objects are a `~collections.abc.MutableSequence` and are + totally ordered: they behave pretty much like a list of `!Range`. Like + Range, they are `~typing.Generic` on the subtype of their range, so you + can declare a variable to be `!Multirange[date]` and mypy will complain if + you try to add it a `Range[Decimal]`. + +Like for `~psycopg.types.range.Range`, built-in multirange objects are adapted +automatically: if a `!Multirange` object contains `!Range` with +`~datetime.date` bounds, it is dumped using the :sql:`datemultirange` OID, and +:sql:`datemultirange` values are loaded back as `!Multirange[date]`. + +If you have created your own range type you can use +`~psycopg.types.multirange.MultirangeInfo` and +`~psycopg.types.multirange.register_multirange()` to associate the resulting +multirange type with its subtype and make it work like the builtin ones. + +.. autoclass:: psycopg.types.multirange.MultirangeInfo + + `!MultirangeInfo` is a `~psycopg.types.TypeInfo` subclass: check its + documentation for generic details, especially the + `~psycopg.types.TypeInfo.fetch()` method. + +.. autofunction:: psycopg.types.multirange.register_multirange + +Example:: + + >>> from psycopg.types.multirange import \ + ... Multirange, MultirangeInfo, register_multirange + >>> from psycopg.types.range import Range + + >>> conn.execute("CREATE TYPE strrange AS RANGE (SUBTYPE = text)") + >>> info = MultirangeInfo.fetch(conn, "strmultirange") + >>> register_multirange(info, conn) + + >>> rec = conn.execute( + ... "SELECT pg_typeof(%(mr)s), %(mr)s", + ... {"mr": Multirange([Range("a", "q"), Range("l", "z")])}).fetchone() + + >>> rec[0] + 'strmultirange' + >>> rec[1] + Multirange([Range('a', 'z', '[)')]) + + +.. index:: + pair: hstore; Data types + pair: dict; Adaptation + +.. _adapt-hstore: + +Hstore adaptation +----------------- + +The |hstore|_ data type is a key-value store embedded in PostgreSQL. It +supports GiST or GIN indexes allowing search by keys or key/value pairs as +well as regular BTree indexes for equality, uniqueness etc. + +.. |hstore| replace:: :sql:`hstore` +.. _hstore: https://www.postgresql.org/docs/current/static/hstore.html + +Psycopg can convert Python `!dict` objects to and from |hstore| structures. +Only dictionaries with string keys and values are supported. `!None` is also +allowed as value but not as a key. + +In order to use the |hstore| data type it is necessary to load it in a +database using: + +.. code:: none + + =# CREATE EXTENSION hstore; + +Because |hstore| is distributed as a contrib module, its oid is not well +known, so it is necessary to use `!TypeInfo`\.\ +`~psycopg.types.TypeInfo.fetch()` to query the database and get its oid. The +resulting object can be passed to +`~psycopg.types.hstore.register_hstore()` to configure dumping `!dict` to +|hstore| and parsing |hstore| back to `!dict`, in the context where the +adapter is registered. + +.. autofunction:: psycopg.types.hstore.register_hstore + +Example:: + + >>> from psycopg.types import TypeInfo + >>> from psycopg.types.hstore import register_hstore + + >>> info = TypeInfo.fetch(conn, "hstore") + >>> register_hstore(info, conn) + + >>> conn.execute("SELECT pg_typeof(%s)", [{"a": "b"}]).fetchone()[0] + 'hstore' + + >>> conn.execute("SELECT 'foo => bar'::hstore").fetchone()[0] + {'foo': 'bar'} + + +.. index:: + pair: geometry; Data types + single: PostGIS; Data types + +.. _adapt-shapely: + +Geometry adaptation using Shapely +--------------------------------- + +When using the PostGIS_ extension, it can be useful to retrieve geometry_ +values and have them automatically converted to Shapely_ instances. Likewise, +you may want to store such instances in the database and have the conversion +happen automatically. + +.. warning:: + Psycopg doesn't have a dependency on the ``shapely`` package: you should + install the library as an additional dependency of your project. + +.. warning:: + This module is experimental and might be changed in the future according + to users' feedback. + +.. _PostGIS: https://postgis.net/ +.. _geometry: https://postgis.net/docs/geometry.html +.. _Shapely: https://github.com/Toblerity/Shapely +.. _shape: https://shapely.readthedocs.io/en/stable/manual.html#shapely.geometry.shape + +Since PostgGIS is an extension, the :sql:`geometry` type oid is not well +known, so it is necessary to use `!TypeInfo`\.\ +`~psycopg.types.TypeInfo.fetch()` to query the database and find it. The +resulting object can be passed to `~psycopg.types.shapely.register_shapely()` +to configure dumping `shape`_ instances to :sql:`geometry` columns and parsing +:sql:`geometry` data back to `!shape` instances, in the context where the +adapters are registered. + +.. function:: psycopg.types.shapely.register_shapely + + Register Shapely dumper and loaders. + + After invoking this function on an adapter, the queries retrieving + PostGIS geometry objects will return Shapely's shape object instances + both in text and binary mode. + + Similarly, shape objects can be sent to the database. + + This requires the Shapely library to be installed. + + :param info: The object with the information about the geometry type. + :param context: The context where to register the adapters. If `!None`, + register it globally. + + .. note:: + + Registering the adapters doesn't affect objects already created, even + if they are children of the registered context. For instance, + registering the adapter globally doesn't affect already existing + connections. + +Example:: + + >>> from psycopg.types import TypeInfo + >>> from psycopg.types.shapely import register_shapely + >>> from shapely.geometry import Point + + >>> info = TypeInfo.fetch(conn, "geometry") + >>> register_shapely(info, conn) + + >>> conn.execute("SELECT pg_typeof(%s)", [Point(1.2, 3.4)]).fetchone()[0] + 'geometry' + + >>> conn.execute(""" + ... SELECT ST_GeomFromGeoJSON('{ + ... "type":"Point", + ... "coordinates":[-48.23456,20.12345]}') + ... """).fetchone()[0] + <shapely.geometry.multipolygon.MultiPolygon object at 0x7fb131f3cd90> + +Notice that, if the geometry adapters are registered on a specific object (a +connection or cursor), other connections and cursors will be unaffected:: + + >>> conn2 = psycopg.connect(CONN_STR) + >>> conn2.execute(""" + ... SELECT ST_GeomFromGeoJSON('{ + ... "type":"Point", + ... "coordinates":[-48.23456,20.12345]}') + ... """).fetchone()[0] + '0101000020E61000009279E40F061E48C0F2B0506B9A1F3440' + diff --git a/docs/basic/transactions.rst b/docs/basic/transactions.rst new file mode 100644 index 0000000..b976046 --- /dev/null +++ b/docs/basic/transactions.rst @@ -0,0 +1,388 @@ +.. currentmodule:: psycopg + +.. index:: Transactions management +.. index:: InFailedSqlTransaction +.. index:: idle in transaction + +.. _transactions: + +Transactions management +======================= + +Psycopg has a behaviour that may seem surprising compared to +:program:`psql`: by default, any database operation will start a new +transaction. As a consequence, changes made by any cursor of the connection +will not be visible until `Connection.commit()` is called, and will be +discarded by `Connection.rollback()`. The following operation on the same +connection will start a new transaction. + +If a database operation fails, the server will refuse further commands, until +a `~rollback()` is called. + +If the cursor is closed with a transaction open, no COMMIT command is sent to +the server, which will then discard the connection. Certain middleware (such +as PgBouncer) will also discard a connection left in transaction state, so, if +possible you will want to commit or rollback a connection before finishing +working with it. + +An example of what will happen, the first time you will use Psycopg (and to be +disappointed by it), is likely: + +.. code:: python + + conn = psycopg.connect() + + # Creating a cursor doesn't start a transaction or affect the connection + # in any way. + cur = conn.cursor() + + cur.execute("SELECT count(*) FROM my_table") + # This function call executes: + # - BEGIN + # - SELECT count(*) FROM my_table + # So now a transaction has started. + + # If your program spends a long time in this state, the server will keep + # a connection "idle in transaction", which is likely something undesired + + cur.execute("INSERT INTO data VALUES (%s)", ("Hello",)) + # This statement is executed inside the transaction + + conn.close() + # No COMMIT was sent: the INSERT was discarded. + +There are a few things going wrong here, let's see how they can be improved. + +One obvious problem after the run above is that, firing up :program:`psql`, +you will see no new record in the table ``data``. One way to fix the problem +is to call `!conn.commit()` before closing the connection. Thankfully, if you +use the :ref:`connection context <with-connection>`, Psycopg will commit the +connection at the end of the block (or roll it back if the block is exited +with an exception): + +The code modified using a connection context will result in the following +sequence of database statements: + +.. code-block:: python + :emphasize-lines: 1 + + with psycopg.connect() as conn: + + cur = conn.cursor() + + cur.execute("SELECT count(*) FROM my_table") + # This function call executes: + # - BEGIN + # - SELECT count(*) FROM my_table + # So now a transaction has started. + + cur.execute("INSERT INTO data VALUES (%s)", ("Hello",)) + # This statement is executed inside the transaction + + # No exception at the end of the block: + # COMMIT is executed. + +This way we don't have to remember to call neither `!close()` nor `!commit()` +and the database operations actually have a persistent effect. The code might +still do something you don't expect: keep a transaction from the first +operation to the connection closure. You can have a finer control over the +transactions using an :ref:`autocommit transaction <autocommit>` and/or +:ref:`transaction contexts <transaction-context>`. + +.. warning:: + + By default even a simple :sql:`SELECT` will start a transaction: in + long-running programs, if no further action is taken, the session will + remain *idle in transaction*, an undesirable condition for several + reasons (locks are held by the session, tables bloat...). For long lived + scripts, either make sure to terminate a transaction as soon as possible or + use an `~Connection.autocommit` connection. + +.. hint:: + + If a database operation fails with an error message such as + *InFailedSqlTransaction: current transaction is aborted, commands ignored + until end of transaction block*, it means that **a previous operation + failed** and the database session is in a state of error. You need to call + `~Connection.rollback()` if you want to keep on using the same connection. + + +.. _autocommit: + +Autocommit transactions +----------------------- + +The manual commit requirement can be suspended using `~Connection.autocommit`, +either as connection attribute or as `~psycopg.Connection.connect()` +parameter. This may be required to run operations that cannot be executed +inside a transaction, such as :sql:`CREATE DATABASE`, :sql:`VACUUM`, +:sql:`CALL` on `stored procedures`__ using transaction control. + +.. __: https://www.postgresql.org/docs/current/xproc.html + +With an autocommit transaction, the above sequence of operation results in: + +.. code-block:: python + :emphasize-lines: 1 + + with psycopg.connect(autocommit=True) as conn: + + cur = conn.cursor() + + cur.execute("SELECT count(*) FROM my_table") + # This function call now only executes: + # - SELECT count(*) FROM my_table + # and no transaction starts. + + cur.execute("INSERT INTO data VALUES (%s)", ("Hello",)) + # The result of this statement is persisted immediately by the database + + # The connection is closed at the end of the block but, because it is not + # in a transaction state, no COMMIT is executed. + +An autocommit transaction behaves more as someone coming from :program:`psql` +would expect. This has a beneficial performance effect, because less queries +are sent and less operations are performed by the database. The statements, +however, are not executed in an atomic transaction; if you need to execute +certain operations inside a transaction, you can achieve that with an +autocommit connection too, using an explicit :ref:`transaction block +<transaction-context>`. + + +.. _transaction-context: + +Transaction contexts +-------------------- + +A more transparent way to make sure that transactions are finalised at the +right time is to use `!with` `Connection.transaction()` to create a +transaction context. When the context is entered, a transaction is started; +when leaving the context the transaction is committed, or it is rolled back if +an exception is raised inside the block. + +Continuing the example above, if you want to use an autocommit connection but +still wrap selected groups of commands inside an atomic transaction, you can +use a `!transaction()` context: + +.. code-block:: python + :emphasize-lines: 8 + + with psycopg.connect(autocommit=True) as conn: + + cur = conn.cursor() + + cur.execute("SELECT count(*) FROM my_table") + # The connection is autocommit, so no BEGIN executed. + + with conn.transaction(): + # BEGIN is executed, a transaction started + + cur.execute("INSERT INTO data VALUES (%s)", ("Hello",)) + cur.execute("INSERT INTO times VALUES (now())") + # These two operation run atomically in the same transaction + + # COMMIT is executed at the end of the block. + # The connection is in idle state again. + + # The connection is closed at the end of the block. + + +Note that connection blocks can also be used with non-autocommit connections: +in this case you still need to pay attention to eventual transactions started +automatically. If an operation starts an implicit transaction, a +`!transaction()` block will only manage :ref:`a savepoint sub-transaction +<nested-transactions>`, leaving the caller to deal with the main transaction, +as explained in :ref:`transactions`: + +.. code:: python + + conn = psycopg.connect() + + cur = conn.cursor() + + cur.execute("SELECT count(*) FROM my_table") + # This function call executes: + # - BEGIN + # - SELECT count(*) FROM my_table + # So now a transaction has started. + + with conn.transaction(): + # The block starts with a transaction already open, so it will execute + # - SAVEPOINT + + cur.execute("INSERT INTO data VALUES (%s)", ("Hello",)) + + # The block was executing a sub-transaction so on exit it will only run: + # - RELEASE SAVEPOINT + # The transaction is still on. + + conn.close() + # No COMMIT was sent: the INSERT was discarded. + +If a `!transaction()` block starts when no transaction is active then it will +manage a proper transaction. In essence, a transaction context tries to leave +a connection in the state it found it, and leaves you to deal with the wider +context. + +.. hint:: + The interaction between non-autocommit transactions and transaction + contexts is probably surprising. Although the non-autocommit default is + what's demanded by the DBAPI, the personal preference of several experienced + developers is to: + + - use a connection block: ``with psycopg.connect(...) as conn``; + - use an autocommit connection, either passing `!autocommit=True` as + `!connect()` parameter or setting the attribute ``conn.autocommit = + True``; + - use `!with conn.transaction()` blocks to manage transactions only where + needed. + + +.. _nested-transactions: + +Nested transactions +^^^^^^^^^^^^^^^^^^^ + +Transaction blocks can be also nested (internal transaction blocks are +implemented using SAVEPOINT__): an exception raised inside an inner block +has a chance of being handled and not completely fail outer operations. The +following is an example where a series of operations interact with the +database: operations are allowed to fail; at the end we also want to store the +number of operations successfully processed. + +.. __: https://www.postgresql.org/docs/current/sql-savepoint.html + +.. code:: python + + with conn.transaction() as tx1: + num_ok = 0 + for operation in operations: + try: + with conn.transaction() as tx2: + unreliable_operation(conn, operation) + except Exception: + logger.exception(f"{operation} failed") + else: + num_ok += 1 + + save_number_of_successes(conn, num_ok) + +If `!unreliable_operation()` causes an error, including an operation causing a +database error, all its changes will be reverted. The exception bubbles up +outside the block: in the example it is intercepted by the `!try` so that the +loop can complete. The outermost block is unaffected (unless other errors +happen there). + +You can also write code to explicitly roll back any currently active +transaction block, by raising the `Rollback` exception. The exception "jumps" +to the end of a transaction block, rolling back its transaction but allowing +the program execution to continue from there. By default the exception rolls +back the innermost transaction block, but any current block can be specified +as the target. In the following example, a hypothetical `!CancelCommand` +may stop the processing and cancel any operation previously performed, +but not entirely committed yet. + +.. code:: python + + from psycopg import Rollback + + with conn.transaction() as outer_tx: + for command in commands(): + with conn.transaction() as inner_tx: + if isinstance(command, CancelCommand): + raise Rollback(outer_tx) + process_command(command) + + # If `Rollback` is raised, it would propagate only up to this block, + # and the program would continue from here with no exception. + + +.. _transaction-characteristics: + +Transaction characteristics +--------------------------- + +You can set `transaction parameters`__ for the transactions that Psycopg +handles. They affect the transactions started implicitly by non-autocommit +transactions and the ones started explicitly by `Connection.transaction()` for +both autocommit and non-autocommit transactions. Leaving these parameters as +`!None` will use the server's default behaviour (which is controlled +by server settings such as default_transaction_isolation__). + +.. __: https://www.postgresql.org/docs/current/sql-set-transaction.html +.. __: https://www.postgresql.org/docs/current/runtime-config-client.html + #GUC-DEFAULT-TRANSACTION-ISOLATION + +In order to set these parameters you can use the connection attributes +`~Connection.isolation_level`, `~Connection.read_only`, +`~Connection.deferrable`. For async connections you must use the equivalent +`~AsyncConnection.set_isolation_level()` method and similar. The parameters +can only be changed if there isn't a transaction already active on the +connection. + +.. warning:: + + Applications running at `~IsolationLevel.REPEATABLE_READ` or + `~IsolationLevel.SERIALIZABLE` isolation level are exposed to serialization + failures. `In certain concurrent update cases`__, PostgreSQL will raise an + exception looking like:: + + psycopg2.errors.SerializationFailure: could not serialize access + due to concurrent update + + In this case the application must be prepared to repeat the operation that + caused the exception. + + .. __: https://www.postgresql.org/docs/current/transaction-iso.html + #XACT-REPEATABLE-READ + + +.. index:: + pair: Two-phase commit; Transaction + +.. _two-phase-commit: + +Two-Phase Commit protocol support +--------------------------------- + +.. versionadded:: 3.1 + +Psycopg exposes the two-phase commit features available in PostgreSQL +implementing the `two-phase commit extensions`__ proposed by the DBAPI. + +The DBAPI model of two-phase commit is inspired by the `XA specification`__, +according to which transaction IDs are formed from three components: + +- a format ID (non-negative 32 bit integer) +- a global transaction ID (string not longer than 64 bytes) +- a branch qualifier (string not longer than 64 bytes) + +For a particular global transaction, the first two components will be the same +for all the resources. Every resource will be assigned a different branch +qualifier. + +According to the DBAPI specification, a transaction ID is created using the +`Connection.xid()` method. Once you have a transaction id, a distributed +transaction can be started with `Connection.tpc_begin()`, prepared using +`~Connection.tpc_prepare()` and completed using `~Connection.tpc_commit()` or +`~Connection.tpc_rollback()`. Transaction IDs can also be retrieved from the +database using `~Connection.tpc_recover()` and completed using the above +`!tpc_commit()` and `!tpc_rollback()`. + +PostgreSQL doesn't follow the XA standard though, and the ID for a PostgreSQL +prepared transaction can be any string up to 200 characters long. Psycopg's +`Xid` objects can represent both XA-style transactions IDs (such as the ones +created by the `!xid()` method) and PostgreSQL transaction IDs identified by +an unparsed string. + +The format in which the Xids are converted into strings passed to the +database is the same employed by the `PostgreSQL JDBC driver`__: this should +allow interoperation between tools written in Python and in Java. For example +a recovery tool written in Python would be able to recognize the components of +transactions produced by a Java program. + +For further details see the documentation for the :ref:`tpc-methods`. + +.. __: https://www.python.org/dev/peps/pep-0249/#optional-two-phase-commit-extensions +.. __: https://publications.opengroup.org/c193 +.. __: https://jdbc.postgresql.org/ diff --git a/docs/basic/usage.rst b/docs/basic/usage.rst new file mode 100644 index 0000000..6c69fe8 --- /dev/null +++ b/docs/basic/usage.rst @@ -0,0 +1,232 @@ +.. currentmodule:: psycopg + +.. _module-usage: + +Basic module usage +================== + +The basic Psycopg usage is common to all the database adapters implementing +the `DB-API`__ protocol. Other database adapters, such as the builtin +`sqlite3` or `psycopg2`, have roughly the same pattern of interaction. + +.. __: https://www.python.org/dev/peps/pep-0249/ + + +.. index:: + pair: Example; Usage + +.. _usage: + +Main objects in Psycopg 3 +------------------------- + +Here is an interactive session showing some of the basic commands: + +.. code:: python + + # Note: the module name is psycopg, not psycopg3 + import psycopg + + # Connect to an existing database + with psycopg.connect("dbname=test user=postgres") as conn: + + # Open a cursor to perform database operations + with conn.cursor() as cur: + + # Execute a command: this creates a new table + cur.execute(""" + CREATE TABLE test ( + id serial PRIMARY KEY, + num integer, + data text) + """) + + # Pass data to fill a query placeholders and let Psycopg perform + # the correct conversion (no SQL injections!) + cur.execute( + "INSERT INTO test (num, data) VALUES (%s, %s)", + (100, "abc'def")) + + # Query the database and obtain data as Python objects. + cur.execute("SELECT * FROM test") + cur.fetchone() + # will return (1, 100, "abc'def") + + # You can use `cur.fetchmany()`, `cur.fetchall()` to return a list + # of several records, or even iterate on the cursor + for record in cur: + print(record) + + # Make the changes to the database persistent + conn.commit() + + +In the example you can see some of the main objects and methods and how they +relate to each other: + +- The function `~Connection.connect()` creates a new database session and + returns a new `Connection` instance. `AsyncConnection.connect()` + creates an `asyncio` connection instead. + +- The `~Connection` class encapsulates a database session. It allows to: + + - create new `~Cursor` instances using the `~Connection.cursor()` method to + execute database commands and queries, + + - terminate transactions using the methods `~Connection.commit()` or + `~Connection.rollback()`. + +- The class `~Cursor` allows interaction with the database: + + - send commands to the database using methods such as `~Cursor.execute()` + and `~Cursor.executemany()`, + + - retrieve data from the database, iterating on the cursor or using methods + such as `~Cursor.fetchone()`, `~Cursor.fetchmany()`, `~Cursor.fetchall()`. + +- Using these objects as context managers (i.e. using `!with`) will make sure + to close them and free their resources at the end of the block (notice that + :ref:`this is different from psycopg2 <diff-with>`). + + +.. seealso:: + + A few important topics you will have to deal with are: + + - :ref:`query-parameters`. + - :ref:`types-adaptation`. + - :ref:`transactions`. + + +Shortcuts +--------- + +The pattern above is familiar to `!psycopg2` users. However, Psycopg 3 also +exposes a few simple extensions which make the above pattern leaner: + +- the `Connection` objects exposes an `~Connection.execute()` method, + equivalent to creating a cursor, calling its `~Cursor.execute()` method, and + returning it. + + .. code:: + + # In Psycopg 2 + cur = conn.cursor() + cur.execute(...) + + # In Psycopg 3 + cur = conn.execute(...) + +- The `Cursor.execute()` method returns `!self`. This means that you can chain + a fetch operation, such as `~Cursor.fetchone()`, to the `!execute()` call: + + .. code:: + + # In Psycopg 2 + cur.execute(...) + record = cur.fetchone() + + cur.execute(...) + for record in cur: + ... + + # In Psycopg 3 + record = cur.execute(...).fetchone() + + for record in cur.execute(...): + ... + +Using them together, in simple cases, you can go from creating a connection to +using a result in a single expression: + +.. code:: + + print(psycopg.connect(DSN).execute("SELECT now()").fetchone()[0]) + # 2042-07-12 18:15:10.706497+01:00 + + +.. index:: + pair: Connection; `!with` + +.. _with-connection: + +Connection context +------------------ + +Psycopg 3 `Connection` can be used as a context manager: + +.. code:: python + + with psycopg.connect() as conn: + ... # use the connection + + # the connection is now closed + +When the block is exited, if there is a transaction open, it will be +committed. If an exception is raised within the block the transaction is +rolled back. In both cases the connection is closed. It is roughly the +equivalent of: + +.. code:: python + + conn = psycopg.connect() + try: + ... # use the connection + except BaseException: + conn.rollback() + else: + conn.commit() + finally: + conn.close() + +.. note:: + This behaviour is not what `!psycopg2` does: in `!psycopg2` :ref:`there is + no final close() <pg2:with>` and the connection can be used in several + `!with` statements to manage different transactions. This behaviour has + been considered non-standard and surprising so it has been replaced by the + more explicit `~Connection.transaction()` block. + +Note that, while the above pattern is what most people would use, `connect()` +doesn't enter a block itself, but returns an "un-entered" connection, so that +it is still possible to use a connection regardless of the code scope and the +developer is free to use (and responsible for calling) `~Connection.commit()`, +`~Connection.rollback()`, `~Connection.close()` as and where needed. + +.. warning:: + If a connection is just left to go out of scope, the way it will behave + with or without the use of a `!with` block is different: + + - if the connection is used without a `!with` block, the server will find + a connection closed INTRANS and roll back the current transaction; + + - if the connection is used with a `!with` block, there will be an + explicit COMMIT and the operations will be finalised. + + You should use a `!with` block when your intention is just to execute a + set of operations and then committing the result, which is the most usual + thing to do with a connection. If your connection life cycle and + transaction pattern is different, and want more control on it, the use + without `!with` might be more convenient. + + See :ref:`transactions` for more information. + +`AsyncConnection` can be also used as context manager, using ``async with``, +but be careful about its quirkiness: see :ref:`async-with` for details. + + +Adapting pyscopg to your program +-------------------------------- + +The above :ref:`pattern of use <usage>` only shows the default behaviour of +the adapter. Psycopg can be customised in several ways, to allow the smoothest +integration between your Python program and your PostgreSQL database: + +- If your program is concurrent and based on `asyncio` instead of on + threads/processes, you can use :ref:`async connections and cursors <async>`. + +- If you want to customise the objects that the cursor returns, instead of + receiving tuples, you can specify your :ref:`row factories <row-factories>`. + +- If you want to customise how Python values and PostgreSQL types are mapped + into each other, beside the :ref:`basic type mapping <types-adaptation>`, + you can :ref:`configure your types <adaptation>`. diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..a20894b --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,110 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + +import sys +from pathlib import Path + +import psycopg + +docs_dir = Path(__file__).parent +sys.path.append(str(docs_dir / "lib")) + + +# -- Project information ----------------------------------------------------- + +project = "psycopg" +copyright = "2020, Daniele Varrazzo and The Psycopg Team" +author = "Daniele Varrazzo" +release = psycopg.__version__ + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.intersphinx", + "sphinx_autodoc_typehints", + "sql_role", + "ticket_role", + "pg3_docs", + "libpq_docs", +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", ".venv"] + + +# -- Options for HTML output ------------------------------------------------- + +# The announcement may be in the website but not shipped with the docs +ann_file = docs_dir / "../../templates/docs3-announcement.html" +if ann_file.exists(): + with ann_file.open() as f: + announcement = f.read() +else: + announcement = "" + +html_css_files = ["psycopg.css"] + +# The name of the Pygments (syntax highlighting) style to use. +# Some that I've check don't suck: +# default lovelace tango algol_nu +# list: from pygments.styles import STYLE_MAP; print(sorted(STYLE_MAP.keys())) +pygments_style = "tango" + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = "furo" +html_show_sphinx = True +html_show_sourcelink = False +html_theme_options = { + "announcement": announcement, + "sidebar_hide_name": False, + "light_logo": "psycopg.svg", + "dark_logo": "psycopg.svg", + "light_css_variables": { + "admonition-font-size": "1rem", + }, +} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + +# The reST default role (used for this markup: `text`) to use for all documents. +default_role = "obj" + +intersphinx_mapping = { + "py": ("https://docs.python.org/3", None), + "pg2": ("https://www.psycopg.org/docs/", None), +} + +autodoc_member_order = "bysource" + +# PostgreSQL docs version to link libpq functions to +libpq_docs_version = "14" + +# Where to point on :ticket: role +ticket_url = "https://github.com/psycopg/psycopg/issues/%s" diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..916eeb0 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,52 @@ +=================================================== +Psycopg 3 -- PostgreSQL database adapter for Python +=================================================== + +Psycopg 3 is a newly designed PostgreSQL_ database adapter for the Python_ +programming language. + +Psycopg 3 presents a familiar interface for everyone who has used +`Psycopg 2`_ or any other `DB-API 2.0`_ database adapter, but allows to use +more modern PostgreSQL and Python features, such as: + +- :ref:`Asynchronous support <async>` +- :ref:`COPY support from Python objects <copy>` +- :ref:`A redesigned connection pool <connection-pools>` +- :ref:`Support for static typing <static-typing>` +- :ref:`Server-side parameters binding <server-side-binding>` +- :ref:`Prepared statements <prepared-statements>` +- :ref:`Statements pipeline <pipeline-mode>` +- :ref:`Binary communication <binary-data>` +- :ref:`Direct access to the libpq functionalities <psycopg.pq>` + +.. _Python: https://www.python.org/ +.. _PostgreSQL: https://www.postgresql.org/ +.. _Psycopg 2: https://www.psycopg.org/docs/ +.. _DB-API 2.0: https://www.python.org/dev/peps/pep-0249/ + + +Documentation +============= + +.. toctree:: + :maxdepth: 2 + + basic/index + advanced/index + api/index + +Release notes +------------- + +.. toctree:: + :maxdepth: 1 + + news + news_pool + + +Indices and tables +------------------ + +* :ref:`genindex` +* :ref:`modindex` diff --git a/docs/lib/libpq_docs.py b/docs/lib/libpq_docs.py new file mode 100644 index 0000000..b8e01f0 --- /dev/null +++ b/docs/lib/libpq_docs.py @@ -0,0 +1,182 @@ +""" +Sphinx plugin to link to the libpq documentation. + +Add the ``:pq:`` role, to create a link to a libpq function, e.g. :: + + :pq:`PQlibVersion()` + +will link to:: + + https://www.postgresql.org/docs/current/libpq-misc.html #LIBPQ-PQLIBVERSION + +""" + +# Copyright (C) 2020 The Psycopg Team + +import os +import logging +import urllib.request +from pathlib import Path +from functools import lru_cache +from html.parser import HTMLParser + +from docutils import nodes, utils +from docutils.parsers.rst import roles + +logger = logging.getLogger("sphinx.libpq_docs") + + +class LibpqParser(HTMLParser): + def __init__(self, data, version="current"): + super().__init__() + self.data = data + self.version = version + + self.section_id = None + self.varlist_id = None + self.in_term = False + self.in_func = False + + def handle_starttag(self, tag, attrs): + if tag == "sect1": + self.handle_sect1(tag, attrs) + elif tag == "varlistentry": + self.handle_varlistentry(tag, attrs) + elif tag == "term": + self.in_term = True + elif tag == "function": + self.in_func = True + + def handle_endtag(self, tag): + if tag == "term": + self.in_term = False + elif tag == "function": + self.in_func = False + + def handle_data(self, data): + if not (self.in_term and self.in_func): + return + + self.add_function(data) + + def handle_sect1(self, tag, attrs): + attrs = dict(attrs) + if "id" in attrs: + self.section_id = attrs["id"] + + def handle_varlistentry(self, tag, attrs): + attrs = dict(attrs) + if "id" in attrs: + self.varlist_id = attrs["id"] + + def add_function(self, func_name): + self.data[func_name] = self.get_func_url() + + def get_func_url(self): + assert self.section_id, "<sect1> tag not found" + assert self.varlist_id, "<varlistentry> tag not found" + return self._url_pattern.format( + version=self.version, + section=self.section_id, + func_id=self.varlist_id.upper(), + ) + + _url_pattern = "https://www.postgresql.org/docs/{version}/{section}.html#{func_id}" + + +class LibpqReader: + # must be set before using the rest of the class. + app = None + + _url_pattern = ( + "https://raw.githubusercontent.com/postgres/postgres/REL_{ver}_STABLE" + "/doc/src/sgml/libpq.sgml" + ) + + data = None + + def get_url(self, func): + if not self.data: + self.parse() + + return self.data[func] + + def parse(self): + if not self.local_file.exists(): + self.download() + + logger.info("parsing libpq docs from %s", self.local_file) + self.data = {} + parser = LibpqParser(self.data, version=self.version) + with self.local_file.open("r") as f: + parser.feed(f.read()) + + def download(self): + filename = os.environ.get("LIBPQ_DOCS_FILE") + if filename: + logger.info("reading postgres libpq docs from %s", filename) + with open(filename, "rb") as f: + data = f.read() + else: + logger.info("downloading postgres libpq docs from %s", self.sgml_url) + data = urllib.request.urlopen(self.sgml_url).read() + + with self.local_file.open("wb") as f: + f.write(data) + + @property + def local_file(self): + return Path(self.app.doctreedir) / f"libpq-{self.version}.sgml" + + @property + def sgml_url(self): + return self._url_pattern.format(ver=self.version) + + @property + def version(self): + return self.app.config.libpq_docs_version + + +@lru_cache() +def get_reader(): + return LibpqReader() + + +def pq_role(name, rawtext, text, lineno, inliner, options={}, content=[]): + text = utils.unescape(text) + + reader = get_reader() + if "(" in text: + func, noise = text.split("(", 1) + noise = "(" + noise + + else: + func = text + noise = "" + + try: + url = reader.get_url(func) + except KeyError: + msg = inliner.reporter.warning( + f"function {func} not found in libpq {reader.version} docs" + ) + prb = inliner.problematic(rawtext, rawtext, msg) + return [prb], [msg] + + # For a function f(), include the () in the signature for consistency + # with a normal `thing()` + if noise == "()": + func, noise = func + noise, "" + + the_nodes = [] + the_nodes.append(nodes.reference(func, func, refuri=url)) + if noise: + the_nodes.append(nodes.Text(noise)) + + return [nodes.literal("", "", *the_nodes, **options)], [] + + +def setup(app): + app.add_config_value("libpq_docs_version", "14", "html") + roles.register_local_role("pq", pq_role) + get_reader().app = app diff --git a/docs/lib/pg3_docs.py b/docs/lib/pg3_docs.py new file mode 100644 index 0000000..05a6876 --- /dev/null +++ b/docs/lib/pg3_docs.py @@ -0,0 +1,197 @@ +""" +Customisation for docs generation. +""" + +# Copyright (C) 2020 The Psycopg Team + +import os +import re +import logging +import importlib +from typing import Dict +from collections import deque + + +def process_docstring(app, what, name, obj, options, lines): + pass + + +def before_process_signature(app, obj, bound_method): + ann = getattr(obj, "__annotations__", {}) + if "return" in ann: + # Drop "return: None" from the function signatures + if ann["return"] is None: + del ann["return"] + + +def process_signature(app, what, name, obj, options, signature, return_annotation): + pass + + +def setup(app): + app.connect("autodoc-process-docstring", process_docstring) + app.connect("autodoc-process-signature", process_signature) + app.connect("autodoc-before-process-signature", before_process_signature) + + import psycopg # type: ignore + + recover_defined_module( + psycopg, skip_modules=["psycopg._dns", "psycopg.types.shapely"] + ) + monkeypatch_autodoc() + + # Disable warnings in sphinx_autodoc_typehints because it doesn't seem that + # there is a workaround for: "WARNING: Cannot resolve forward reference in + # type annotations" + logger = logging.getLogger("sphinx.sphinx_autodoc_typehints") + logger.setLevel(logging.ERROR) + + +# Classes which may have __module__ overwritten +recovered_classes: Dict[type, str] = {} + + +def recover_defined_module(m, skip_modules=()): + """ + Find the module where classes with __module__ attribute hacked were defined. + + Autodoc will get confused and will fail to inspect attribute docstrings + (e.g. from enums and named tuples). + + Save the classes recovered in `recovered_classes`, to be used by + `monkeypatch_autodoc()`. + + """ + mdir = os.path.split(m.__file__)[0] + for fn in walk_modules(mdir): + assert fn.startswith(mdir) + modname = os.path.splitext(fn[len(mdir) + 1 :])[0].replace("/", ".") + modname = f"{m.__name__}.{modname}" + if modname in skip_modules: + continue + with open(fn) as f: + classnames = re.findall(r"^class\s+([^(:]+)", f.read(), re.M) + for cls in classnames: + cls = deep_import(f"{modname}.{cls}") + if cls.__module__ != modname: + recovered_classes[cls] = modname + + +def monkeypatch_autodoc(): + """ + Patch autodoc in order to use information found by `recover_defined_module`. + """ + from sphinx.ext.autodoc import Documenter, AttributeDocumenter + + orig_doc_get_real_modname = Documenter.get_real_modname + orig_attr_get_real_modname = AttributeDocumenter.get_real_modname + orig_attr_add_content = AttributeDocumenter.add_content + + def fixed_doc_get_real_modname(self): + if self.object in recovered_classes: + return recovered_classes[self.object] + return orig_doc_get_real_modname(self) + + def fixed_attr_get_real_modname(self): + if self.parent in recovered_classes: + return recovered_classes[self.parent] + return orig_attr_get_real_modname(self) + + def fixed_attr_add_content(self, more_content): + """ + Replace a docstring such as:: + + .. py:attribute:: ConnectionInfo.dbname + :module: psycopg + + The database name of the connection. + + :rtype: :py:class:`str` + + into: + + .. py:attribute:: ConnectionInfo.dbname + :type: str + :module: psycopg + + The database name of the connection. + + which creates a more compact representation of a property. + + """ + orig_attr_add_content(self, more_content) + if not isinstance(self.object, property): + return + iret, mret = match_in_lines(r"\s*:rtype: (.*)", self.directive.result) + iatt, matt = match_in_lines(r"\.\.", self.directive.result) + if not (mret and matt): + return + self.directive.result.pop(iret) + self.directive.result.insert( + iatt + 1, + f"{self.indent}:type: {unrest(mret.group(1))}", + source=self.get_sourcename(), + ) + + Documenter.get_real_modname = fixed_doc_get_real_modname + AttributeDocumenter.get_real_modname = fixed_attr_get_real_modname + AttributeDocumenter.add_content = fixed_attr_add_content + + +def match_in_lines(pattern, lines): + """Match a regular expression against a list of strings. + + Return the index of the first matched line and the match object. + None, None if nothing matched. + """ + for i, line in enumerate(lines): + m = re.match(pattern, line) + if m: + return i, m + else: + return None, None + + +def unrest(s): + r"""remove the reST markup from a string + + e.g. :py:data:`~typing.Optional`\[:py:class:`int`] -> Optional[int] + + required because :type: does the types lookup itself apparently. + """ + s = re.sub(r":[^`]*:`~?([^`]*)`", r"\1", s) # drop role + s = re.sub(r"\\(.)", r"\1", s) # drop escape + + # note that ~psycopg.pq.ConnStatus is converted to pq.ConnStatus + # which should be interpreted well if currentmodule is set ok. + s = re.sub(r"(?:typing|psycopg)\.", "", s) # drop unneeded modules + s = re.sub(r"~", "", s) # drop the tilde + + return s + + +def walk_modules(d): + for root, dirs, files in os.walk(d): + for f in files: + if f.endswith(".py"): + yield f"{root}/{f}" + + +def deep_import(name): + parts = deque(name.split(".")) + seen = [] + if not parts: + raise ValueError("name must be a dot-separated name") + + seen.append(parts.popleft()) + thing = importlib.import_module(seen[-1]) + while parts: + attr = parts.popleft() + seen.append(attr) + + if hasattr(thing, attr): + thing = getattr(thing, attr) + else: + thing = importlib.import_module(".".join(seen)) + + return thing diff --git a/docs/lib/sql_role.py b/docs/lib/sql_role.py new file mode 100644 index 0000000..a40c9f4 --- /dev/null +++ b/docs/lib/sql_role.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +""" + sql role + ~~~~~~~~ + + An interpreted text role to style SQL syntax in Psycopg documentation. + + :copyright: Copyright 2010 by Daniele Varrazzo. + :copyright: Copyright 2020 The Psycopg Team. +""" + +from docutils import nodes, utils +from docutils.parsers.rst import roles + + +def sql_role(name, rawtext, text, lineno, inliner, options={}, content=[]): + text = utils.unescape(text) + options["classes"] = ["sql"] + return [nodes.literal(rawtext, text, **options)], [] + + +def setup(app): + roles.register_local_role("sql", sql_role) diff --git a/docs/lib/ticket_role.py b/docs/lib/ticket_role.py new file mode 100644 index 0000000..24ec873 --- /dev/null +++ b/docs/lib/ticket_role.py @@ -0,0 +1,50 @@ +# type: ignore +""" + ticket role + ~~~~~~~~~~~ + + An interpreted text role to link docs to tickets issues. + + :copyright: Copyright 2013 by Daniele Varrazzo. + :copyright: Copyright 2021 The Psycopg Team +""" + +import re +from docutils import nodes, utils +from docutils.parsers.rst import roles + + +def ticket_role(name, rawtext, text, lineno, inliner, options={}, content=[]): + cfg = inliner.document.settings.env.app.config + if cfg.ticket_url is None: + msg = inliner.reporter.warning( + "ticket not configured: please configure ticket_url in conf.py" + ) + prb = inliner.problematic(rawtext, rawtext, msg) + return [prb], [msg] + + rv = [nodes.Text(name + " ")] + tokens = re.findall(r"(#?\d+)|([^\d#]+)", text) + for ticket, noise in tokens: + if ticket: + num = int(ticket.replace("#", "")) + + url = cfg.ticket_url % num + roles.set_classes(options) + node = nodes.reference( + ticket, utils.unescape(ticket), refuri=url, **options + ) + + rv.append(node) + + else: + assert noise + rv.append(nodes.Text(noise)) + + return rv, [] + + +def setup(app): + app.add_config_value("ticket_url", None, "env") + app.add_role("ticket", ticket_role) + app.add_role("tickets", ticket_role) diff --git a/docs/news.rst b/docs/news.rst new file mode 100644 index 0000000..46dfbe7 --- /dev/null +++ b/docs/news.rst @@ -0,0 +1,285 @@ +.. currentmodule:: psycopg + +.. index:: + single: Release notes + single: News + +``psycopg`` release notes +========================= + +Current release +--------------- + +Psycopg 3.1.7 +^^^^^^^^^^^^^ + +- Fix server-side cursors using row factories (:ticket:`#464`). + + +Psycopg 3.1.6 +^^^^^^^^^^^^^ + +- Fix `cursor.copy()` with cursors using row factories (:ticket:`#460`). + + +Psycopg 3.1.5 +^^^^^^^^^^^^^ + +- Fix array loading slowness compared to psycopg2 (:ticket:`#359`). +- Improve performance around network communication (:ticket:`#414`). +- Return `!bytes` instead of `!memoryview` from `pq.Encoding` methods + (:ticket:`#422`). +- Fix `Cursor.rownumber` to return `!None` when the result has no row to fetch + (:ticket:`#437`). +- Avoid error in Pyright caused by aliasing `!TypeAlias` (:ticket:`#439`). +- Fix `Copy.set_types()` used with `varchar` and `name` types (:ticket:`#452`). +- Improve performance using :ref:`row-factories` (:ticket:`#457`). + + +Psycopg 3.1.4 +^^^^^^^^^^^^^ + +- Include :ref:`error classes <sqlstate-exceptions>` defined in PostgreSQL 15. +- Add support for Python 3.11 (:ticket:`#305`). +- Build binary packages with libpq from PostgreSQL 15.0. + + +Psycopg 3.1.3 +^^^^^^^^^^^^^ + +- Restore the state of the connection if `Cursor.stream()` is terminated + prematurely (:ticket:`#382`). +- Fix regression introduced in 3.1 with different named tuples mangling rules + for non-ascii attribute names (:ticket:`#386`). +- Fix handling of queries with escaped percent signs (``%%``) in `ClientCursor` + (:ticket:`#399`). +- Fix possible duplicated BEGIN statements emitted in pipeline mode + (:ticket:`#401`). + + +Psycopg 3.1.2 +^^^^^^^^^^^^^ + +- Fix handling of certain invalid time zones causing problems on Windows + (:ticket:`#371`). +- Fix segfault occurring when a loader fails initialization (:ticket:`#372`). +- Fix invalid SAVEPOINT issued when entering `Connection.transaction()` within + a pipeline using an implicit transaction (:ticket:`#374`). +- Fix queries with repeated named parameters in `ClientCursor` (:ticket:`#378`). +- Distribute macOS arm64 (Apple M1) binary packages (:ticket:`#344`). + + +Psycopg 3.1.1 +^^^^^^^^^^^^^ + +- Work around broken Homebrew installation of the libpq in a non-standard path + (:ticket:`#364`) +- Fix possible "unrecognized service" error in async connection when no port + is specified (:ticket:`#366`). + + +Psycopg 3.1 +----------- + +- Add :ref:`Pipeline mode <pipeline-mode>` (:ticket:`#74`). +- Add :ref:`client-side-binding-cursors` (:ticket:`#101`). +- Add `CockroachDB <https://www.cockroachlabs.com/>`__ support in `psycopg.crdb` + (:ticket:`#313`). +- Add :ref:`Two-Phase Commit <two-phase-commit>` support (:ticket:`#72`). +- Add :ref:`adapt-enum` (:ticket:`#274`). +- Add ``returning`` parameter to `~Cursor.executemany()` to retrieve query + results (:ticket:`#164`). +- `~Cursor.executemany()` performance improved by using batch mode internally + (:ticket:`#145`). +- Add parameters to `~Cursor.copy()`. +- Add :ref:`COPY Writer objects <copy-writers>`. +- Resolve domain names asynchronously in `AsyncConnection.connect()` + (:ticket:`#259`). +- Add `pq.PGconn.trace()` and related trace functions (:ticket:`#167`). +- Add ``prepare_threshold`` parameter to `Connection` init (:ticket:`#200`). +- Add ``cursor_factory`` parameter to `Connection` init. +- Add `Error.pgconn` and `Error.pgresult` attributes (:ticket:`#242`). +- Restrict queries to be `~typing.LiteralString` as per :pep:`675` + (:ticket:`#323`). +- Add explicit type cast to values converted by `sql.Literal` (:ticket:`#205`). +- Drop support for Python 3.6. + + +Psycopg 3.0.17 +^^^^^^^^^^^^^^ + +- Fix segfaults on fork on some Linux systems using `ctypes` implementation + (:ticket:`#300`). +- Load bytea as bytes, not memoryview, using `ctypes` implementation. + + +Psycopg 3.0.16 +^^^^^^^^^^^^^^ + +- Fix missing `~Cursor.rowcount` after SHOW (:ticket:`#343`). +- Add scripts to build macOS arm64 packages (:ticket:`#162`). + + +Psycopg 3.0.15 +^^^^^^^^^^^^^^ + +- Fix wrong escaping of unprintable chars in COPY (nonetheless correctly + interpreted by PostgreSQL). +- Restore the connection to usable state after an error in `~Cursor.stream()`. +- Raise `DataError` instead of `OverflowError` loading binary intervals + out-of-range. +- Distribute ``manylinux2014`` wheel packages (:ticket:`#124`). + + +Psycopg 3.0.14 +^^^^^^^^^^^^^^ + +- Raise `DataError` dumping arrays of mixed types (:ticket:`#301`). +- Fix handling of incorrect server results, with blank sqlstate (:ticket:`#303`). +- Fix bad Float4 conversion on ppc64le/musllinux (:ticket:`#304`). + + +Psycopg 3.0.13 +^^^^^^^^^^^^^^ + +- Fix `Cursor.stream()` slowness (:ticket:`#286`). +- Fix oid for lists of integers, which might cause the server choosing + bad plans (:ticket:`#293`). +- Make `Connection.cancel()` on a closed connection a no-op instead of an + error. + + +Psycopg 3.0.12 +^^^^^^^^^^^^^^ + +- Allow `bytearray`/`memoryview` data too as `Copy.write()` input + (:ticket:`#254`). +- Fix dumping `~enum.IntEnum` in text mode, Python implementation. + + +Psycopg 3.0.11 +^^^^^^^^^^^^^^ + +- Fix `DataError` loading arrays with dimensions information (:ticket:`#253`). +- Fix hanging during COPY in case of memory error (:ticket:`#255`). +- Fix error propagation from COPY worker thread (mentioned in :ticket:`#255`). + + +Psycopg 3.0.10 +^^^^^^^^^^^^^^ + +- Leave the connection in working state after interrupting a query with Ctrl-C + (:ticket:`#231`). +- Fix `Cursor.description` after a COPY ... TO STDOUT operation + (:ticket:`#235`). +- Fix building on FreeBSD and likely other BSD flavours (:ticket:`#241`). + + +Psycopg 3.0.9 +^^^^^^^^^^^^^ + +- Set `Error.sqlstate` when an unknown code is received (:ticket:`#225`). +- Add the `!tzdata` package as a dependency on Windows in order to handle time + zones (:ticket:`#223`). + + +Psycopg 3.0.8 +^^^^^^^^^^^^^ + +- Decode connection errors in the ``client_encoding`` specified in the + connection string, if available (:ticket:`#194`). +- Fix possible warnings in objects deletion on interpreter shutdown + (:ticket:`#198`). +- Don't leave connections in ACTIVE state in case of error during COPY ... TO + STDOUT (:ticket:`#203`). + + +Psycopg 3.0.7 +^^^^^^^^^^^^^ + +- Fix crash in `~Cursor.executemany()` with no input sequence + (:ticket:`#179`). +- Fix wrong `~Cursor.rowcount` after an `~Cursor.executemany()` returning no + rows (:ticket:`#178`). + + +Psycopg 3.0.6 +^^^^^^^^^^^^^ + +- Allow to use `Cursor.description` if the connection is closed + (:ticket:`#172`). +- Don't raise exceptions on `ServerCursor.close()` if the connection is closed + (:ticket:`#173`). +- Fail on `Connection.cursor()` if the connection is closed (:ticket:`#174`). +- Raise `ProgrammingError` if out-of-order exit from transaction contexts is + detected (:tickets:`#176, #177`). +- Add `!CHECK_STANDBY` value to `~pq.ConnStatus` enum. + + +Psycopg 3.0.5 +^^^^^^^^^^^^^ + +- Fix possible "Too many open files" OS error, reported on macOS but possible + on other platforms too (:ticket:`#158`). +- Don't clobber exceptions if a transaction block exit with error and rollback + fails (:ticket:`#165`). + + +Psycopg 3.0.4 +^^^^^^^^^^^^^ + +- Allow to use the module with strict strings comparison (:ticket:`#147`). +- Fix segfault on Python 3.6 running in ``-W error`` mode, related to + `!backport.zoneinfo` `ticket #109 + <https://github.com/pganssle/zoneinfo/issues/109>`__. +- Build binary package with libpq versions not affected by `CVE-2021-23222 + <https://www.postgresql.org/support/security/CVE-2021-23222/>`__ + (:ticket:`#149`). + + +Psycopg 3.0.3 +^^^^^^^^^^^^^ + +- Release musllinux binary packages, compatible with Alpine Linux + (:ticket:`#141`). +- Reduce size of binary package by stripping debug symbols (:ticket:`#142`). +- Include typing information in the `!psycopg_binary` package. + + +Psycopg 3.0.2 +^^^^^^^^^^^^^ + +- Fix type hint for `sql.SQL.join()` (:ticket:`#127`). +- Fix type hint for `Connection.notifies()` (:ticket:`#128`). +- Fix call to `MultiRange.__setitem__()` with a non-iterable value and a + slice, now raising a `TypeError` (:ticket:`#129`). +- Fix disable cursors methods after close() (:ticket:`#125`). + + +Psycopg 3.0.1 +^^^^^^^^^^^^^ + +- Fix use of the wrong dumper reusing cursors with the same query but different + parameter types (:ticket:`#112`). + + +Psycopg 3.0 +----------- + +First stable release. Changed from 3.0b1: + +- Add :ref:`adapt-shapely` (:ticket:`#80`). +- Add :ref:`adapt-multirange` (:ticket:`#75`). +- Add `pq.__build_version__` constant. +- Don't use the extended protocol with COPY, (:tickets:`#78, #82`). +- Add ``context`` parameter to `~Connection.connect()` (:ticket:`#83`). +- Fix selection of dumper by oid after `~Copy.set_types()`. +- Drop `!Connection.client_encoding`. Use `ConnectionInfo.encoding` to read + it, and a :sql:`SET` statement to change it. +- Add binary packages for Python 3.10 (:ticket:`#103`). + + +Psycopg 3.0b1 +^^^^^^^^^^^^^ + +- First public release on PyPI. diff --git a/docs/news_pool.rst b/docs/news_pool.rst new file mode 100644 index 0000000..7f212e0 --- /dev/null +++ b/docs/news_pool.rst @@ -0,0 +1,81 @@ +.. currentmodule:: psycopg_pool + +.. index:: + single: Release notes + single: News + +``psycopg_pool`` release notes +============================== + +Current release +--------------- + +psycopg_pool 3.1.5 +^^^^^^^^^^^^^^^^^^ + +- Make sure that `!ConnectionPool.check()` refills an empty pool + (:ticket:`#438`). +- Avoid error in Pyright caused by aliasing `!TypeAlias` (:ticket:`#439`). + + +psycopg_pool 3.1.4 +^^^^^^^^^^^^^^^^^^ + +- Fix async pool exhausting connections, happening if the pool is created + before the event loop is started (:ticket:`#219`). + + +psycopg_pool 3.1.3 +^^^^^^^^^^^^^^^^^^ + +- Add support for Python 3.11 (:ticket:`#305`). + + +psycopg_pool 3.1.2 +^^^^^^^^^^^^^^^^^^ + +- Fix possible failure to reconnect after losing connection from the server + (:ticket:`#370`). + + +psycopg_pool 3.1.1 +^^^^^^^^^^^^^^^^^^ + +- Fix race condition on pool creation which might result in the pool not + filling (:ticket:`#230`). + + +psycopg_pool 3.1.0 +------------------ + +- Add :ref:`null-pool` (:ticket:`#148`). +- Add `ConnectionPool.open()` and ``open`` parameter to the pool init + (:ticket:`#151`). +- Drop support for Python 3.6. + + +psycopg_pool 3.0.3 +^^^^^^^^^^^^^^^^^^ + +- Raise `!ValueError` if `ConnectionPool` `!min_size` and `!max_size` are both + set to 0 (instead of hanging). +- Raise `PoolClosed` calling `~ConnectionPool.wait()` on a closed pool. + + +psycopg_pool 3.0.2 +^^^^^^^^^^^^^^^^^^ + +- Remove dependency on the internal `!psycopg._compat` module. + + +psycopg_pool 3.0.1 +^^^^^^^^^^^^^^^^^^ + +- Don't leave connections idle in transaction after calling + `~ConnectionPool.check()` (:ticket:`#144`). + + +psycopg_pool 3.0 +---------------- + +- First release on PyPI. diff --git a/docs/pictures/adapt.drawio b/docs/pictures/adapt.drawio new file mode 100644 index 0000000..75f61ed --- /dev/null +++ b/docs/pictures/adapt.drawio @@ -0,0 +1,107 @@ +<mxfile host="Electron" modified="2021-07-12T13:26:05.192Z" agent="5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/14.6.13 Chrome/89.0.4389.128 Electron/12.0.7 Safari/537.36" etag="kKU1DyIkJcQFc1Rxt__U" compressed="false" version="14.6.13" type="device"> + <diagram id="THISp3X85jFCtBEH0bao" name="Page-1"> + <mxGraphModel dx="675" dy="400" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0"> + <root> + <mxCell id="0" /> + <mxCell id="1" parent="0" /> + <mxCell id="uy255Msn6vtulWmyCIR1-12" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;fontFamily=Courier New;exitX=1;exitY=0.5;exitDx=0;exitDy=0;" edge="1" parent="1" source="uy255Msn6vtulWmyCIR1-29" target="uy255Msn6vtulWmyCIR1-11"> + <mxGeometry relative="1" as="geometry"> + <mxPoint x="280" y="210" as="sourcePoint" /> + </mxGeometry> + </mxCell> + <mxCell id="uy255Msn6vtulWmyCIR1-15" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;fontFamily=Courier New;exitX=1;exitY=0.5;exitDx=0;exitDy=0;" edge="1" parent="1" source="uy255Msn6vtulWmyCIR1-30" target="uy255Msn6vtulWmyCIR1-14"> + <mxGeometry relative="1" as="geometry"> + <mxPoint x="280" y="320" as="sourcePoint" /> + </mxGeometry> + </mxCell> + <mxCell id="uy255Msn6vtulWmyCIR1-39" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;fontFamily=Courier New;" edge="1" parent="1" source="uy255Msn6vtulWmyCIR1-11" target="uy255Msn6vtulWmyCIR1-14"> + <mxGeometry relative="1" as="geometry" /> + </mxCell> + <mxCell id="uy255Msn6vtulWmyCIR1-11" value=".adapters" style="text;html=1;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontFamily=Courier New;" vertex="1" parent="1"> + <mxGeometry x="330" y="185" width="80" height="20" as="geometry" /> + </mxCell> + <mxCell id="uy255Msn6vtulWmyCIR1-40" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;fontFamily=Courier New;" edge="1" parent="1" source="uy255Msn6vtulWmyCIR1-14" target="uy255Msn6vtulWmyCIR1-27"> + <mxGeometry relative="1" as="geometry" /> + </mxCell> + <mxCell id="uy255Msn6vtulWmyCIR1-14" value=".adapters" style="text;html=1;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontFamily=Courier New;" vertex="1" parent="1"> + <mxGeometry x="330" y="285" width="80" height="20" as="geometry" /> + </mxCell> + <mxCell id="uy255Msn6vtulWmyCIR1-28" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;fontFamily=Courier New;exitX=1;exitY=0.5;exitDx=0;exitDy=0;" edge="1" parent="1" source="uy255Msn6vtulWmyCIR1-31" target="uy255Msn6vtulWmyCIR1-27"> + <mxGeometry relative="1" as="geometry"> + <mxPoint x="280" y="440" as="sourcePoint" /> + </mxGeometry> + </mxCell> + <mxCell id="uy255Msn6vtulWmyCIR1-18" value=".cursor()" style="text;html=1;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontFamily=Courier New;" vertex="1" parent="1"> + <mxGeometry x="220" y="220" width="80" height="20" as="geometry" /> + </mxCell> + <mxCell id="uy255Msn6vtulWmyCIR1-26" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;fontFamily=Courier New;" edge="1" parent="1" source="uy255Msn6vtulWmyCIR1-19" target="uy255Msn6vtulWmyCIR1-25"> + <mxGeometry relative="1" as="geometry" /> + </mxCell> + <mxCell id="uy255Msn6vtulWmyCIR1-34" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;fontFamily=Courier New;" edge="1" parent="1" source="uy255Msn6vtulWmyCIR1-19" target="uy255Msn6vtulWmyCIR1-29"> + <mxGeometry relative="1" as="geometry" /> + </mxCell> + <mxCell id="uy255Msn6vtulWmyCIR1-19" value="<b>psycopg</b><br><font face="Helvetica">module</font>" style="rounded=1;whiteSpace=wrap;html=1;fontFamily=Courier New;" vertex="1" parent="1"> + <mxGeometry x="160" y="75" width="120" height="50" as="geometry" /> + </mxCell> + <UserObject label=".connect()" link="../api/connections.html" id="uy255Msn6vtulWmyCIR1-20"> + <mxCell style="text;html=1;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontFamily=Courier New;" vertex="1" parent="1"> + <mxGeometry x="220" y="125" width="80" height="20" as="geometry" /> + </mxCell> + </UserObject> + <mxCell id="uy255Msn6vtulWmyCIR1-21" value=".execute()" style="text;html=1;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontFamily=Courier New;" vertex="1" parent="1"> + <mxGeometry x="220" y="320" width="80" height="20" as="geometry" /> + </mxCell> + <mxCell id="uy255Msn6vtulWmyCIR1-37" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;fontFamily=Courier New;" edge="1" parent="1" source="uy255Msn6vtulWmyCIR1-25" target="uy255Msn6vtulWmyCIR1-11"> + <mxGeometry relative="1" as="geometry" /> + </mxCell> + <mxCell id="uy255Msn6vtulWmyCIR1-25" value=".adapters" style="text;html=1;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontFamily=Courier New;" vertex="1" parent="1"> + <mxGeometry x="330" y="90" width="80" height="20" as="geometry" /> + </mxCell> + <mxCell id="uy255Msn6vtulWmyCIR1-27" value=".adapters" style="text;html=1;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontFamily=Courier New;" vertex="1" parent="1"> + <mxGeometry x="330" y="385" width="80" height="20" as="geometry" /> + </mxCell> + <mxCell id="uy255Msn6vtulWmyCIR1-35" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;fontFamily=Courier New;" edge="1" parent="1" source="uy255Msn6vtulWmyCIR1-29" target="uy255Msn6vtulWmyCIR1-30"> + <mxGeometry relative="1" as="geometry" /> + </mxCell> + <UserObject label="<b>Connection</b><br><font face="Helvetica">object</font>" link="../api/connections.html" id="uy255Msn6vtulWmyCIR1-29"> + <mxCell style="rounded=1;whiteSpace=wrap;html=1;fontFamily=Courier New;" vertex="1" parent="1"> + <mxGeometry x="160" y="170" width="120" height="50" as="geometry" /> + </mxCell> + </UserObject> + <mxCell id="uy255Msn6vtulWmyCIR1-36" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;fontFamily=Courier New;" edge="1" parent="1" source="uy255Msn6vtulWmyCIR1-30" target="uy255Msn6vtulWmyCIR1-31"> + <mxGeometry relative="1" as="geometry" /> + </mxCell> + <mxCell id="uy255Msn6vtulWmyCIR1-30" value="<b>Cursor</b><br><font face="Helvetica">object</font>" style="rounded=1;whiteSpace=wrap;html=1;fontFamily=Courier New;" vertex="1" parent="1"> + <mxGeometry x="160" y="270" width="120" height="50" as="geometry" /> + </mxCell> + <mxCell id="uy255Msn6vtulWmyCIR1-31" value="<b>Transformer</b><br><font face="Helvetica">object</font>" style="rounded=1;whiteSpace=wrap;html=1;fontFamily=Courier New;" vertex="1" parent="1"> + <mxGeometry x="160" y="370" width="120" height="50" as="geometry" /> + </mxCell> + <mxCell id="uy255Msn6vtulWmyCIR1-46" style="rounded=0;orthogonalLoop=1;jettySize=auto;html=1;fontFamily=Helvetica;endArrow=none;endFill=0;dashed=1;dashPattern=1 1;" edge="1" parent="1" source="uy255Msn6vtulWmyCIR1-41"> + <mxGeometry relative="1" as="geometry"> + <mxPoint x="310" y="100" as="targetPoint" /> + </mxGeometry> + </mxCell> + <mxCell id="uy255Msn6vtulWmyCIR1-41" value="Has a" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontFamily=Helvetica;" vertex="1" parent="1"> + <mxGeometry x="300" y="55" width="40" height="20" as="geometry" /> + </mxCell> + <mxCell id="uy255Msn6vtulWmyCIR1-45" style="rounded=0;orthogonalLoop=1;jettySize=auto;html=1;fontFamily=Helvetica;endArrow=none;endFill=0;dashed=1;dashPattern=1 1;startSize=4;" edge="1" parent="1" source="uy255Msn6vtulWmyCIR1-42"> + <mxGeometry relative="1" as="geometry"> + <mxPoint x="220" y="150" as="targetPoint" /> + </mxGeometry> + </mxCell> + <mxCell id="uy255Msn6vtulWmyCIR1-42" value="Create" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontFamily=Helvetica;" vertex="1" parent="1"> + <mxGeometry x="150" y="130" width="40" height="20" as="geometry" /> + </mxCell> + <mxCell id="uy255Msn6vtulWmyCIR1-47" style="edgeStyle=none;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;fontFamily=Helvetica;endArrow=none;endFill=0;dashed=1;dashPattern=1 1;" edge="1" parent="1" source="uy255Msn6vtulWmyCIR1-43"> + <mxGeometry relative="1" as="geometry"> + <mxPoint x="370" y="150" as="targetPoint" /> + </mxGeometry> + </mxCell> + <mxCell id="uy255Msn6vtulWmyCIR1-43" value="Copy" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontFamily=Helvetica;" vertex="1" parent="1"> + <mxGeometry x="394" y="130" width="40" height="20" as="geometry" /> + </mxCell> + </root> + </mxGraphModel> + </diagram> +</mxfile> diff --git a/docs/pictures/adapt.svg b/docs/pictures/adapt.svg new file mode 100644 index 0000000..2c39755 --- /dev/null +++ b/docs/pictures/adapt.svg @@ -0,0 +1,3 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"> +<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" width="285px" height="366px" viewBox="-0.5 -0.5 285 366" style="background-color: rgb(255, 255, 255);"><defs/><g><path d="M 130 140 L 173.63 140" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 178.88 140 L 171.88 143.5 L 173.63 140 L 171.88 136.5 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="all"/><path d="M 130 240 L 173.63 240" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 178.88 240 L 171.88 243.5 L 173.63 240 L 171.88 236.5 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="all"/><path d="M 220 150 L 220 223.63" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 220 228.88 L 216.5 221.88 L 220 223.63 L 223.5 221.88 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="all"/><rect x="180" y="130" width="80" height="20" fill="none" stroke="none" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe flex-start; width: 78px; height: 1px; padding-top: 140px; margin-left: 182px;"><div style="box-sizing: border-box; font-size: 0; text-align: left; "><div style="display: inline-block; font-size: 12px; font-family: Courier New; color: #000000; line-height: 1.2; pointer-events: all; white-space: normal; word-wrap: normal; ">.adapters</div></div></div></foreignObject><text x="182" y="144" fill="#000000" font-family="Courier New" font-size="12px">.adapters</text></switch></g><path d="M 220 250 L 220 323.63" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 220 328.88 L 216.5 321.88 L 220 323.63 L 223.5 321.88 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="all"/><rect x="180" y="230" width="80" height="20" fill="none" stroke="none" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe flex-start; width: 78px; height: 1px; padding-top: 240px; margin-left: 182px;"><div style="box-sizing: border-box; font-size: 0; text-align: left; "><div style="display: inline-block; font-size: 12px; font-family: Courier New; color: #000000; line-height: 1.2; pointer-events: all; white-space: normal; word-wrap: normal; ">.adapters</div></div></div></foreignObject><text x="182" y="244" fill="#000000" font-family="Courier New" font-size="12px">.adapters</text></switch></g><path d="M 130 340 L 173.63 340" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 178.88 340 L 171.88 343.5 L 173.63 340 L 171.88 336.5 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="all"/><rect x="70" y="165" width="80" height="20" fill="none" stroke="none" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe flex-start; width: 78px; height: 1px; padding-top: 175px; margin-left: 72px;"><div style="box-sizing: border-box; font-size: 0; text-align: left; "><div style="display: inline-block; font-size: 12px; font-family: Courier New; color: #000000; line-height: 1.2; pointer-events: all; white-space: normal; word-wrap: normal; ">.cursor()</div></div></div></foreignObject><text x="72" y="179" fill="#000000" font-family="Courier New" font-size="12px">.cursor()</text></switch></g><path d="M 130 45 L 173.63 45" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 178.88 45 L 171.88 48.5 L 173.63 45 L 171.88 41.5 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="all"/><path d="M 70 70 L 70 108.63" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 70 113.88 L 66.5 106.88 L 70 108.63 L 73.5 106.88 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="all"/><rect x="10" y="20" width="120" height="50" rx="7.5" ry="7.5" fill="#ffffff" stroke="#000000" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 118px; height: 1px; padding-top: 45px; margin-left: 11px;"><div style="box-sizing: border-box; font-size: 0; text-align: center; "><div style="display: inline-block; font-size: 12px; font-family: Courier New; color: #000000; line-height: 1.2; pointer-events: all; white-space: normal; word-wrap: normal; "><b>psycopg</b><br /><font face="Helvetica">module</font></div></div></div></foreignObject><text x="70" y="49" fill="#000000" font-family="Courier New" font-size="12px" text-anchor="middle">psycopg...</text></switch></g><a xlink:href="../api/connections.html"><rect x="70" y="70" width="80" height="20" fill="none" stroke="none" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe flex-start; width: 78px; height: 1px; padding-top: 80px; margin-left: 72px;"><div style="box-sizing: border-box; font-size: 0; text-align: left; "><div style="display: inline-block; font-size: 12px; font-family: Courier New; color: #000000; line-height: 1.2; pointer-events: all; white-space: normal; word-wrap: normal; ">.connect()</div></div></div></foreignObject><text x="72" y="84" fill="#000000" font-family="Courier New" font-size="12px">.connect()</text></switch></g></a><rect x="70" y="265" width="80" height="20" fill="none" stroke="none" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe flex-start; width: 78px; height: 1px; padding-top: 275px; margin-left: 72px;"><div style="box-sizing: border-box; font-size: 0; text-align: left; "><div style="display: inline-block; font-size: 12px; font-family: Courier New; color: #000000; line-height: 1.2; pointer-events: all; white-space: normal; word-wrap: normal; ">.execute()</div></div></div></foreignObject><text x="72" y="279" fill="#000000" font-family="Courier New" font-size="12px">.execute()</text></switch></g><path d="M 220 55 L 220 123.63" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 220 128.88 L 216.5 121.88 L 220 123.63 L 223.5 121.88 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="all"/><rect x="180" y="35" width="80" height="20" fill="none" stroke="none" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe flex-start; width: 78px; height: 1px; padding-top: 45px; margin-left: 182px;"><div style="box-sizing: border-box; font-size: 0; text-align: left; "><div style="display: inline-block; font-size: 12px; font-family: Courier New; color: #000000; line-height: 1.2; pointer-events: all; white-space: normal; word-wrap: normal; ">.adapters</div></div></div></foreignObject><text x="182" y="49" fill="#000000" font-family="Courier New" font-size="12px">.adapters</text></switch></g><rect x="180" y="330" width="80" height="20" fill="none" stroke="none" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe flex-start; width: 78px; height: 1px; padding-top: 340px; margin-left: 182px;"><div style="box-sizing: border-box; font-size: 0; text-align: left; "><div style="display: inline-block; font-size: 12px; font-family: Courier New; color: #000000; line-height: 1.2; pointer-events: all; white-space: normal; word-wrap: normal; ">.adapters</div></div></div></foreignObject><text x="182" y="344" fill="#000000" font-family="Courier New" font-size="12px">.adapters</text></switch></g><path d="M 70 165 L 70 208.63" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 70 213.88 L 66.5 206.88 L 70 208.63 L 73.5 206.88 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="all"/><a xlink:href="../api/connections.html"><rect x="10" y="115" width="120" height="50" rx="7.5" ry="7.5" fill="#ffffff" stroke="#000000" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 118px; height: 1px; padding-top: 140px; margin-left: 11px;"><div style="box-sizing: border-box; font-size: 0; text-align: center; "><div style="display: inline-block; font-size: 12px; font-family: Courier New; color: #000000; line-height: 1.2; pointer-events: all; white-space: normal; word-wrap: normal; "><b>Connection</b><br /><font face="Helvetica">object</font></div></div></div></foreignObject><text x="70" y="144" fill="#000000" font-family="Courier New" font-size="12px" text-anchor="middle">Connection...</text></switch></g></a><path d="M 70 265 L 70 308.63" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 70 313.88 L 66.5 306.88 L 70 308.63 L 73.5 306.88 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="all"/><rect x="10" y="215" width="120" height="50" rx="7.5" ry="7.5" fill="#ffffff" stroke="#000000" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 118px; height: 1px; padding-top: 240px; margin-left: 11px;"><div style="box-sizing: border-box; font-size: 0; text-align: center; "><div style="display: inline-block; font-size: 12px; font-family: Courier New; color: #000000; line-height: 1.2; pointer-events: all; white-space: normal; word-wrap: normal; "><b>Cursor</b><br /><font face="Helvetica">object</font></div></div></div></foreignObject><text x="70" y="244" fill="#000000" font-family="Courier New" font-size="12px" text-anchor="middle">Cursor...</text></switch></g><rect x="10" y="315" width="120" height="50" rx="7.5" ry="7.5" fill="#ffffff" stroke="#000000" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 118px; height: 1px; padding-top: 340px; margin-left: 11px;"><div style="box-sizing: border-box; font-size: 0; text-align: center; "><div style="display: inline-block; font-size: 12px; font-family: Courier New; color: #000000; line-height: 1.2; pointer-events: all; white-space: normal; word-wrap: normal; "><b>Transformer</b><br /><font face="Helvetica">object</font></div></div></div></foreignObject><text x="70" y="344" fill="#000000" font-family="Courier New" font-size="12px" text-anchor="middle">Transformer...</text></switch></g><path d="M 167.14 20 L 160 45" fill="none" stroke="#000000" stroke-miterlimit="10" stroke-dasharray="1 1" pointer-events="stroke"/><rect x="150" y="0" width="40" height="20" fill="none" stroke="none" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 38px; height: 1px; padding-top: 10px; margin-left: 151px;"><div style="box-sizing: border-box; font-size: 0; text-align: center; "><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: #000000; line-height: 1.2; pointer-events: all; white-space: normal; word-wrap: normal; ">Has a</div></div></div></foreignObject><text x="170" y="14" fill="#000000" font-family="Helvetica" font-size="12px" text-anchor="middle">Has a</text></switch></g><path d="M 40 89 L 70 95" fill="none" stroke="#000000" stroke-miterlimit="10" stroke-dasharray="1 1" pointer-events="stroke"/><rect x="0" y="75" width="40" height="20" fill="none" stroke="none" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 38px; height: 1px; padding-top: 85px; margin-left: 1px;"><div style="box-sizing: border-box; font-size: 0; text-align: center; "><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: #000000; line-height: 1.2; pointer-events: all; white-space: normal; word-wrap: normal; ">Create</div></div></div></foreignObject><text x="20" y="89" fill="#000000" font-family="Helvetica" font-size="12px" text-anchor="middle">Create</text></switch></g><path d="M 244 89.55 L 220 95" fill="none" stroke="#000000" stroke-miterlimit="10" stroke-dasharray="1 1" pointer-events="stroke"/><rect x="244" y="75" width="40" height="20" fill="none" stroke="none" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 38px; height: 1px; padding-top: 85px; margin-left: 245px;"><div style="box-sizing: border-box; font-size: 0; text-align: center; "><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: #000000; line-height: 1.2; pointer-events: all; white-space: normal; word-wrap: normal; ">Copy</div></div></div></foreignObject><text x="264" y="89" fill="#000000" font-family="Helvetica" font-size="12px" text-anchor="middle">Copy</text></switch></g></g><switch><g requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"/><a transform="translate(0,-5)" xlink:href="https://www.diagrams.net/doc/faq/svg-export-text-problems" target="_blank"><text text-anchor="middle" font-size="10px" x="50%" y="100%">Viewer does not support full SVG 1.1</text></a></switch></svg>
\ No newline at end of file diff --git a/docs/release.rst b/docs/release.rst new file mode 100644 index 0000000..8fcadaf --- /dev/null +++ b/docs/release.rst @@ -0,0 +1,39 @@ +:orphan: + +How to make a psycopg release +============================= + +- Change version number in: + + - ``psycopg_c/psycopg_c/version.py`` + - ``psycopg/psycopg/version.py`` + - ``psycopg_pool/psycopg_pool/version.py`` + +- Change docs/news.rst to drop the "unreleased" mark from the version + +- Push to GitHub to run `the tests workflow`__. + + .. __: https://github.com/psycopg/psycopg/actions/workflows/tests.yml + +- Build the packages by triggering manually the `Build packages workflow`__. + + .. __: https://github.com/psycopg/psycopg/actions/workflows/packages.yml + +- If all went fine, create a tag named after the version:: + + git tag -a -s 3.0.dev1 + git push --tags + +- Download the ``artifacts.zip`` package from the last Packages workflow run. + +- Unpack the packages locally:: + + mkdir tmp + cd tmp + unzip ~/Downloads/artifact.zip + +- If the package is a testing one, upload it on TestPyPI with:: + + $ twine upload -s -r testpypi * + +- If the package is stable, omit ``-r testpypi``. |
