From 3f184f528143c2a5d7b458dad6281d8dbd61de65 Mon Sep 17 00:00:00 2001 From: Jaroslaw Zabiello Date: Sun, 22 Aug 2021 07:49:40 +0100 Subject: [PATCH 001/727] Update orjson.txt (#180) orjson >= 3.5.1 will work with M1 arm64 processors --- requirements/extras/orjson.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/extras/orjson.txt b/requirements/extras/orjson.txt index b21219704..31eb4cfa7 100644 --- a/requirements/extras/orjson.txt +++ b/requirements/extras/orjson.txt @@ -1 +1 @@ -orjson==3.4.1 +orjson>=3.5.1 From 99343ea975d540d8f19982d50952923fd83bc6cf Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 22 Aug 2021 10:15:35 +0100 Subject: [PATCH 002/727] fix error with unhashable types and compare_dicts (#182) --- .../apps/migrations/auto/diffable_table.py | 23 ++++++- .../migrations/auto/test_diffable_table.py | 63 ++++++++++++++++++- 2 files changed, 83 insertions(+), 3 deletions(-) diff --git a/piccolo/apps/migrations/auto/diffable_table.py b/piccolo/apps/migrations/auto/diffable_table.py index 199a67cfa..3e3ed03b9 100644 --- a/piccolo/apps/migrations/auto/diffable_table.py +++ b/piccolo/apps/migrations/auto/diffable_table.py @@ -20,8 +20,29 @@ def compare_dicts(dict_1, dict_2) -> t.Dict[str, t.Any]: """ Returns a new dictionary which only contains key, value pairs which are in the first dictionary and not the second. + + For example: + dict_1 = {'a': 1, 'b': 2} + dict_2 = {'a': 1} + returns {'b': 2} + + dict_1 = {'a': 2, 'b': 2} + dict_2 = {'a': 1} + returns {'a': 2, 'b': 2} + """ - return dict(set(dict_1.items()) - set(dict_2.items())) + output = {} + + for key, value in dict_1.items(): + + dict_2_value = dict_2.get(key, ...) + if dict_2_value is ...: + output[key] = value + else: + if dict_2_value != value: + output[key] = value + + return output @dataclass diff --git a/tests/apps/migrations/auto/test_diffable_table.py b/tests/apps/migrations/auto/test_diffable_table.py index 327ed6b59..7c0cda9f3 100644 --- a/tests/apps/migrations/auto/test_diffable_table.py +++ b/tests/apps/migrations/auto/test_diffable_table.py @@ -4,16 +4,75 @@ DiffableTable, compare_dicts, ) -from piccolo.columns import Varchar +from piccolo.columns import OnDelete, Varchar class TestCompareDicts(TestCase): - def test_compare_dicts(self): + def test_simple(self): + """ + Make sure that simple values are compared properly. + """ dict_1 = {"a": 1, "b": 2} dict_2 = {"a": 1, "b": 3} response = compare_dicts(dict_1, dict_2) self.assertEqual(response, {"b": 2}) + def test_missing_keys(self): + """ + Make sure that if one dictionary has keys that the other doesn't, + it works as expected. + """ + dict_1 = {"a": 1} + dict_2 = {"b": 2, "c": 3} + response = compare_dicts(dict_1, dict_2) + self.assertEqual(response, {"a": 1}) + + def test_list_value(self): + """ + Make sure list values work correctly. + """ + dict_1 = {"a": 1, "b": [1]} + dict_2 = {"a": 1, "b": [2]} + response = compare_dicts(dict_1, dict_2) + self.assertEqual(response, {"b": [1]}) + + def test_dict_value(self): + """ + Make sure dictionary values work correctly. + """ + dict_1 = {"a": 1, "b": {"x": 1}} + dict_2 = {"a": 1, "b": {"x": 1}} + response = compare_dicts(dict_1, dict_2) + self.assertEqual(response, {}) + + dict_1 = {"a": 1, "b": {"x": 1}} + dict_2 = {"a": 1, "b": {"x": 2}} + response = compare_dicts(dict_1, dict_2) + self.assertEqual(response, {"b": {"x": 1}}) + + def test_none_values(self): + """ + Make sure there are no edge cases when using None values. + """ + dict_1 = {"a": None, "b": 1} + dict_2 = {"a": None} + response = compare_dicts(dict_1, dict_2) + self.assertEqual(response, {"b": 1}) + + def test_enum_values(self): + """ + Make sure Enum values can be compared correctly. + """ + dict_1 = {"a": OnDelete.cascade} + dict_2 = {"a": OnDelete.cascade} + response = compare_dicts(dict_1, dict_2) + self.assertEqual(response, {}) + + dict_1 = {"a": OnDelete.set_default} + dict_2 = {"a": OnDelete.cascade} + response = compare_dicts(dict_1, dict_2) + self.assertEqual(response, {"a": OnDelete.set_default}) + class TestDiffableTable(TestCase): def test_subtract(self): From bf63f911326caac21ecf5f09848b0dab309fec70 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 22 Aug 2021 10:37:06 +0100 Subject: [PATCH 003/727] bumped version --- CHANGES | 8 ++++++++ piccolo/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index dbb0d182a..42448ba4d 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,14 @@ Changes ======= +0.33.1 +------ + * Bug fix, where ``compare_dicts`` was failing in migrations if any ``Column`` + had an unhashable type as an argument. For example: ``Array(default=[])``. + Thanks to @hipertracker for reporting this problem. + * Increased the minimum version of orjson, so binaries are available for Macs + running on Apple silicon (courtesy @hipertracker). + 0.33.0 ------ Fix for auto migrations when using custom primary keys (thanks to @adriangb and diff --git a/piccolo/__init__.py b/piccolo/__init__.py index b6ea1bd86..7a60ac51c 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.33.0" +__VERSION__ = "0.33.1" From a0e90bed8bfcea6bac2d4032134069ffc2002c37 Mon Sep 17 00:00:00 2001 From: Amin Alaee Date: Sun, 22 Aug 2021 22:04:23 +0430 Subject: [PATCH 004/727] `get_or_create` method (#181) * very simple draft of `get_or_create` * add `GetOrCreate` and update docs * added empty dict as default, and allow direct await for convenience Co-authored-by: Daniel Townsend --- docs/src/piccolo/query_types/objects.rst | 17 +++++++++ piccolo/query/methods/objects.py | 46 ++++++++++++++++++++++++ tests/table/test_objects.py | 25 +++++++++++++ 3 files changed, 88 insertions(+) diff --git a/docs/src/piccolo/query_types/objects.rst b/docs/src/piccolo/query_types/objects.rst index fd9cac66a..71d9d299e 100644 --- a/docs/src/piccolo/query_types/objects.rst +++ b/docs/src/piccolo/query_types/objects.rst @@ -89,6 +89,23 @@ object, you can do so using ``get_related``. >>> print(manager.name) 'Guido' +get_or_create +------------- + +With ``get_or_create`` you can get an existing record matching the criteria, +or create a new one with the ``defaults`` arguments: + +.. code-block:: python + + band = Band.objects().get_or_create( + Band.name == 'Pythonistas', defaults={Band.popularity: 100} + ).run_sync() + + # Or using string column names + band = Band.objects().get_or_create( + Band.name == 'Pythonistas', defaults={'popularity': 100} + ).run_sync() + Query clauses ------------- diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index aa7a3692e..1aa474e8b 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -14,6 +14,7 @@ WhereDelegate, ) from piccolo.querystring import QueryString +from piccolo.utils.sync import run_sync from .select import Select @@ -22,6 +23,44 @@ from piccolo.table import Table +@dataclass +class GetOrCreate: + query: Objects + where: Combinable + defaults: t.Dict[t.Union[Column, str], t.Any] + + async def run(self): + instance = await self.query.where(self.where).first().run() + if instance: + return instance + + instance = self.query.table() + setattr( + instance, + self.where.column._meta.name, # type: ignore + self.where.value, # type: ignore + ) + + for column, value in self.defaults.items(): + if isinstance(column, str): + column = instance._meta.get_column_by_name(column) + setattr(instance, column._meta.name, value) + + await instance.save().run() + + return instance + + def __await__(self): + """ + If the user doesn't explicity call .run(), proxy to it as a + convenience. + """ + return self.run().__await__() + + def run_sync(self): + return run_sync(self.run()) + + @dataclass class Objects(Query): """ @@ -64,6 +103,13 @@ def offset(self, number: int) -> Objects: self.offset_delegate.offset(number) return self + def get_or_create( + self, + where: Combinable, + defaults: t.Dict[t.Union[Column, str], t.Any] = {}, + ): + return GetOrCreate(query=self, where=where, defaults=defaults) + def order_by(self, *columns: Column, ascending=True) -> Objects: self.order_by_delegate.order_by(*columns, ascending=ascending) return self diff --git a/tests/table/test_objects.py b/tests/table/test_objects.py index 9269362cc..d1d295cea 100644 --- a/tests/table/test_objects.py +++ b/tests/table/test_objects.py @@ -59,3 +59,28 @@ def test_offset_sqlite(self): self.assertEqual( [i.name for i in response], ["Pythonistas", "Rustaceans"] ) + + def test_get_or_create(self): + Band.objects().get_or_create( + Band.name == "Pink Floyd", defaults={"popularity": 100} + ).run_sync() + + instance = ( + Band.objects().where(Band.name == "Pink Floyd").first().run_sync() + ) + + self.assertTrue(isinstance(instance, Band)) + self.assertTrue(instance.name == "Pink Floyd") + self.assertTrue(instance.popularity == 100) + + Band.objects().get_or_create( + Band.name == "Pink Floyd", defaults={Band.popularity: 100} + ).run_sync() + + instance = ( + Band.objects().where(Band.name == "Pink Floyd").first().run_sync() + ) + + self.assertTrue(isinstance(instance, Band)) + self.assertTrue(instance.name == "Pink Floyd") + self.assertTrue(instance.popularity == 100) From 1b645cb5a1b30282ca2f72b184c721f1dc8175d1 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 22 Aug 2021 20:01:46 +0100 Subject: [PATCH 005/727] bumped version --- CHANGES | 11 +++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 42448ba4d..096ba2d57 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,17 @@ Changes ======= +0.34.0 +------ +Added the ``get_or_create`` convenience method (courtesy @aminalaee). Example +usage: + +.. code-block:: python + + manager = await Manager.objects().get_or_create( + Manager.name == 'Guido' + ).run() + 0.33.1 ------ * Bug fix, where ``compare_dicts`` was failing in migrations if any ``Column`` diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 7a60ac51c..0f5f2bb33 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.33.1" +__VERSION__ = "0.34.0" From 321843267c15c4cf39bc1e5b4afb880b0bbd112b Mon Sep 17 00:00:00 2001 From: Amin Alaee Date: Tue, 24 Aug 2021 23:50:05 +0430 Subject: [PATCH 006/727] add objects `get` method (#183) * add simple get method * update docs * minor tweak to docs - mention ``None`` is returned if ``get`` finds no match Co-authored-by: Daniel Townsend --- docs/src/piccolo/query_types/objects.rst | 7 +++++++ piccolo/query/methods/objects.py | 7 ++++++- tests/table/test_objects.py | 7 +++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/docs/src/piccolo/query_types/objects.rst b/docs/src/piccolo/query_types/objects.rst index 71d9d299e..d76837eb5 100644 --- a/docs/src/piccolo/query_types/objects.rst +++ b/docs/src/piccolo/query_types/objects.rst @@ -28,6 +28,13 @@ To get certain rows: >>> Band.objects().where(Band.name == 'Pythonistas').run_sync() [] +To get a single row (or ``None`` if it doesn't exist): + +.. code-block:: python + + >>> Band.objects().get(Band.name == 'Pythonistas').run_sync() + + To get the first row: .. code-block:: python diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index 1aa474e8b..cc38c57e8 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -30,7 +30,7 @@ class GetOrCreate: defaults: t.Dict[t.Union[Column, str], t.Any] async def run(self): - instance = await self.query.where(self.where).first().run() + instance = await self.query.get(self.where).run() if instance: return instance @@ -99,6 +99,11 @@ def first(self) -> Objects: self.limit_delegate.first() return self + def get(self, where: Combinable) -> Objects: + self.where_delegate.where(where) + self.limit_delegate.first() + return self + def offset(self, number: int) -> Objects: self.offset_delegate.offset(number) return self diff --git a/tests/table/test_objects.py b/tests/table/test_objects.py index d1d295cea..055b1a9d3 100644 --- a/tests/table/test_objects.py +++ b/tests/table/test_objects.py @@ -60,6 +60,13 @@ def test_offset_sqlite(self): [i.name for i in response], ["Pythonistas", "Rustaceans"] ) + def test_get(self): + self.insert_row() + + band = Band.objects().get(Band.name == "Pythonistas").run_sync() + + self.assertTrue(band.name == "Pythonistas") + def test_get_or_create(self): Band.objects().get_or_create( Band.name == "Pink Floyd", defaults={"popularity": 100} From d1d63009a7989cefe7a4fb37574859afde026c15 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 25 Aug 2021 11:45:50 +0100 Subject: [PATCH 007/727] initial prototype for auto schema generation (#157) * initial prototype for auto schema generation * make sure the correct column type is being instantiated * use black to nicely format the string * add Timestamptz * detect if column is nullable * introspect Varchar length * slight refactor, and output required imports * added initial tests * fix tests * remove some kwargs from output * fix bug with duplicate id columns * add extra imports if foreign keys are present * add docs * fixing a bug with primary key detection * add test for unique constraints * log the db version instead of printing to avoid adding noise to output Otherwise if someone pipes `piccolo schema generate` straight to a file, it won't be valid Python as it'll have something like 'Running Postgres version 9.6' at the top. * increase code coverage * add a link from the schema docs to the `piccolo schema generate` docs --- .../projects_and_apps/included_apps.rst | 17 + docs/src/piccolo/schema/defining.rst | 4 + piccolo/apps/schema/__init__.py | 0 piccolo/apps/schema/commands/__init__.py | 0 piccolo/apps/schema/commands/generate.py | 396 ++++++++++++++++++ piccolo/apps/schema/piccolo_app.py | 9 + piccolo/engine/base.py | 6 +- piccolo/main.py | 2 + piccolo/table.py | 9 +- piccolo/utils/naming.py | 7 + requirements/requirements.txt | 1 + tests/apps/schema/__init__.py | 0 tests/apps/schema/commands/test_generate.py | 124 ++++++ tests/conftest.py | 2 + 14 files changed, 575 insertions(+), 2 deletions(-) create mode 100644 piccolo/apps/schema/__init__.py create mode 100644 piccolo/apps/schema/commands/__init__.py create mode 100644 piccolo/apps/schema/commands/generate.py create mode 100644 piccolo/apps/schema/piccolo_app.py create mode 100644 tests/apps/schema/__init__.py create mode 100644 tests/apps/schema/commands/test_generate.py diff --git a/docs/src/piccolo/projects_and_apps/included_apps.rst b/docs/src/piccolo/projects_and_apps/included_apps.rst index b7d279333..27ee45df7 100644 --- a/docs/src/piccolo/projects_and_apps/included_apps.rst +++ b/docs/src/piccolo/projects_and_apps/included_apps.rst @@ -60,6 +60,23 @@ Lets you create a new ``piccolo_conf.py`` file. See :ref:`PiccoloProjects`. piccolo project new +.. _SchemaApp: + +schema +~~~~~~ + +Lets you auto generate Piccolo ``Table`` classes from an existing database. +Make sure the credentials in ``piccolo_conf.py`` are for the database you're +interested in, then run the following: + +.. code-block:: bash + + piccolo schema generate > tables.py + +.. warning:: This feature is still a work in progress. However, even in it's + current form it will save you a lot of time. Make sure you check the + generated code to make sure it's correct. + shell ~~~~~ diff --git a/docs/src/piccolo/schema/defining.rst b/docs/src/piccolo/schema/defining.rst index 4da56c8ef..63683321c 100644 --- a/docs/src/piccolo/schema/defining.rst +++ b/docs/src/piccolo/schema/defining.rst @@ -21,6 +21,10 @@ columns. Here's a very simple schema: For a full list of columns, see :ref:`ColumnTypes`. +.. hint:: If you're using an existing database, see Piccolo's + :ref:`auto schema generation command`, which will save you some + time. + ------------------------------------------------------------------------------- Primary Key diff --git a/piccolo/apps/schema/__init__.py b/piccolo/apps/schema/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/piccolo/apps/schema/commands/__init__.py b/piccolo/apps/schema/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py new file mode 100644 index 000000000..a6042fdd9 --- /dev/null +++ b/piccolo/apps/schema/commands/generate.py @@ -0,0 +1,396 @@ +from __future__ import annotations + +import dataclasses +import typing as t + +import black +from typing_extensions import Literal + +from piccolo.columns.base import Column +from piccolo.columns.column_types import ( + JSON, + JSONB, + UUID, + BigInt, + Boolean, + Bytea, + Date, + ForeignKey, + Integer, + Interval, + Numeric, + Real, + Serial, + SmallInt, + Text, + Timestamp, + Timestamptz, + Varchar, +) +from piccolo.engine.finder import engine_finder +from piccolo.engine.postgres import PostgresEngine +from piccolo.table import Table, create_table_class +from piccolo.utils.naming import _snake_to_camel + +if t.TYPE_CHECKING: # pragma: no cover + from piccolo.engine.base import Engine + + +class ForeignKeyPlaceholder(Table): + pass + + +@dataclasses.dataclass +class RowMeta: + column_default: str + column_name: str + is_nullable: Literal["YES", "NO"] + table_name: str + character_maximum_length: t.Optional[int] + data_type: str + + @classmethod + def get_column_name_str(cls) -> str: + return ", ".join([i.name for i in dataclasses.fields(cls)]) + + +@dataclasses.dataclass +class Constraint: + constraint_type: Literal["PRIMARY KEY", "UNIQUE", "FOREIGN KEY", "CHECK"] + constraint_name: str + column_name: t.Optional[str] = None + + +@dataclasses.dataclass +class TableConstraints: + """ + All of the constraints for a certain table in the database. + """ + + tablename: str + constraints: t.List[Constraint] + + def __post_init__(self): + foreign_key_constraints: t.List[Constraint] = [] + unique_constraints: t.List[Constraint] = [] + primary_key_constraints: t.List[Constraint] = [] + + for constraint in self.constraints: + if constraint.constraint_type == "FOREIGN KEY": + foreign_key_constraints.append(constraint) + elif constraint.constraint_type == "PRIMARY KEY": + primary_key_constraints.append(constraint) + elif constraint.constraint_type == "UNIQUE": + unique_constraints.append(constraint) + + self.foreign_key_constraints = foreign_key_constraints + self.unique_constraints = unique_constraints + self.primary_key_constraints = primary_key_constraints + + def is_primary_key(self, column_name: str) -> bool: + for i in self.primary_key_constraints: + if i.column_name == column_name: + return True + return False + + def is_unique(self, column_name: str) -> bool: + for i in self.unique_constraints: + if i.column_name == column_name: + return True + return False + + def is_foreign_key(self, column_name: str) -> bool: + for i in self.foreign_key_constraints: + if i.column_name == column_name: + return True + return False + + def get_foreign_key_constraint_name(self, column_name) -> str: + for i in self.foreign_key_constraints: + if i.column_name == column_name: + return i.constraint_name + + raise ValueError("No matching constraint found") + + +@dataclasses.dataclass +class OutputSchema: + """ + Represents the schema which will be printed out. + + :param imports: + e.g. ["from piccolo.table import Table"] + :param warnings: + e.g. ["some_table.some_column unrecognised_type"] + :param tables: + e.g. ["class MyTable(Table): ..."] + + """ + + imports: t.List[str] + warnings: t.List[str] + tables: t.List[t.Type[Table]] + + def get_table_with_name(self, name: str) -> t.Optional[t.Type[Table]]: + """ + Just used by unit tests. + """ + try: + return next( + table for table in self.tables if table.__name__ == name + ) + except StopIteration: + return None + + +COLUMN_TYPE_MAP = { + "bigint": BigInt, + "boolean": Boolean, + "bytea": Bytea, + "character varying": Varchar, + "date": Date, + "integer": Integer, + "interval": Interval, + "json": JSON, + "jsonb": JSONB, + "numeric": Numeric, + "real": Real, + "smallint": SmallInt, + "text": Text, + "timestamp with time zone": Timestamptz, + "timestamp without time zone": Timestamp, + "uuid": UUID, +} + + +async def get_contraints( + table_class: t.Type[Table], tablename: str, schema_name: str = "public" +) -> TableConstraints: + """ + Get all of the constraints for a table. + + :param table_class: + Any Table subclass - just used to execute raw queries on the database. + + """ + constraints = await table_class.raw( + ( + "SELECT tc.constraint_name, tc.constraint_type, kcu.column_name " # noqa: E501 + "FROM information_schema.table_constraints tc " + "LEFT JOIN information_schema.key_column_usage kcu " + "ON tc.constraint_name = kcu.constraint_name " + "WHERE tc.table_schema = {} " + "AND tc.table_name = {} " + ), + schema_name, + tablename, + ) + return TableConstraints( + tablename=tablename, + constraints=[Constraint(**i) for i in constraints], + ) + + +async def get_tablenames( + table_class: t.Type[Table], schema_name: str = "public" +) -> t.List[str]: + """ + Get the tablenames for the schema. + + :param table_class: + Any Table subclass - just used to execute raw queries on the database. + :returns: + A list of tablenames for the given schema. + + """ + tablenames: t.List[str] = [ + i["tablename"] + for i in await table_class.raw( + ( + "SELECT tablename FROM pg_catalog.pg_tables WHERE " + "schemaname = {}" + ), + schema_name, + ).run() + ] + return tablenames + + +async def get_table_schema( + table_class: t.Type[Table], tablename: str, schema_name: str = "public" +) -> t.List[RowMeta]: + """ + Get the schema from the database. + + :param table_class: + Any Table subclass - just used to execute raw queries on the database. + :param tablename: + The name of the table whose schema we want from the database. + :param schema_name: + A Postgres database can have multiple schemas, this is the name of the + one you're interested in. + :returns: + A list, with each item containing information about a column in the + table. + + """ + row_meta_list = await table_class.raw( + ( + f"SELECT {RowMeta.get_column_name_str()} FROM " + "information_schema.columns " + "WHERE table_schema = {} " + "AND table_name = {}" + ), + schema_name, + tablename, + ).run() + return [RowMeta(**row_meta) for row_meta in row_meta_list] + + +async def get_foreign_key_reference( + table_class: t.Type[Table], constraint_name: str +) -> t.Optional[str]: + """ + Retrieve the name of the table that a foreign key is referencing. + """ + response = await table_class.raw( + ( + "SELECT table_name " + "FROM information_schema.constraint_column_usage " + "WHERE constraint_name = {};" + ), + constraint_name, + ) + if len(response) > 0: + return response[0]["table_name"] + else: + return None + + +async def get_output_schema(schema_name: str = "public") -> OutputSchema: + engine: t.Optional[Engine] = engine_finder() + + if engine is None: + raise ValueError( + "Unable to find the engine - make sure piccolo_conf is on the " + "path." + ) + + if not isinstance(engine, PostgresEngine): + raise ValueError( + "This feature is currently only supported in Postgres." + ) + + class Schema(Table, db=engine): + """ + Just used for making raw queries on the db. + """ + + pass + + tablenames = await get_tablenames(Schema, schema_name=schema_name) + + tables: t.List[t.Type[Table]] = [] + imports: t.Set[str] = {"from piccolo.table import Table"} + warnings: t.List[str] = [] + + for tablename in tablenames: + constraints = await get_contraints( + table_class=Schema, tablename=tablename, schema_name=schema_name + ) + table_schema = await get_table_schema( + table_class=Schema, tablename=tablename, schema_name=schema_name + ) + + columns: t.Dict[str, Column] = {} + + for pg_row_meta in table_schema: + data_type = pg_row_meta.data_type + column_type = COLUMN_TYPE_MAP.get(data_type, None) + column_name = pg_row_meta.column_name + + if column_type: + kwargs: t.Dict[str, t.Any] = { + "null": pg_row_meta.is_nullable == "YES", + "unique": constraints.is_unique(column_name=column_name), + } + + if constraints.is_primary_key(column_name=column_name): + kwargs["primary_key"] = True + if column_type == Integer: + column_type = Serial + + if constraints.is_foreign_key(column_name=column_name): + fk_constraint_name = ( + constraints.get_foreign_key_constraint_name( + column_name=column_name + ) + ) + column_type = ForeignKey + referenced_tablename = await get_foreign_key_reference( + table_class=Schema, constraint_name=fk_constraint_name + ) + if referenced_tablename: + kwargs["references"] = create_table_class( + _snake_to_camel(referenced_tablename) + ) + else: + kwargs["references"] = ForeignKeyPlaceholder + imports.add( + "from piccolo.columns.base import OnDelete, OnUpdate" + ) + + imports.add( + "from piccolo.column_types import " + column_type.__name__ + ) + + if column_type is Varchar: + kwargs["length"] = pg_row_meta.character_maximum_length + + columns[column_name] = column_type(**kwargs) + else: + warnings.append(f"{tablename}.{column_name} ['{data_type}']") + + table = create_table_class( + class_name=_snake_to_camel(tablename), + class_kwargs={"tablename": tablename}, + class_members=columns, + ) + tables.append(table) + + return OutputSchema( + imports=sorted(list(imports)), warnings=warnings, tables=tables + ) + + +# This is currently a beta version, and can be improved. However, having +# something working is still useful for people migrating large schemas to +# Piccolo. +async def generate(schema_name: str = "public"): + """ + Automatically generates Piccolo Table classes by introspecting the + database. Please check the generated code in case there are errors. + + """ + output_schema = await get_output_schema(schema_name=schema_name) + + output = output_schema.imports + [ + i._table_str(excluded_params=["index_method", "index", "choices"]) + for i in output_schema.tables + ] + + if output_schema.warnings: + warning_str = "\n".join(output_schema.warnings) + + output.append('"""') + output.append( + "WARNING: Unrecognised column types, added `Column` as a " + "placeholder:" + ) + output.append(warning_str) + output.append('"""') + + nicely_formatted = black.format_str( + "\n".join(output), mode=black.FileMode(line_length=79) + ) + print(nicely_formatted) diff --git a/piccolo/apps/schema/piccolo_app.py b/piccolo/apps/schema/piccolo_app.py new file mode 100644 index 000000000..ab8f2d13f --- /dev/null +++ b/piccolo/apps/schema/piccolo_app.py @@ -0,0 +1,9 @@ +from piccolo.conf.apps import AppConfig, Command + +from .commands.generate import generate + +APP_CONFIG = AppConfig( + app_name="schema", + migrations_folder_path="", + commands=[Command(callable=generate, aliases=["g", "create", "new"])], +) diff --git a/piccolo/engine/base.py b/piccolo/engine/base.py index 3f20ed0d8..968fe049d 100644 --- a/piccolo/engine/base.py +++ b/piccolo/engine/base.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging import typing as t from abc import ABCMeta, abstractmethod @@ -11,6 +12,9 @@ from piccolo.query.base import Query +logger = logging.getLogger(__file__) + + class Batch: pass @@ -60,7 +64,7 @@ async def check_version(self): return engine_type = self.engine_type.capitalize() - print(f"Running {engine_type} version {version_number}") + logger.info(f"Running {engine_type} version {version_number}") if version_number < self.min_version_number: message = ( f"This version of {self.engine_type} isn't supported " diff --git a/piccolo/main.py b/piccolo/main.py index 01596f8c7..581a44b7a 100644 --- a/piccolo/main.py +++ b/piccolo/main.py @@ -17,6 +17,7 @@ from piccolo.apps.migrations.piccolo_app import APP_CONFIG as migrations_config from piccolo.apps.playground.piccolo_app import APP_CONFIG as playground_config from piccolo.apps.project.piccolo_app import APP_CONFIG as project_config +from piccolo.apps.schema.piccolo_app import APP_CONFIG as schema_config from piccolo.apps.shell.piccolo_app import APP_CONFIG as shell_config from piccolo.apps.sql_shell.piccolo_app import APP_CONFIG as sql_shell_config from piccolo.apps.user.piccolo_app import APP_CONFIG as user_config @@ -62,6 +63,7 @@ def main(): migrations_config, playground_config, project_config, + schema_config, shell_config, sql_shell_config, user_config, diff --git a/piccolo/table.py b/piccolo/table.py index 1b05ace58..5ebaf400e 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -743,13 +743,17 @@ def _get_index_name(cls, column_names: t.List[str]) -> str: ########################################################################### @classmethod - def _table_str(cls, abbreviated=False): + def _table_str(cls, abbreviated=False, excluded_params: t.List[str] = []): """ Returns a basic string representation of the table and its columns. Used by the playground. :param abbreviated: If True, a very high level representation is printed out. + :param excluded_params: + Lets us find a middle ground between outputting every kwarg, and + the abbreviated version with very few kwargs. For example + `['index_method']`, if we want to show all kwargs but index_method. """ spacer = "\n " @@ -757,6 +761,9 @@ def _table_str(cls, abbreviated=False): for col in cls._meta.columns: params: t.List[str] = [] for key, value in col._meta.params.items(): + if key in excluded_params: + continue + _value: str = "" if inspect.isclass(value): _value = value.__name__ diff --git a/piccolo/utils/naming.py b/piccolo/utils/naming.py index 62cd03b53..efc04cc2c 100644 --- a/piccolo/utils/naming.py +++ b/piccolo/utils/naming.py @@ -6,3 +6,10 @@ def _camel_to_snake(string: str): Convert CamelCase to snake_case. """ return inflection.underscore(string) + + +def _snake_to_camel(string: str): + """ + Convert snake_case to CamelCase. + """ + return inflection.camelize(string) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 06c21db8a..56ca3c765 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -3,3 +3,4 @@ colorama>=0.4.0 Jinja2>=2.11.0 targ>=0.3.3 inflection>=0.5.1 +typing-extensions==3.10.0.0 diff --git a/tests/apps/schema/__init__.py b/tests/apps/schema/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/apps/schema/commands/test_generate.py b/tests/apps/schema/commands/test_generate.py new file mode 100644 index 000000000..db3ee04db --- /dev/null +++ b/tests/apps/schema/commands/test_generate.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +import typing as t +from unittest import TestCase + +from piccolo.apps.schema.commands.generate import ( + OutputSchema, + generate, + get_output_schema, +) +from piccolo.columns.column_types import ( + JSON, + JSONB, + UUID, + BigInt, + Boolean, + Bytea, + Date, + ForeignKey, + Integer, + Interval, + Numeric, + Real, + SmallInt, + Text, + Timestamp, + Timestamptz, + Varchar, +) +from piccolo.table import Table +from piccolo.utils.sync import run_sync +from tests.base import postgres_only + + +class SmallTable(Table): + varchar_col = Varchar() + + +class MegaTable(Table): + """ + A table containing all of the column types, and different column kwargs. + """ + + bigint_col = BigInt() + boolean_col = Boolean() + bytea_col = Bytea() + date_col = Date() + foreignkey_col = ForeignKey(SmallTable) + integer_col = Integer() + interval_col = Interval() + json_col = JSON() + jsonb_col = JSONB() + numeric_col = Numeric() + real_col = Real() + smallint_col = SmallInt() + text_col = Text() + timestamp_col = Timestamp() + timestamptz_col = Timestamptz() + uuid_col = UUID() + varchar_col = Varchar() + + unique_col = Varchar(unique=True) + null_col = Varchar(null=True) + not_null_col = Varchar(null=False) + + +@postgres_only +class TestGenerate(TestCase): + def setUp(self): + for table_class in (SmallTable, MegaTable): + table_class.create_table().run_sync() + + def tearDown(self): + for table_class in (MegaTable, SmallTable): + table_class.alter().drop_table().run_sync() + + def _compare_table_columns( + self, table_1: t.Type[Table], table_2: t.Type[Table] + ): + """ + Make sure that for each column in table_1, there is a corresponding + column in table_2 of the same type. + """ + column_names = [ + column._meta.name for column in table_1._meta.non_default_columns + ] + for column_name in column_names: + col_1 = table_1._meta.get_column_by_name(column_name) + col_2 = table_2._meta.get_column_by_name(column_name) + + # Make sure they're the same type + self.assertEqual(type(col_1), type(col_2)) + + # Make sure they're both nullable or not + self.assertEqual(col_1._meta.null, col_2._meta.null) + + # Make sure the max length is the same + if isinstance(col_1, Varchar) and isinstance(col_2, Varchar): + self.assertEqual(col_1.length, col_2.length) + + # Make sure the unique constraint is the same + self.assertEqual(col_1._meta.unique, col_2._meta.unique) + + def test_get_output_schema(self): + """ + Make sure that the a Piccolo schema can be generated from the database. + """ + output_schema: OutputSchema = run_sync(get_output_schema()) + + self.assertTrue(len(output_schema.warnings) == 0) + self.assertTrue(len(output_schema.tables) == 2) + self.assertTrue(len(output_schema.imports) > 0) + + MegaTable_ = output_schema.get_table_with_name("MegaTable") + self._compare_table_columns(MegaTable, MegaTable_) + + SmallTable_ = output_schema.get_table_with_name("SmallTable") + self._compare_table_columns(SmallTable, SmallTable_) + + def test_generate(self): + """ + Test the main generate command runs without errors. + """ + run_sync(generate()) diff --git a/tests/conftest.py b/tests/conftest.py index b1c7257be..fe425df69 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,6 +18,8 @@ async def drop_tables(): "my_table", "recording_studio", "shirt", + "mega_table", + "small_table", ]: await ENGINE._run_in_new_connection(f"DROP TABLE IF EXISTS {table}") From e0af3235e872e2554ecac32ee6013ed387f98602 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 25 Aug 2021 11:51:39 +0100 Subject: [PATCH 008/727] added tester app (#187) * added tester app * reset PICCOLO_CONF once the tests complete * added an additional test for when an exception is raised in the context manager body * add a test for the `tester run` command * added docs * make it more obvious how to send multiple args to pytest --- docs/src/piccolo/engines/index.rst | 2 + .../projects_and_apps/included_apps.rst | 33 +++++++++ piccolo/apps/tester/__init__.py | 0 piccolo/apps/tester/commands/__init__.py | 0 piccolo/apps/tester/commands/run.py | 71 ++++++++++++++++++ piccolo/apps/tester/piccolo_app.py | 11 +++ piccolo/main.py | 2 + requirements/extras/pytest.txt | 1 + tests/apps/tester/__init__.py | 0 tests/apps/tester/commands/test_run.py | 73 +++++++++++++++++++ 10 files changed, 193 insertions(+) create mode 100644 piccolo/apps/tester/__init__.py create mode 100644 piccolo/apps/tester/commands/__init__.py create mode 100644 piccolo/apps/tester/commands/run.py create mode 100644 piccolo/apps/tester/piccolo_app.py create mode 100644 requirements/extras/pytest.txt create mode 100644 tests/apps/tester/__init__.py create mode 100644 tests/apps/tester/commands/test_run.py diff --git a/docs/src/piccolo/engines/index.rst b/docs/src/piccolo/engines/index.rst index 031cf3eb5..c3b3352fa 100644 --- a/docs/src/piccolo/engines/index.rst +++ b/docs/src/piccolo/engines/index.rst @@ -61,6 +61,8 @@ Here's an example ``piccolo_conf.py`` file: .. hint:: A good place for your piccolo_conf file is at the root of your project, where the Python interpreter will be launched. +.. _PICCOLO_CONF: + PICCOLO_CONF environment variable ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/src/piccolo/projects_and_apps/included_apps.rst b/docs/src/piccolo/projects_and_apps/included_apps.rst index 27ee45df7..abb3aa9f1 100644 --- a/docs/src/piccolo/projects_and_apps/included_apps.rst +++ b/docs/src/piccolo/projects_and_apps/included_apps.rst @@ -101,6 +101,39 @@ need to run raw SQL queries on your database. For it to work, the underlying command needs to be on the path (i.e. ``psql`` or ``sqlite3`` depending on which you're using). +tester +~~~~~~ + +Launches `pytest `_ , which runs your unit test suite. The +advantage of using this rather than running ``pytest`` directly, is the +``PICCOLO_CONF`` environment variable will automatically be set before the +testing starts, and will be restored to it's initial value once the tests +finish. + +.. code-block:: bash + + piccolo tester run + +Setting the :ref:`PICCOLO_CONF` environment variable means your +code will use the database engine specified in that file for the duration of +the testing. + +By default ``piccolo tester run`` sets ``PICCOLO_CONF`` to +``'piccolo_conf_test'``, meaning that a file called ``piccolo_conf_test.py`` +will be imported. + +If you prefer, you can set a custom ``PICCOLO_CONF`` value: + +.. code-block:: bash + + piccolo tester run --piccolo_conf=my_custom_piccolo_conf + +You can also pass arguments to pytest: + +.. code-block:: bash + + piccolo tester run --pytest_args="-s foo" + ------------------------------------------------------------------------------- Optional includes diff --git a/piccolo/apps/tester/__init__.py b/piccolo/apps/tester/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/piccolo/apps/tester/commands/__init__.py b/piccolo/apps/tester/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/piccolo/apps/tester/commands/run.py b/piccolo/apps/tester/commands/run.py new file mode 100644 index 000000000..cd7afd1b3 --- /dev/null +++ b/piccolo/apps/tester/commands/run.py @@ -0,0 +1,71 @@ +import os +import sys +import typing as t + +from _pytest.config import ExitCode + + +class set_env_var: + def __init__(self, var_name: str, temp_value: str): + """ + Temporarily set an environment variable. + + :param var_name: + The name of the environment variable to temporarily change. + :temp_value: + The value that the environment variable will temporarily be set to, + before being reset to it's pre-existing value. + + """ + self.var_name = var_name + self.temp_value = temp_value + + def set_var(self, value: str): + os.environ[self.var_name] = value + + def get_var(self) -> t.Optional[str]: + return os.environ.get(self.var_name) + + def __enter__(self): + self.existing_value = self.get_var() + self.set_var(self.temp_value) + + def __exit__(self, *args): + if self.existing_value is None: + del os.environ[self.var_name] + else: + self.set_var(self.existing_value) + + +def run_pytest( + pytest_args: t.List[str], +) -> t.Union[int, ExitCode]: # pragma: no cover + try: + import pytest + except ImportError: + sys.exit( + "Couldn't find pytest. Please use `pip install 'piccolo[pytest]' " + "to use this feature." + ) + + return pytest.main(pytest_args) + + +def run( + pytest_args: str = "", piccolo_conf: str = "piccolo_conf_test" +) -> None: + """ + Run your unit test suite using Pytest. + + :param piccolo_conf: + The piccolo_conf module to use when running your tests. This will + contain the database settings you want to use. For example + `my_folder.piccolo_conf_test`. + :param pytest_args: + Any options you want to pass to Pytest. For example + `piccolo tester run --pytest_args="-s"`. + + """ + with set_env_var(var_name="PICCOLO_CONF", temp_value=piccolo_conf): + args = pytest_args.split(" ") + sys.exit(run_pytest(args)) diff --git a/piccolo/apps/tester/piccolo_app.py b/piccolo/apps/tester/piccolo_app.py new file mode 100644 index 000000000..1bda3fd0c --- /dev/null +++ b/piccolo/apps/tester/piccolo_app.py @@ -0,0 +1,11 @@ +from piccolo.conf.apps import AppConfig + +from .commands.run import run + +APP_CONFIG = AppConfig( + app_name="tester", + migrations_folder_path="", + table_classes=[], + migration_dependencies=[], + commands=[run], +) diff --git a/piccolo/main.py b/piccolo/main.py index 581a44b7a..2f58f493c 100644 --- a/piccolo/main.py +++ b/piccolo/main.py @@ -20,6 +20,7 @@ from piccolo.apps.schema.piccolo_app import APP_CONFIG as schema_config from piccolo.apps.shell.piccolo_app import APP_CONFIG as shell_config from piccolo.apps.sql_shell.piccolo_app import APP_CONFIG as sql_shell_config +from piccolo.apps.tester.piccolo_app import APP_CONFIG as tester_config from piccolo.apps.user.piccolo_app import APP_CONFIG as user_config from piccolo.conf.apps import AppRegistry, Finder from piccolo.utils.sync import run_sync @@ -66,6 +67,7 @@ def main(): schema_config, shell_config, sql_shell_config, + tester_config, user_config, ]: for command in _app_config.commands: diff --git a/requirements/extras/pytest.txt b/requirements/extras/pytest.txt new file mode 100644 index 000000000..e079f8a60 --- /dev/null +++ b/requirements/extras/pytest.txt @@ -0,0 +1 @@ +pytest diff --git a/tests/apps/tester/__init__.py b/tests/apps/tester/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/apps/tester/commands/test_run.py b/tests/apps/tester/commands/test_run.py new file mode 100644 index 000000000..640910e81 --- /dev/null +++ b/tests/apps/tester/commands/test_run.py @@ -0,0 +1,73 @@ +import os +from unittest import TestCase +from unittest.mock import MagicMock, patch + +from piccolo.apps.tester.commands.run import run, set_env_var + + +class TestSetEnvVar(TestCase): + def test_no_existing_value(self): + """ + Make sure the environment variable is set correctly, when there is + no existing value. + """ + var_name = "PICCOLO_TEST_1" + + # Make sure it definitely doesn't exist already + if os.environ.get(var_name) is not None: + del os.environ[var_name] + + new_value = "hello world" + + with set_env_var(var_name=var_name, temp_value=new_value): + self.assertEqual(os.environ.get(var_name), new_value) + + self.assertEqual(os.environ.get(var_name), None) + + def test_existing_value(self): + """ + Make sure the environment variable is set correctly, when there is + an existing value. + """ + var_name = "PICCOLO_TEST_2" + initial_value = "hello" + new_value = "goodbye" + + os.environ[var_name] = initial_value + + with set_env_var(var_name=var_name, temp_value=new_value): + self.assertEqual(os.environ.get(var_name), new_value) + + self.assertEqual(os.environ.get(var_name), initial_value) + + def test_raise_exception(self): + """ + Make sure the environment variable is still reset, even if an exception + is raised within the context manager body. + """ + var_name = "PICCOLO_TEST_3" + initial_value = "hello" + new_value = "goodbye" + + os.environ[var_name] = initial_value + + class FakeException(Exception): + pass + + try: + with set_env_var(var_name=var_name, temp_value=new_value): + self.assertEqual(os.environ.get(var_name), new_value) + raise FakeException("Something went wrong ...") + except FakeException: + pass + + self.assertEqual(os.environ.get(var_name), initial_value) + + +class TestRun(TestCase): + @patch("piccolo.apps.tester.commands.run.run_pytest") + def test_success(self, pytest: MagicMock): + with self.assertRaises(SystemExit): + run(pytest_args="-s foo", piccolo_conf="my_piccolo_conf") + + pytest.assert_called_once_with(["-s", "foo"]) From 1ce52f9029b24c5bcf10e7acb6b463bbd4b2d098 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Wed, 25 Aug 2021 21:12:25 +1000 Subject: [PATCH 009/727] Update PrimaryKey deprecation warning (#176) * Update PrimaryKey deprecation warning * tweaks to deprecation warning Co-authored-by: Daniel Townsend --- piccolo/columns/column_types.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index ba90aafe1..bdcc424b3 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -529,8 +529,11 @@ def __init__(self, **kwargs) -> None: kwargs.update({"primary_key": True, "index": False}) colored_warning( - "`PrimaryKey` is deprecated and " - "will be removed in future versions.", + "`PrimaryKey` is deprecated and will be removed in future " + "versions. Use `UUID(primary_key=True)` or " + "`Serial(primary_key=True)` instead. If no primary key column is " + "specified, Piccolo will automatically add one for you called " + "`id`.", category=DeprecationWarning, ) From 26c1ccc619adc176782b3b9a3529bfbd0a7ea48b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 25 Aug 2021 12:30:54 +0100 Subject: [PATCH 010/727] bumped version --- CHANGES | 18 ++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 096ba2d57..8f2b6216c 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,24 @@ Changes ======= +0.35.0 +------ + + * Improved ``ForeignKey`` deprecation warning (courtesy @tonybaloney). + * Added ``piccolo schema generate`` which creates a Piccolo schema from an + existing database. + * Added ``piccolo tester run`` which is a wrapper around pytest, and + temporarily sets ``PICCOLO_CONF``, so a test database is used. + * Added the ``get`` convenience method (courtesy @aminalaee). It returns the + first matching record, or ``None`` if there's no match. For example: + + .. code-block:: python + + manager = await Manager.objects().get(Manager.name == 'Guido').run() + + # This is equivalent to: + manager = await Manager.objects().where(Manager.name == 'Guido').first().run() + 0.34.0 ------ Added the ``get_or_create`` convenience method (courtesy @aminalaee). Example diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 0f5f2bb33..953e0a7e5 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.34.0" +__VERSION__ = "0.35.0" From 98248660f8e348c7f612d369726d994d18cd1c7a Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 25 Aug 2021 12:51:30 +0100 Subject: [PATCH 011/727] fix typo --- CHANGES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 8f2b6216c..9e5fd7a8c 100644 --- a/CHANGES +++ b/CHANGES @@ -4,7 +4,7 @@ Changes 0.35.0 ------ - * Improved ``ForeignKey`` deprecation warning (courtesy @tonybaloney). + * Improved ``PrimaryKey`` deprecation warning (courtesy @tonybaloney). * Added ``piccolo schema generate`` which creates a Piccolo schema from an existing database. * Added ``piccolo tester run`` which is a wrapper around pytest, and From e8803f7d4488d37618c43b73afc01f4c506998ac Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 25 Aug 2021 17:05:29 +0100 Subject: [PATCH 012/727] allow like clauses without wildcards (#192) It previously wasn't possible to do a LIKE clause with no wildcards. So if you just wanted to match on 'FOO', 'Foo', and 'foo', and not 'foo bar' or 'baz foo'. --- docs/src/piccolo/query_clauses/where.rst | 6 +- piccolo/columns/base.py | 27 +++- tests/columns/test_base.py | 19 --- tests/table/test_select.py | 169 ++++++++++++++++------- 4 files changed, 144 insertions(+), 77 deletions(-) diff --git a/docs/src/piccolo/query_clauses/where.rst b/docs/src/piccolo/query_clauses/where.rst index 4137c2cd2..6878e2297 100644 --- a/docs/src/piccolo/query_clauses/where.rst +++ b/docs/src/piccolo/query_clauses/where.rst @@ -72,7 +72,11 @@ The percentage operator is required to designate where the match should occur. b.name.like('%is%') # Matches anywhere in string ).run_sync() -``ilike`` is identical, except it's case insensitive. + b.select().where( + b.name.like('Pythonistas') # Matches the entire string + ).run_sync() + +``ilike`` is identical, except it's Postgres specific and case insensitive. ------------------------------------------------------------------------------- diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 8fff7e600..17d76eb20 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -424,25 +424,38 @@ def not_in(self, values: t.List[t.Any]) -> Where: return Where(column=self, values=values, operator=NotIn) def like(self, value: str) -> Where: - if "%" not in value: - raise ValueError("% is required for like operators") + """ + Both SQLite and Postgres support LIKE, but they mean different things. + + In Postgres, LIKE is case sensitive (i.e. 'foo' equals 'foo', but + 'foo' doesn't equal 'Foo'). + + In SQLite, LIKE is case insensitive for ASCII characters + (i.e. 'foo' equals 'Foo'). But not for non-ASCII characters. To learn + more, see the docs: + + https://sqlite.org/lang_expr.html#the_like_glob_regexp_and_match_operators + + """ return Where(column=self, value=value, operator=Like) def ilike(self, value: str) -> Where: - if "%" not in value: - raise ValueError("% is required for ilike operators") + """ + Only Postgres supports ILIKE. It's used for case insensitive matching. + + For SQLite, it's just proxied to a LIKE query instead. + + """ if self._meta.engine_type == "postgres": operator: t.Type[ComparisonOperator] = ILike else: colored_warning( - "SQLite doesn't support ILIKE currently, falling back to LIKE." + "SQLite doesn't support ILIKE, falling back to LIKE." ) operator = Like return Where(column=self, value=value, operator=operator) def not_like(self, value: str) -> Where: - if "%" not in value: - raise ValueError("% is required for like operators") return Where(column=self, value=value, operator=NotLike) def __lt__(self, value) -> Where: diff --git a/tests/columns/test_base.py b/tests/columns/test_base.py index 714b0dd0d..091e1b764 100644 --- a/tests/columns/test_base.py +++ b/tests/columns/test_base.py @@ -10,25 +10,6 @@ class MyTable(Table): name = Varchar() -class TestColumn(TestCase): - def test_like_raises(self): - """ - Make sure an invalid 'like' argument raises an exception. Should - contain a % symbol. - """ - column = MyTable.name - with self.assertRaises(ValueError): - column.like("guido") - - with self.assertRaises(ValueError): - column.ilike("guido") - - # Make sure valid args don't raise an exception. - for arg in ["%guido", "guido%", "%guido%"]: - column.like("%foo") - column.ilike("foo%") - - class TestCopy(TestCase): def test_copy(self): """ diff --git a/tests/table/test_select.py b/tests/table/test_select.py index 27593775e..9ceba55bd 100644 --- a/tests/table/test_select.py +++ b/tests/table/test_select.py @@ -65,27 +65,134 @@ def test_where_equals(self): ) self.assertEqual(response, [{"name": "Pythonistas"}]) - def test_where_like(self): + @postgres_only + def test_where_like_postgres(self): + """ + Postgres' LIKE is case sensitive. + """ self.insert_rows() - response = ( - Band.select(Band.name).where(Band.name.like("Python%")).run_sync() - ) + for like_query in ("Python%", "Pythonistas", "%istas", "%ist%"): + response = ( + Band.select(Band.name) + .where(Band.name.like(like_query)) + .run_sync() + ) - print(f"response = {response}") + self.assertEqual(response, [{"name": "Pythonistas"}]) + + for like_query in ( + "PyThonISTAs", + "PYth%", + "%ISTAS", + "%Ist%", + "PYTHONISTAS", + ): + response = ( + Band.select(Band.name) + .where(Band.name.like(like_query)) + .run_sync() + ) - self.assertEqual(response, [{"name": "Pythonistas"}]) + self.assertEqual(response, []) - def test_where_ilike(self): + @sqlite_only + def test_where_like_sqlite(self): + """ + SQLite's LIKE is case insensitive for ASCII characters, + i.e. a == 'A' -> True. + """ self.insert_rows() - response = ( - Band.select(Band.name).where(Band.name.ilike("python%")).run_sync() - ) + for like_query in ( + "Python%", + "Pythonistas", + "%istas", + "%ist%", + "PYTHONISTAS", + ): + response = ( + Band.select(Band.name) + .where(Band.name.like(like_query)) + .run_sync() + ) - print(f"response = {response}") + self.assertEqual(response, [{"name": "Pythonistas"}]) + + for like_query in ( + "xyz", + "XYZ%", + "%xyz", + "%xyz%", + ): + response = ( + Band.select(Band.name) + .where(Band.name.like(like_query)) + .run_sync() + ) - self.assertEqual(response, [{"name": "Pythonistas"}]) + self.assertEqual(response, []) + + @sqlite_only + def test_where_ilike_sqlite(self): + """ + SQLite doesn't support ILIKE, so it's just a proxy to LIKE. We still + have a test to make sure it proxies correctly. + """ + self.insert_rows() + + for ilike_query in ( + "Python%", + "Pythonistas", + "pythonistas", + "PytHonIStas", + "%istas", + "%ist%", + "%IST%", + ): + + self.assertEqual( + Band.select(Band.name) + .where(Band.name.ilike(ilike_query)) + .run_sync(), + Band.select(Band.name) + .where(Band.name.like(ilike_query)) + .run_sync(), + ) + + @postgres_only + def test_where_ilike_postgres(self): + """ + Only Postgres has ILIKE - it's not in the SQL standard. It's for + case insensitive matching, i.e. 'Foo' == 'foo' -> True. + """ + self.insert_rows() + + for ilike_query in ( + "Python%", + "Pythonistas", + "pythonistas", + "PytHonIStas", + "%istas", + "%ist%", + "%IST%", + ): + response = ( + Band.select(Band.name) + .where(Band.name.ilike(ilike_query)) + .run_sync() + ) + + self.assertEqual(response, [{"name": "Pythonistas"}]) + + for ilike_query in ("Pythonistas1", "%123", "%xyz%"): + response = ( + Band.select(Band.name) + .where(Band.name.ilike(ilike_query)) + .run_sync() + ) + + self.assertEqual(response, []) def test_where_not_like(self): self.insert_rows() @@ -97,8 +204,6 @@ def test_where_not_like(self): .run_sync() ) - print(f"response = {response}") - self.assertEqual( response, [{"name": "CSharps"}, {"name": "Rustaceans"}] ) @@ -110,8 +215,6 @@ def test_where_greater_than(self): Band.select(Band.name).where(Band.popularity > 1000).run_sync() ) - print(f"response = {response}") - self.assertEqual(response, [{"name": "Rustaceans"}]) def test_where_is_null(self): @@ -159,8 +262,6 @@ def test_where_greater_equal_than(self): .run_sync() ) - print(f"response = {response}") - self.assertEqual( response, [{"name": "Pythonistas"}, {"name": "Rustaceans"}] ) @@ -172,8 +273,6 @@ def test_where_less_than(self): Band.select(Band.name).where(Band.popularity < 1000).run_sync() ) - print(f"response = {response}") - self.assertEqual(response, [{"name": "CSharps"}]) def test_where_less_equal_than(self): @@ -183,8 +282,6 @@ def test_where_less_equal_than(self): Band.select(Band.name).where(Band.popularity <= 1000).run_sync() ) - print(f"response = {response}") - self.assertEqual( response, [{"name": "Pythonistas"}, {"name": "CSharps"}] ) @@ -201,8 +298,6 @@ def test_where_raw(self): .run_sync() ) - print(f"response = {response}") - self.assertEqual(response, [{"name": "Pythonistas"}]) def test_where_raw_with_args(self): @@ -218,8 +313,6 @@ def test_where_raw_with_args(self): .run_sync() ) - print(f"response = {response}") - self.assertEqual(response, [{"name": "Pythonistas"}]) def test_where_raw_combined_with_where(self): @@ -236,8 +329,6 @@ def test_where_raw_combined_with_where(self): .run_sync() ) - print(f"response = {response}") - self.assertEqual( response, [{"name": "Pythonistas"}, {"name": "Rustaceans"}] ) @@ -251,8 +342,6 @@ def test_where_and(self): .run_sync() ) - print(f"response = {response}") - self.assertEqual(response, [{"name": "Pythonistas"}]) def test_where_or(self): @@ -265,8 +354,6 @@ def test_where_or(self): .run_sync() ) - print(f"response = {response}") - self.assertEqual( response, [{"name": "CSharps"}, {"name": "Rustaceans"}] ) @@ -285,8 +372,6 @@ def test_multiple_where(self): response = query.run_sync() - print(f"response = {response}") - self.assertEqual(response, [{"name": "Rustaceans"}]) self.assertTrue("AND" in query.__str__()) @@ -307,8 +392,6 @@ def test_complex_where(self): response = query.run_sync() - print(f"response = {response}") - self.assertEqual( response, [{"name": "CSharps"}, {"name": "Rustaceans"}] ) @@ -320,8 +403,6 @@ def test_limit(self): Band.select(Band.name).order_by(Band.name).limit(1).run_sync() ) - print(f"response = {response}") - self.assertEqual(response, [{"name": "CSharps"}]) @postgres_only @@ -332,8 +413,6 @@ def test_offset_postgres(self): Band.select(Band.name).order_by(Band.name).offset(1).run_sync() ) - print(f"response = {response}") - self.assertEqual( response, [{"name": "Pythonistas"}, {"name": "Rustaceans"}] ) @@ -353,8 +432,6 @@ def test_offset_sqlite(self): query = query.limit(5) response = query.run_sync() - print(f"response = {response}") - self.assertEqual( response, [{"name": "Pythonistas"}, {"name": "Rustaceans"}] ) @@ -366,8 +443,6 @@ def test_first(self): Band.select(Band.name).order_by(Band.name).first().run_sync() ) - print(f"response = {response}") - self.assertEqual(response, {"name": "CSharps"}) def test_order_by_ascending(self): @@ -377,8 +452,6 @@ def test_order_by_ascending(self): Band.select(Band.name).order_by(Band.name).limit(1).run_sync() ) - print(f"response = {response}") - self.assertEqual(response, [{"name": "CSharps"}]) def test_order_by_decending(self): @@ -391,8 +464,6 @@ def test_order_by_decending(self): .run_sync() ) - print(f"response = {response}") - self.assertEqual(response, [{"name": "Rustaceans"}]) def test_count(self): @@ -400,8 +471,6 @@ def test_count(self): response = Band.count().where(Band.name == "Pythonistas").run_sync() - print(f"response = {response}") - self.assertEqual(response, 1) def test_distinct(self): From 9bddedb90032e2cdb836cf097545b4039b1d5f4e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 25 Aug 2021 17:12:46 +0100 Subject: [PATCH 013/727] bumped version --- CHANGES | 20 +++++++++++++++++++- piccolo/__init__.py | 2 +- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 9e5fd7a8c..c34c6ae36 100644 --- a/CHANGES +++ b/CHANGES @@ -1,9 +1,27 @@ Changes ======= -0.35.0 +0.36.0 ------ +Fixed an issue where ``like`` and ``ilike`` clauses required a wildcard. For +example: + +.. code-block:: python + await Manager.select().where(Manager.name.ilike('Guido%')).run() + +You can now omit wildcards if you like: + +.. code-block:: python + + await Manager.select().where(Manager.name.ilike('Guido')).run() + +Which would match on ``'guido'`` and ``'Guido'``, but not ``'Guidoxyz'``. + +Thanks to @wmshort for reporting this issue. + +0.35.0 +------ * Improved ``PrimaryKey`` deprecation warning (courtesy @tonybaloney). * Added ``piccolo schema generate`` which creates a Piccolo schema from an existing database. diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 953e0a7e5..50d934979 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.35.0" +__VERSION__ = "0.36.0" From 74d160998a4141595138192a8d56c251022a9c19 Mon Sep 17 00:00:00 2001 From: Amin Alaee Date: Thu, 26 Aug 2021 00:40:50 +0430 Subject: [PATCH 014/727] Random model builder (#169) * add random model builders * add `build` and `build_sync` methods * Use `run_sync` * Make `build` classmethods, use Piccolo for column checking and update docs * updated docs Made sure `await` is used where appropriate, and added link to the new tester app Co-authored-by: Daniel Townsend --- docs/src/index.rst | 1 + .../projects_and_apps/included_apps.rst | 2 + docs/src/piccolo/testing/index.rst | 73 ++++++++ piccolo/testing/__init__.py | 3 + piccolo/testing/model_builder.py | 156 ++++++++++++++++++ piccolo/testing/random_builder.py | 73 ++++++++ tests/testing/__init__.py | 0 tests/testing/test_model_builder.py | 113 +++++++++++++ tests/testing/test_random_builder.py | 54 ++++++ 9 files changed, 475 insertions(+) create mode 100644 docs/src/piccolo/testing/index.rst create mode 100644 piccolo/testing/__init__.py create mode 100644 piccolo/testing/model_builder.py create mode 100644 piccolo/testing/random_builder.py create mode 100644 tests/testing/__init__.py create mode 100644 tests/testing/test_model_builder.py create mode 100644 tests/testing/test_random_builder.py diff --git a/docs/src/index.rst b/docs/src/index.rst index b38222dd6..253ed6d90 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -14,6 +14,7 @@ Welcome to Piccolo's documentation! piccolo/migrations/index piccolo/authentication/index piccolo/asgi/index + piccolo/testing/index piccolo/features/index piccolo/playground/index piccolo/deployment/index diff --git a/docs/src/piccolo/projects_and_apps/included_apps.rst b/docs/src/piccolo/projects_and_apps/included_apps.rst index abb3aa9f1..2e9fb54ba 100644 --- a/docs/src/piccolo/projects_and_apps/included_apps.rst +++ b/docs/src/piccolo/projects_and_apps/included_apps.rst @@ -101,6 +101,8 @@ need to run raw SQL queries on your database. For it to work, the underlying command needs to be on the path (i.e. ``psql`` or ``sqlite3`` depending on which you're using). +.. _TesterApp: + tester ~~~~~~ diff --git a/docs/src/piccolo/testing/index.rst b/docs/src/piccolo/testing/index.rst new file mode 100644 index 000000000..f2d4bb8f0 --- /dev/null +++ b/docs/src/piccolo/testing/index.rst @@ -0,0 +1,73 @@ +Testing +======= + +Piccolo provides a few tools to make testing easier and decrease manual work. + +Model Builder +------------- + +When writing unit tests, it's usually required to have some data seeded into the database. +You can build and save the records manually or use ``ModelBuilder`` to generate random records for you. + +This way you can randomize the fields you don't care about and specify important fields explicitly and +reduce the amount of manual work required. +``ModelBuilder`` currently supports all Piccolo column types and features. + +Let's say we have the following schema: + +.. code-block:: python + + from piccolo.columns import ForeignKey, Varchar + + class Manager(Table): + name = Varchar(length=50) + + class Band(Table): + name = Varchar(length=50) + manager = ForeignKey(Manager, null=True) + +You can build a random ``Band`` which will also build and save a random ``Manager``: + +.. code-block:: python + + from piccolo.testing.model_builder import ModelBuilder + + band = await ModelBuilder.build(Band) # Band instance with random values persisted + +.. note:: ``ModelBuilder.build(Band)`` persists record into the database by default. + +You can also run it synchronously if you prefer: + +.. code-block:: python + + manager = ModelBuilder.build_sync(Manager) + + +To specify any attribute, pass the ``defaults`` dictionary to the ``build`` method: + +.. code-block:: python + + manager = ModelBuilder.build(Manager) + + # Using table columns + band = await ModelBuilder.build(Band, defaults={Band.name: "Guido", Band.manager: manager}) + + # Or using strings as keys + band = await ModelBuilder.build(Band, defaults={"name": "Guido", "manager": manager}) + +To build objects without persisting them into the database: + +.. code-block:: python + + band = await ModelBuilder.build(Band, persist=False) + +To build object with minimal attributes, leaving nullable fields empty: + +.. code-block:: python + + band = await ModelBuilder.build(Band, minimal=True) # Leaves manager empty + +Test runner +----------- + +See the :ref:`tester app`. diff --git a/piccolo/testing/__init__.py b/piccolo/testing/__init__.py new file mode 100644 index 000000000..3cce94e1f --- /dev/null +++ b/piccolo/testing/__init__.py @@ -0,0 +1,3 @@ +from piccolo.testing.model_builder import ModelBuilder + +__all__ = ["ModelBuilder"] diff --git a/piccolo/testing/model_builder.py b/piccolo/testing/model_builder.py new file mode 100644 index 000000000..6bede86c8 --- /dev/null +++ b/piccolo/testing/model_builder.py @@ -0,0 +1,156 @@ +import json +import typing as t +from datetime import date, datetime, time, timedelta +from decimal import Decimal +from uuid import UUID + +from piccolo.columns.base import Column +from piccolo.table import Table +from piccolo.testing.random_builder import RandomBuilder +from piccolo.utils.sync import run_sync + + +class ModelBuilder: + __DEFAULT_MAPPER: t.Dict[t.Type, t.Callable] = { + bool: RandomBuilder.next_bool, + bytes: RandomBuilder.next_bytes, + date: RandomBuilder.next_date, + datetime: RandomBuilder.next_datetime, + float: RandomBuilder.next_float, + int: RandomBuilder.next_int, + str: RandomBuilder.next_str, + time: RandomBuilder.next_time, + timedelta: RandomBuilder.next_timedelta, + UUID: RandomBuilder.next_uuid, + } + + @classmethod + async def build( + cls, + table_class: t.Type[Table], + defaults: t.Dict[t.Union[Column, str], t.Any] = None, + persist: bool = True, + minimal: bool = False, + ) -> Table: + """ + Build Table instance with random data and save async. + This can build relationships, supported data types and parameters. + + :param table_class: + Table class to randomize. + + Examples: + + manager = ModelBuilder.build(Manager) + manager = ModelBuilder.build(Manager, name='Guido') + manager = ModelBuilder(persist=False).build(Manager) + manager = ModelBuilder(minimal=True).build(Manager) + band = ModelBuilder.build(Band, manager=manager) + """ + return await cls._build( + table_class=table_class, + defaults=defaults, + persist=persist, + minimal=minimal, + ) + + @classmethod + def build_sync( + cls, + table_class: t.Type[Table], + defaults: t.Dict[t.Union[Column, str], t.Any] = None, + persist: bool = True, + minimal: bool = False, + ) -> Table: + """ + Build Table instance with random data and save sync. + This can build relationships, supported data types and parameters. + + :param table_class: + Table class to randomize. + + Examples: + + manager = ModelBuilder.build_sync(Manager) + manager = ModelBuilder.build_sync(Manager, name='Guido') + manager = ModelBuilder(persist=False).build_sync(Manager) + manager = ModelBuilder(minimal=True).build_sync(Manager) + band = ModelBuilder.build_sync(Band, manager=manager) + """ + return run_sync( + cls.build( + table_class=table_class, + defaults=defaults, + persist=persist, + minimal=minimal, + ) + ) + + @classmethod + async def _build( + cls, + table_class: t.Type[Table], + defaults: t.Dict[t.Union[Column, str], t.Any] = None, + minimal: bool = False, + persist: bool = True, + ) -> Table: + model = table_class() + defaults = {} if not defaults else defaults + + for column, value in defaults.items(): + if isinstance(column, str): + column = model._meta.get_column_by_name(column) + + setattr(model, column._meta.name, value) + + for column in model._meta.columns: + + if column._meta.null and minimal: + continue + + if column._meta.name in defaults: + continue # Column value exists + + if "references" in column._meta.params and persist: + reference_model = await cls._build( + column._meta.params["references"], + persist=True, + ) + random_value = getattr( + reference_model, + reference_model._meta.primary_key._meta.name, + ) + else: + random_value = cls._randomize_attribute(column) + + setattr(model, column._meta.name, random_value) + + if persist: + await model.save().run() + + return model + + @classmethod + def _randomize_attribute(cls, column: Column) -> t.Any: + """ + Generate a random value for a column and apply formattings. + + :param column: + Column class to randomize. + """ + if column.value_type == Decimal: + precision, scale = column._meta.params["digits"] + random_value = RandomBuilder.next_float( + maximum=10 ** (precision - scale), scale=scale + ) + elif column._meta.choices: + random_value = RandomBuilder.next_enum(column._meta.choices) + else: + random_value = cls.__DEFAULT_MAPPER[column.value_type]() + + if "length" in column._meta.params and isinstance(random_value, str): + return random_value[: column._meta.params["length"]] + elif column.column_type in ["JSON", "JSONB"]: + return json.dumps(random_value) + + return random_value diff --git a/piccolo/testing/random_builder.py b/piccolo/testing/random_builder.py new file mode 100644 index 000000000..dfc46a9f2 --- /dev/null +++ b/piccolo/testing/random_builder.py @@ -0,0 +1,73 @@ +import enum +import random +import string +import typing as t +import uuid +from datetime import date, datetime, time, timedelta + + +class RandomBuilder: + @classmethod + def next_bool(cls) -> bool: + return random.choice([True, False]) + + @classmethod + def next_bytes(cls, length=8) -> bytes: + return random.getrandbits(length * 8).to_bytes(length, "little") + + @classmethod + def next_date(cls) -> date: + return date( + year=random.randint(2000, 2050), + month=random.randint(1, 12), + day=random.randint(1, 28), + ) + + @classmethod + def next_datetime(cls) -> datetime: + return datetime( + year=random.randint(2000, 2050), + month=random.randint(1, 12), + day=random.randint(1, 28), + hour=random.randint(0, 23), + minute=random.randint(0, 59), + second=random.randint(0, 59), + ) + + @classmethod + def next_enum(cls, e: t.Type[enum.Enum]) -> t.Any: + return random.choice([item.value for item in e]) + + @classmethod + def next_float(cls, minimum=0, maximum=2147483647, scale=5) -> float: + return round(random.uniform(minimum, maximum), scale) + + @classmethod + def next_int(cls, minimum=0, maximum=2147483647) -> int: + return random.randint(minimum, maximum) + + @classmethod + def next_str(cls, length=16) -> str: + return "".join( + random.choice(string.ascii_letters) for _ in range(length) + ) + + @classmethod + def next_time(cls) -> time: + return time( + hour=random.randint(0, 23), + minute=random.randint(0, 59), + second=random.randint(0, 59), + ) + + @classmethod + def next_timedelta(cls) -> timedelta: + return timedelta( + days=random.randint(1, 7), + hours=random.randint(1, 23), + minutes=random.randint(0, 59), + ) + + @classmethod + def next_uuid(cls) -> uuid.UUID: + return uuid.uuid4() diff --git a/tests/testing/__init__.py b/tests/testing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/testing/test_model_builder.py b/tests/testing/test_model_builder.py new file mode 100644 index 000000000..83b797b40 --- /dev/null +++ b/tests/testing/test_model_builder.py @@ -0,0 +1,113 @@ +import asyncio +import unittest + +from piccolo.testing.model_builder import ModelBuilder + +from ..example_app.tables import ( + Band, + Manager, + Poster, + RecordingStudio, + Shirt, + Ticket, +) + + +class TestModelBuilder(unittest.TestCase): + @classmethod + def setUpClass(cls): + Manager.create_table().run_sync() + Band.create_table().run_sync() + Poster.create_table().run_sync() + RecordingStudio.create_table().run_sync() + Shirt.create_table().run_sync() + Ticket.create_table().run_sync() + + def test_model_builder_async(self): + async def build_model(model): + return await ModelBuilder.build(model) + + asyncio.run(build_model(Manager)) + asyncio.run(build_model(Ticket)) + asyncio.run(build_model(Poster)) + asyncio.run(build_model(RecordingStudio)) + + def test_model_builder_sync(self): + ModelBuilder.build_sync(Manager) + ModelBuilder.build_sync(Ticket) + ModelBuilder.build_sync(Poster) + ModelBuilder.build_sync(RecordingStudio) + + def test_model_builder_with_choices(self): + shirt = ModelBuilder.build_sync(Shirt) + queried_shirt = ( + Shirt.objects().where(Shirt.id == shirt.id).first().run_sync() + ) + + self.assertIn( + queried_shirt.size, + ["s", "l", "m"], + ) + + def test_model_builder_with_foreign_key(self): + ModelBuilder.build_sync(Band) + + def test_model_builder_with_invalid_column(self): + with self.assertRaises(ValueError): + ModelBuilder.build_sync(Band, defaults={"X": 1}) + + def test_model_builder_with_minimal(self): + band = ModelBuilder.build_sync(Band, minimal=True) + + self.assertEqual( + Band.exists().where(Band.id == band.id).run_sync(), + True, + ) + + def test_model_builder_with_no_persist(self): + band = ModelBuilder.build_sync(Band, persist=False) + + self.assertEqual( + Band.exists().where(Band.id == band.id).run_sync(), + False, + ) + + def test_model_builder_with_valid_column(self): + manager = ModelBuilder.build_sync( + Manager, defaults={Manager.name: "Guido"} + ) + + queried_manager = ( + Manager.objects() + .where(Manager.id == manager.id) + .first() + .run_sync() + ) + + self.assertEqual(queried_manager.name, "Guido") + + def test_model_builder_with_valid_column_string(self): + manager = ModelBuilder.build_sync(Manager, defaults={"name": "Guido"}) + + queried_manager = ( + Manager.objects() + .where(Manager.id == manager.id) + .first() + .run_sync() + ) + + self.assertEqual(queried_manager.name, "Guido") + + def test_model_builder_with_valid_foreign_key(self): + manager = ModelBuilder.build_sync(Manager) + + band = ModelBuilder.build_sync(Band, defaults={Band.manager: manager}) + + self.assertEqual(manager._meta.primary_key, band.manager) + + def test_model_builder_with_valid_foreign_key_string(self): + manager = ModelBuilder.build_sync(Manager) + + band = ModelBuilder.build_sync(Band, defaults={"manager": manager}) + + self.assertEqual(manager._meta.primary_key, band.manager) diff --git a/tests/testing/test_random_builder.py b/tests/testing/test_random_builder.py new file mode 100644 index 000000000..1f078cb9e --- /dev/null +++ b/tests/testing/test_random_builder.py @@ -0,0 +1,54 @@ +import unittest +from enum import Enum + +from piccolo.testing.random_builder import RandomBuilder + + +class TestRandomBuilder(unittest.TestCase): + def test_next_bool(self): + random_bool = RandomBuilder.next_bool() + self.assertIn(random_bool, [True, False]) + + def test_next_bytes(self): + random_bytes = RandomBuilder.next_bytes(length=100) + self.assertEqual(len(random_bytes), 100) + + def test_next_date(self): + random_date = RandomBuilder.next_date() + self.assertGreaterEqual(random_date.year, 2000) + self.assertLessEqual(random_date.year, 2050) + + def test_next_datetime(self): + random_datetime = RandomBuilder.next_datetime() + self.assertGreaterEqual(random_datetime.year, 2000) + self.assertLessEqual(random_datetime.year, 2050) + + def test_next_enum(self): + class Color(Enum): + RED = 1 + BLUE = 2 + + random_enum = RandomBuilder.next_enum(Color) + self.assertIsInstance(random_enum, int) + + def test_next_float(self): + random_float = RandomBuilder.next_float(maximum=1000) + self.assertLessEqual(random_float, 1000) + + def test_next_int(self): + random_int = RandomBuilder.next_int() + self.assertLessEqual(random_int, 2147483647) + + def test_next_str(self): + random_str = RandomBuilder.next_str(length=64) + self.assertLessEqual(len(random_str), 64) + + def test_next_time(self): + RandomBuilder.next_time() + + def test_next_timedelta(self): + random_timedelta = RandomBuilder.next_timedelta() + self.assertLessEqual(random_timedelta.days, 7) + + def test_next_uuid(self): + RandomBuilder.next_uuid() From b3fce67ca437c4fcc16b24c16c38e64b50a7dba5 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 25 Aug 2021 21:23:52 +0100 Subject: [PATCH 015/727] update ModelBuilder docstrings --- piccolo/testing/model_builder.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/piccolo/testing/model_builder.py b/piccolo/testing/model_builder.py index 6bede86c8..72fcc26a2 100644 --- a/piccolo/testing/model_builder.py +++ b/piccolo/testing/model_builder.py @@ -40,12 +40,12 @@ async def build( Table class to randomize. Examples: + manager = await ModelBuilder.build(Manager) + manager = await ModelBuilder.build(Manager, name='Guido') + manager = await ModelBuilder(persist=False).build(Manager) + manager = await ModelBuilder(minimal=True).build(Manager) + band = await ModelBuilder.build(Band, manager=manager) - manager = ModelBuilder.build(Manager) - manager = ModelBuilder.build(Manager, name='Guido') - manager = ModelBuilder(persist=False).build(Manager) - manager = ModelBuilder(minimal=True).build(Manager) - band = ModelBuilder.build(Band, manager=manager) """ return await cls._build( table_class=table_class, @@ -70,12 +70,12 @@ def build_sync( Table class to randomize. Examples: - manager = ModelBuilder.build_sync(Manager) manager = ModelBuilder.build_sync(Manager, name='Guido') manager = ModelBuilder(persist=False).build_sync(Manager) manager = ModelBuilder(minimal=True).build_sync(Manager) band = ModelBuilder.build_sync(Band, manager=manager) + """ return run_sync( cls.build( @@ -133,10 +133,11 @@ async def _build( @classmethod def _randomize_attribute(cls, column: Column) -> t.Any: """ - Generate a random value for a column and apply formattings. + Generate a random value for a column and apply formatting. :param column: Column class to randomize. + """ if column.value_type == Decimal: precision, scale = column._meta.params["digits"] From 7d99e8323246ad0233a6e706df876732981df1dc Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 25 Aug 2021 21:26:43 +0100 Subject: [PATCH 016/727] bumped version --- CHANGES | 5 +++++ piccolo/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index c34c6ae36..ff098d0a3 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,11 @@ Changes ======= +0.37.0 +------ +Added ``ModelBuilder``, which can be used to generate data for tests (courtesy +@aminalaee). + 0.36.0 ------ Fixed an issue where ``like`` and ``ilike`` clauses required a wildcard. For diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 50d934979..b55c2f36d 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.36.0" +__VERSION__ = "0.37.0" From db095797560f903b61d01dca2bb9658c19049063 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 25 Aug 2021 23:16:02 +0100 Subject: [PATCH 017/727] make get_or_create work for complex where clauses (#195) * make get_or_create work for complex where clauses * update get_or_create docs * fix LGTM warning, and add `_was_created` attribute --- docs/src/piccolo/query_types/objects.rst | 26 +++++++++ piccolo/columns/combination.py | 31 +++++++++- piccolo/query/methods/objects.py | 21 +++++-- tests/table/test_objects.py | 73 +++++++++++++++++++++++- 4 files changed, 143 insertions(+), 8 deletions(-) diff --git a/docs/src/piccolo/query_types/objects.rst b/docs/src/piccolo/query_types/objects.rst index d76837eb5..6cf15303b 100644 --- a/docs/src/piccolo/query_types/objects.rst +++ b/docs/src/piccolo/query_types/objects.rst @@ -113,6 +113,32 @@ or create a new one with the ``defaults`` arguments: Band.name == 'Pythonistas', defaults={'popularity': 100} ).run_sync() +You can find out if an existing row was found, or if a new row was created: + +.. code-block:: python + + band = Band.objects.get_or_create( + Band.name == 'Pythonistas' + ).run_sync() + band._was_created # True if it was created, otherwise False if it was already in the db + +Complex where clauses are supported, but only within reason. For example: + +.. code-block:: python + + # This works OK: + band = Band.objects().get_or_create( + (Band.name == 'Pythonistas') & (Band.popularity == 1000), + ).run_sync() + + # This is problematic, as it's unclear what the name should be if we + # need to create the row: + band = Band.objects().get_or_create( + (Band.name == 'Pythonistas') | (Band.name == 'Rustaceans'), + defaults={'popularity': 100} + ).run_sync() + + Query clauses ------------- diff --git a/piccolo/columns/combination.py b/piccolo/columns/combination.py index cca8296da..d254ba359 100644 --- a/piccolo/columns/combination.py +++ b/piccolo/columns/combination.py @@ -2,7 +2,7 @@ import typing as t -from piccolo.columns.operators.comparison import ComparisonOperator +from piccolo.columns.operators.comparison import ComparisonOperator, Equal from piccolo.custom_types import Combinable, Iterable from piccolo.querystring import QueryString from piccolo.utils.sql_values import convert_to_sql_value @@ -44,6 +44,35 @@ def __str__(self): class And(Combination): operator = "AND" + def get_column_values(self) -> t.Dict[Column, t.Any]: + """ + This is used by `get_or_create` to know which values to assign if + the row doesn't exist in the database. + + For example, if we have: + + (Band.name == 'Pythonistas') & (Band.popularity == 1000) + + We will return {Band.name: 'Pythonistas', Band.popularity: 1000}. + + If the operator is anything besides equals, we don't return it, for + example: + + (Band.name == 'Pythonistas') & (Band.popularity > 1000) + + Returns {Band.name: 'Pythonistas'} + + """ + output = {} + for combinable in (self.first, self.second): + if isinstance(combinable, Where): + if combinable.operator == Equal: + output[combinable.column] = combinable.value + elif isinstance(combinable, And): + output.update(combinable.get_column_values()) + + return output + class Or(Combination): operator = "OR" diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index cc38c57e8..261837fd8 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -3,6 +3,7 @@ import typing as t from dataclasses import dataclass +from piccolo.columns.combination import And, Where from piccolo.custom_types import Combinable from piccolo.engine.base import Batch from piccolo.query.base import Query @@ -32,14 +33,22 @@ class GetOrCreate: async def run(self): instance = await self.query.get(self.where).run() if instance: + instance._was_created = False return instance instance = self.query.table() - setattr( - instance, - self.where.column._meta.name, # type: ignore - self.where.value, # type: ignore - ) + + # If it's a complex `where`, there can be several column values to + # extract e.g. (Band.name == 'Pythonistas') & (Band.popularity == 1000) + if isinstance(self.where, Where): + setattr( + instance, + self.where.column._meta.name, # type: ignore + self.where.value, # type: ignore + ) + elif isinstance(self.where, And): + for column, value in self.where.get_column_values().items(): + setattr(instance, column._meta.name, value) for column, value in self.defaults.items(): if isinstance(column, str): @@ -48,6 +57,8 @@ async def run(self): await instance.save().run() + instance._was_created = True + return instance def __await__(self): diff --git a/tests/table/test_objects.py b/tests/table/test_objects.py index 055b1a9d3..facbf119a 100644 --- a/tests/table/test_objects.py +++ b/tests/table/test_objects.py @@ -68,6 +68,10 @@ def test_get(self): self.assertTrue(band.name == "Pythonistas") def test_get_or_create(self): + """ + Make sure `get_or_create` works for simple where clauses. + """ + # When the row doesn't exist in the db: Band.objects().get_or_create( Band.name == "Pink Floyd", defaults={"popularity": 100} ).run_sync() @@ -76,10 +80,11 @@ def test_get_or_create(self): Band.objects().where(Band.name == "Pink Floyd").first().run_sync() ) - self.assertTrue(isinstance(instance, Band)) + self.assertIsInstance(instance, Band) self.assertTrue(instance.name == "Pink Floyd") self.assertTrue(instance.popularity == 100) + # When the row already exists in the db: Band.objects().get_or_create( Band.name == "Pink Floyd", defaults={Band.popularity: 100} ).run_sync() @@ -88,6 +93,70 @@ def test_get_or_create(self): Band.objects().where(Band.name == "Pink Floyd").first().run_sync() ) - self.assertTrue(isinstance(instance, Band)) + self.assertIsInstance(instance, Band) self.assertTrue(instance.name == "Pink Floyd") self.assertTrue(instance.popularity == 100) + + def test_get_or_create_complex(self): + """ + Make sure `get_or_create` works with complex where clauses. + """ + self.insert_rows() + + # When the row already exists in the db: + instance = ( + Band.objects() + .get_or_create( + (Band.name == "Pythonistas") & (Band.popularity == 1000) + ) + .run_sync() + ) + self.assertIsInstance(instance, Band) + self.assertEqual(instance._was_created, False) + + # When the row doesn't exist in the db: + instance = ( + Band.objects() + .get_or_create( + (Band.name == "Pythonistas2") & (Band.popularity == 2000) + ) + .run_sync() + ) + self.assertIsInstance(instance, Band) + self.assertEqual(instance._was_created, True) + + def test_get_or_create_very_complex(self): + """ + Make sure `get_or_create` works with very complex where clauses. + """ + self.insert_rows() + + # When the row already exists in the db: + instance = ( + Band.objects() + .get_or_create( + (Band.name == "Pythonistas") + & (Band.popularity > 0) + & (Band.popularity < 5000) + ) + .run_sync() + ) + self.assertIsInstance(instance, Band) + self.assertEqual(instance._was_created, False) + + # When the row doesn't exist in the db: + instance = ( + Band.objects() + .get_or_create( + (Band.name == "Pythonistas2") + & (Band.popularity > 10) + & (Band.popularity < 5000) + ) + .run_sync() + ) + self.assertIsInstance(instance, Band) + self.assertEqual(instance._was_created, True) + + # The values in the > and < should be ignored, and the default should + # be used for the column. + self.assertEqual(instance.popularity, 0) From df10b499b0dac68e860281ccd88058b65011abe5 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 25 Aug 2021 23:21:24 +0100 Subject: [PATCH 018/727] bumped version --- CHANGES | 15 +++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index ff098d0a3..6b82c01d9 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,21 @@ Changes ======= +0.38.0 +------ +``get_or_create`` now supports more complex where clauses. For example: + +.. code-block:: python + + row = await Band.objects().get_or_create( + (Band.name == 'Pythonistas') & (Band.popularity == 1000) + ).run() + +And you can find out whether the row was created or not using +``row._was_created``. + +Thanks to @wmshort for reporting this issue. + 0.37.0 ------ Added ``ModelBuilder``, which can be used to generate data for tests (courtesy diff --git a/piccolo/__init__.py b/piccolo/__init__.py index b55c2f36d..44ed6bb81 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.37.0" +__VERSION__ = "0.38.0" From a3fc4e69329a5bce5d4d89307ad37a56a5b937d0 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 26 Aug 2021 07:00:39 +0100 Subject: [PATCH 019/727] make sure get_or_create works correctly with joins --- piccolo/query/methods/objects.py | 5 ++++- tests/table/test_objects.py | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index 261837fd8..426a4934c 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -48,7 +48,10 @@ async def run(self): ) elif isinstance(self.where, And): for column, value in self.where.get_column_values().items(): - setattr(instance, column._meta.name, value) + if len(column._meta.call_chain) == 0: + # Make sure we only set the value if the column belongs + # to this table. + setattr(instance, column._meta.name, value) for column, value in self.defaults.items(): if isinstance(column, str): diff --git a/tests/table/test_objects.py b/tests/table/test_objects.py index facbf119a..30893b528 100644 --- a/tests/table/test_objects.py +++ b/tests/table/test_objects.py @@ -160,3 +160,23 @@ def test_get_or_create_very_complex(self): # The values in the > and < should be ignored, and the default should # be used for the column. self.assertEqual(instance.popularity, 0) + + def test_get_or_create_with_joins(self): + """ + Make sure that that `get_or_create` creates rows correctly when using + joins. + """ + instance = ( + Band.objects() + .get_or_create( + (Band.name == "My new band") + & (Band.manager.name == "Excellent manager") + ) + .run_sync() + ) + self.assertIsInstance(instance, Band) + self.assertEqual(instance._was_created, True) + + # We want to make sure the band name isn't 'Excellent manager' by + # mistake. + self.assertEqual(Band.name, "My new band") From bf280b18411a6dc2caa681a84473f91629a69522 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 26 Aug 2021 07:26:40 +0100 Subject: [PATCH 020/727] make sure get_or_create works correctly with joins (#197) --- piccolo/query/methods/objects.py | 5 ++++- tests/table/test_objects.py | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index 261837fd8..426a4934c 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -48,7 +48,10 @@ async def run(self): ) elif isinstance(self.where, And): for column, value in self.where.get_column_values().items(): - setattr(instance, column._meta.name, value) + if len(column._meta.call_chain) == 0: + # Make sure we only set the value if the column belongs + # to this table. + setattr(instance, column._meta.name, value) for column, value in self.defaults.items(): if isinstance(column, str): diff --git a/tests/table/test_objects.py b/tests/table/test_objects.py index facbf119a..30893b528 100644 --- a/tests/table/test_objects.py +++ b/tests/table/test_objects.py @@ -160,3 +160,23 @@ def test_get_or_create_very_complex(self): # The values in the > and < should be ignored, and the default should # be used for the column. self.assertEqual(instance.popularity, 0) + + def test_get_or_create_with_joins(self): + """ + Make sure that that `get_or_create` creates rows correctly when using + joins. + """ + instance = ( + Band.objects() + .get_or_create( + (Band.name == "My new band") + & (Band.manager.name == "Excellent manager") + ) + .run_sync() + ) + self.assertIsInstance(instance, Band) + self.assertEqual(instance._was_created, True) + + # We want to make sure the band name isn't 'Excellent manager' by + # mistake. + self.assertEqual(Band.name, "My new band") From 58f3e111df2658fdf887d42d8c50fcba1ba6325a Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 26 Aug 2021 07:29:07 +0100 Subject: [PATCH 021/727] bumped version --- CHANGES | 18 ++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 6b82c01d9..e44bb95dd 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,24 @@ Changes ======= +0.38.1 +------ +Minor changes to ``get_or_create`` to make sure it handles joins correctly. + +.. code-block:: python + + instance = ( + Band.objects() + .get_or_create( + (Band.name == "My new band") + & (Band.manager.name == "Excellent manager") + ) + .run_sync() + ) + +In this situation, there are two columns called ``name`` - we need to make sure +the correct value is applied if the row doesn't exist. + 0.38.0 ------ ``get_or_create`` now supports more complex where clauses. For example: diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 44ed6bb81..640ba9f7b 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.38.0" +__VERSION__ = "0.38.1" From 95ccef110ffc604779e2664c0084ff1151952a2c Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 26 Aug 2021 08:06:28 +0100 Subject: [PATCH 022/727] add docs about directly awaiting queries --- .../getting_started/sync_and_async.rst | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/src/piccolo/getting_started/sync_and_async.rst b/docs/src/piccolo/getting_started/sync_and_async.rst index b063ed583..bc28ac14a 100644 --- a/docs/src/piccolo/getting_started/sync_and_async.rst +++ b/docs/src/piccolo/getting_started/sync_and_async.rst @@ -9,6 +9,8 @@ ORMs which support asyncio. However, you can use Piccolo in synchronous apps as well, whether that be a WSGI web app, or a data science script. +------------------------------------------------------------------------------- + Sync example ------------ @@ -24,6 +26,8 @@ Sync example if __name__ == '__main__': main() +------------------------------------------------------------------------------- + Async example ------------- @@ -40,6 +44,22 @@ Async example if __name__ == '__main__': asyncio.run(main()) +Direct await +~~~~~~~~~~~~ + +You can directly await a query if you prefer. For example: + +.. code-block:: python + + >>> await Band.select() + [{'id': 1, 'name': 'Pythonistas', 'manager': 1, 'popularity': 1000}, + {'id': 2, 'name': 'Rustaceans', 'manager': 2, 'popularity': 500}] + +By convention, we await the ``run`` method (``await Band.select().run()``), but +you can use this shorter form if you prefer. + +------------------------------------------------------------------------------- + Which to use? ------------- @@ -50,6 +70,8 @@ Using the async version is useful for web applications which require high throughput, based on `ASGI frameworks `_. Piccolo makes building an ASGI web app really simple - see :ref:`ASGICommand`. +------------------------------------------------------------------------------- + Explicit -------- From 6b8f3c39f06ab4bfe66a2c75f04fb6c1c501d612 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 26 Aug 2021 08:14:10 +0100 Subject: [PATCH 023/727] improve docs for PICCOLO_CONF - link to the new tester app --- docs/src/piccolo/engines/index.rst | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/src/piccolo/engines/index.rst b/docs/src/piccolo/engines/index.rst index c3b3352fa..6807596ce 100644 --- a/docs/src/piccolo/engines/index.rst +++ b/docs/src/piccolo/engines/index.rst @@ -75,7 +75,7 @@ In your terminal: export PICCOLO_CONF=piccolo_conf_test -Or at the entypoint for your app, before any other imports: +Or at the entypoint of your app, before any other imports: .. code-block:: python @@ -84,10 +84,11 @@ Or at the entypoint for your app, before any other imports: This is helpful during tests - you can specify a different configuration file -which contains the connection details for a test database. Similarly, -it's useful if you're deploying your code to different environments (e.g. -staging and production). Have two configuration files, and set the environment -variable accordingly. +which contains the connection details for a test database. + +.. hint:: Piccolo has a builtin command which will do this for you - + automatically setting ``PICCOLO_CONF`` for the duration of your tests. See + :ref:`TesterApp`. .. code-block:: python @@ -97,6 +98,11 @@ variable accordingly. DB = SQLiteEngine(path='my_test_db.sqlite') + +It's also useful if you're deploying your code to different environments (e.g. +staging and production). Have two configuration files, and set the environment +variable accordingly. + If the ``piccolo_conf.py`` file is located in a sub-module (rather than the root of your project) you can specify the path like this: From 7c2cf2b6419b4530b627eb5499459c8a2fa57f62 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 26 Aug 2021 08:20:13 +0100 Subject: [PATCH 024/727] tweak the migration docs * Clearer wording around migration descriptions * Add some missing backticks --- docs/src/piccolo/migrations/create.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/src/piccolo/migrations/create.rst b/docs/src/piccolo/migrations/create.rst index 47ae35c36..cd8924513 100644 --- a/docs/src/piccolo/migrations/create.rst +++ b/docs/src/piccolo/migrations/create.rst @@ -83,8 +83,8 @@ supports `auto migrations` which can save a great deal of time. Piccolo will work out which tables to add by comparing previous auto migrations, and your current tables. In order for this to work, you have to register -your app's tables with the `AppConfig` in the piccolo_app.py file at the root -of your app (see :ref:`PiccoloApps`). +your app's tables with the ``AppConfig`` in the ``piccolo_app.py`` file at the +root of your app (see :ref:`PiccoloApps`). Creating an auto migration: @@ -122,5 +122,5 @@ can specify it when creating the migration: piccolo migrations new my_app --auto --desc="Adding name column" -The Piccolo CLI will then use this description where appropriate when dealing -with migrations. +The Piccolo CLI will then use this description when listing migrations, to make +them easier to identify. From 556c9bfec6109d51932e7e849eeb80e9433be4de Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 26 Aug 2021 08:30:46 +0100 Subject: [PATCH 025/727] update ecosystem docs --- docs/src/piccolo/ecosystem/index.rst | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/src/piccolo/ecosystem/index.rst b/docs/src/piccolo/ecosystem/index.rst index d1bb9a85a..2d90bb1fc 100644 --- a/docs/src/piccolo/ecosystem/index.rst +++ b/docs/src/piccolo/ecosystem/index.rst @@ -7,8 +7,15 @@ Piccolo API ----------- Provides some handy utilities for creating an API around your Piccolo tables. -Examples include easy CRUD endpoints for ASGI apps, authentication and -rate limiting. `Read the docs `_. +Examples include: + + * Easily creating CRUD endpoints for ASGI apps, based on Piccolo tables. + * Automatically creating Pydantic models from your Piccolo tables. + * Great FastAPI integration. + * Authentication and rate limiting. + +`See the docs `_ for +more information. Piccolo Admin ------------- @@ -18,6 +25,9 @@ project on `Github `_. .. image:: https://raw.githubusercontent.com/piccolo-orm/piccolo_admin/master/docs/images/screenshot.png +It's a modern UI built with Vue JS, which supports powerful data filtering, and +CSV exports. It's the crown jewel in the Piccolo ecosystem! + Piccolo Examples ---------------- From ed23fa3490ce98ca4818ce54549a022d2847ce13 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 26 Aug 2021 08:39:26 +0100 Subject: [PATCH 026/727] update homepage docs --- docs/src/index.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/src/index.rst b/docs/src/index.rst index 253ed6d90..3a3e9b0c6 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -1,6 +1,9 @@ Welcome to Piccolo's documentation! =================================== +Piccolo is a modern, async query builder and ORM for Python, with lots of +batteries included. + .. toctree:: :maxdepth: 1 :caption: Contents: @@ -46,3 +49,6 @@ Give me an ASGI web app! .. code-block:: bash piccolo asgi new + +FastAPI, Starlette, and BlackSheep are currently supported, with more coming +soon. From 33eb6caf21c4ee6cf63ea2ed81e69cce883c9770 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 26 Aug 2021 13:26:29 +0100 Subject: [PATCH 027/727] remove problematic type hint (#198) --- piccolo/apps/tester/commands/run.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/piccolo/apps/tester/commands/run.py b/piccolo/apps/tester/commands/run.py index cd7afd1b3..98dc35536 100644 --- a/piccolo/apps/tester/commands/run.py +++ b/piccolo/apps/tester/commands/run.py @@ -2,8 +2,6 @@ import sys import typing as t -from _pytest.config import ExitCode - class set_env_var: def __init__(self, var_name: str, temp_value: str): @@ -37,9 +35,7 @@ def __exit__(self, *args): self.set_var(self.existing_value) -def run_pytest( - pytest_args: t.List[str], -) -> t.Union[int, ExitCode]: # pragma: no cover +def run_pytest(pytest_args: t.List[str]) -> int: # pragma: no cover try: import pytest except ImportError: From 2a37052cd1e55151dcb53b30b99a103b51778d9f Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 26 Aug 2021 13:27:13 +0100 Subject: [PATCH 028/727] use future annotations --- piccolo/apps/tester/commands/run.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/piccolo/apps/tester/commands/run.py b/piccolo/apps/tester/commands/run.py index 98dc35536..def7fd356 100644 --- a/piccolo/apps/tester/commands/run.py +++ b/piccolo/apps/tester/commands/run.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import os import sys import typing as t From b04f695605c3748a1138fc227dcf61a550b7a838 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 26 Aug 2021 13:27:50 +0100 Subject: [PATCH 029/727] bumped version --- CHANGES | 4 ++++ piccolo/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index e44bb95dd..949be7460 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,10 @@ Changes ======= +0.38.2 +------ +Removed problematic type hint which assumed pytest was installed. + 0.38.1 ------ Minor changes to ``get_or_create`` to make sure it handles joins correctly. diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 640ba9f7b..dc556cad7 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.38.1" +__VERSION__ = "0.38.2" From fe6767ae119acf9c9a54068e30e2b4c0e4115435 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 27 Aug 2021 14:35:36 +0100 Subject: [PATCH 030/727] add link to tutorial videos in docs --- docs/src/index.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/src/index.rst b/docs/src/index.rst index 3a3e9b0c6..2614ca6cb 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -52,3 +52,11 @@ Give me an ASGI web app! FastAPI, Starlette, and BlackSheep are currently supported, with more coming soon. + +------------------------------------------------------------------------------- + +Videos +------ + +Piccolo has some `tutorial videos on YouTube `_, +which are a great companion to the docs. From 0c0b3b8d7b6c2ea15e0dc6e767896109e9910646 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 28 Aug 2021 21:28:27 +0100 Subject: [PATCH 031/727] added `to_dict` (#201) * added `to_dict` * add test for `to_dict` * support aliases in `to_dict` * add docs for `to_dict` * fix typo in docstring * allow a subset of columns to be specified * update docs --- docs/src/piccolo/query_types/objects.rst | 23 +++++++++++++++ piccolo/table.py | 35 +++++++++++++++++++++++ tests/table/instance/test_to_dict.py | 36 ++++++++++++++++++++++++ 3 files changed, 94 insertions(+) create mode 100644 tests/table/instance/test_to_dict.py diff --git a/docs/src/piccolo/query_types/objects.rst b/docs/src/piccolo/query_types/objects.rst index 6cf15303b..d5430d6cb 100644 --- a/docs/src/piccolo/query_types/objects.rst +++ b/docs/src/piccolo/query_types/objects.rst @@ -138,6 +138,29 @@ Complex where clauses are supported, but only within reason. For example: defaults={'popularity': 100} ).run_sync() +to_dict +------- + +If you need to convert an object into a dictionary, you can do so using the +``to_dict`` method. + +.. code-block:: python + + band = Band.objects().first().run_sync() + + >>> band.to_dict() + {'id': 1, 'name': 'Pythonistas', 'manager': 1, 'popularity': 1000} + +If you only want a subset of the columns, or want to use aliases for some of +the columns: + +.. code-block:: python + + band = Band.objects().first().run_sync() + + >>> band.to_dict(Band.id, Band.name.as_alias('title')) + {'id': 1, 'title': 'Pythonistas'} + Query clauses ------------- diff --git a/piccolo/table.py b/piccolo/table.py index 5ebaf400e..eadb53330 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -407,6 +407,41 @@ def get_related(self, foreign_key: t.Union[ForeignKey, str]) -> Objects: .first() ) + def to_dict(self, *columns: Column) -> t.Dict[str, t.Any]: + """ + A convenience method which returns a dictionary, mapping column names + to values for this table instance. + + .. code-block:: + + instance = await Manager.objects().get( + Manager.name == 'Guido' + ).run() + + >>> instance.to_dict() + {'id': 1, 'name': 'Guido'} + + If the columns argument is provided, only those columns are included in + the output. It also works with column aliases. + + .. code-block:: + + >>> instance.to_dict(Manager.id, Manager.name.as_alias('title')) + {'id': 1, 'title': 'Guido'} + + """ + alias_names = { + column._meta.name: getattr(column, "alias", None) + for column in columns + } + + output = {} + for column in columns or self._meta.columns: + output[ + alias_names.get(column._meta.name) or column._meta.name + ] = getattr(self, column._meta.name) + return output + def __setitem__(self, key: str, value: t.Any): setattr(self, key, value) diff --git a/tests/table/instance/test_to_dict.py b/tests/table/instance/test_to_dict.py new file mode 100644 index 000000000..9a597c909 --- /dev/null +++ b/tests/table/instance/test_to_dict.py @@ -0,0 +1,36 @@ +from tests.base import DBTestCase +from tests.example_app.tables import Manager + + +class TestToDict(DBTestCase): + def test_to_dict(self): + """ + Make sure that `to_dict` works correctly. + """ + self.insert_row() + + instance = Manager.objects().first().run_sync() + dictionary = instance.to_dict() + self.assertEqual(dictionary, {"id": 1, "name": "Guido"}) + + def test_filter_rows(self): + """ + Make sure that `to_dict` works correctly with a subset of columns. + """ + self.insert_row() + + instance = Manager.objects().first().run_sync() + dictionary = instance.to_dict(Manager.name) + self.assertEqual(dictionary, {"name": "Guido"}) + + def test_to_dict_aliases(self): + """ + Make sure that `to_dict` works correctly with aliases. + """ + self.insert_row() + + instance = Manager.objects().first().run_sync() + dictionary = instance.to_dict( + Manager.id, Manager.name.as_alias("title") + ) + self.assertEqual(dictionary, {"id": 1, "title": "Guido"}) From ceb12d93a7b33048b19859d2724ac3ebef54770f Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 28 Aug 2021 21:35:48 +0100 Subject: [PATCH 032/727] bumped version --- CHANGES | 23 +++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 949be7460..e92d6268b 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,29 @@ Changes ======= +0.39.0 +------ +Added ``to_dict`` method to ``Table``. + +If you just use ``__dict__`` on a ``Table`` instance, you get some non-column +values. By using ``to_dict`` it's just the column values. Here's an example: + +.. code-block:: python + + class MyTable(Table): + name = Varchar() + + instance = MyTable.objects().first().run_sync() + + >>> instance.__dict__ + {'_exists_in_db': True, 'id': 1, 'name': 'foo'} + + >>> instance.to_dict() + {'id': 1, 'name': 'foo'} + +Thanks to @wmshort for the idea, and @aminalaee and @sinisaos for investigating +edge cases. + 0.38.2 ------ Removed problematic type hint which assumed pytest was installed. diff --git a/piccolo/__init__.py b/piccolo/__init__.py index dc556cad7..eb6142e41 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.38.2" +__VERSION__ = "0.39.0" From b6d6fe6b9f228f0e0f0e0f576e9e1606f35f27db Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 29 Aug 2021 23:21:13 +0100 Subject: [PATCH 033/727] add `nested` output option (#204) * add `nested` output option * fix typo in comment * fix a typo in docstring * updated docs --- docs/src/piccolo/query_clauses/output.rst | 10 +++++ docs/src/piccolo/query_types/select.rst | 49 +++++++++++++++-------- piccolo/query/methods/select.py | 19 ++++++++- piccolo/query/mixins.py | 6 +++ piccolo/utils/dictionary.py | 35 ++++++++++++++++ tests/table/test_output.py | 27 +++++++++++++ tests/utils/test_dictionary.py | 28 +++++++++++++ 7 files changed, 156 insertions(+), 18 deletions(-) create mode 100644 piccolo/utils/dictionary.py create mode 100644 tests/utils/test_dictionary.py diff --git a/docs/src/piccolo/query_clauses/output.rst b/docs/src/piccolo/query_clauses/output.rst index 73d7a484e..f18b37723 100644 --- a/docs/src/piccolo/query_clauses/output.rst +++ b/docs/src/piccolo/query_clauses/output.rst @@ -39,6 +39,16 @@ If you're just querying a single column from a database table, you can use >>> Band.select(Band.id).output(as_list=True).run_sync() [1, 2] +nested +~~~~~~ + +Output any data from related tables in nested dictionaries. + +.. code-block:: python + + >>> Band.select(Band.name, Band.manager.name).first().output(nested=True).run_sync() + {'name': 'Pythonistas', 'manager': {'name': 'Guido'}} + ------------------------------------------------------------------------------- Select and Objects queries diff --git a/docs/src/piccolo/query_types/select.rst b/docs/src/piccolo/query_types/select.rst index 16c24bbe5..3db3b2cfa 100644 --- a/docs/src/piccolo/query_types/select.rst +++ b/docs/src/piccolo/query_types/select.rst @@ -20,7 +20,7 @@ To get certain columns: >>> Band.select(Band.name).run_sync() [{'name': 'Rustaceans'}, {'name': 'Pythonistas'}] -Or making an alias to make it shorter: +Or use an alias to make it shorter: .. code-block:: python @@ -46,12 +46,11 @@ This is equivalent to ``SELECT name AS title FROM band`` in SQL. Joins ----- -One of the most powerful things about select is it's support for joins. +One of the most powerful things about ``select`` is it's support for joins. .. code-block:: python - >>> b = Band - >>> b.select(b.name, b.manager.name).run_sync() + >>> Band.select(Band.name, Band.manager.name).run_sync() [{'name': 'Pythonistas', 'manager.name': 'Guido'}, {'name': 'Rustaceans', 'manager.name': 'Graydon'}] @@ -59,11 +58,29 @@ The joins can go several layers deep. .. code-block:: python - c = Concert - c.select( - c.id, - c.band_1.manager.name - ).run_sync() + >>> Concert.select(Concert.id, Concert.band_1.manager.name).run_sync() + [{'id': 1, 'band_1.manager.name': 'Guido'}] + +If you want all of the columns from a related table, there's a useful shortcut +which saves you from typing them all out: + +.. code-block:: python + + >>> Band.select(Band.name, *Band.manager.all_columns()).run_sync() + [ + {'name': 'Pythonistas', 'manager.id': 1, 'manager.name': 'Guido'}, + {'name': 'Rustaceans', 'manager.id': 2, 'manager.name': 'Graydon'} + ] + +You can also get the response as nested dictionaries, which can be very useful: + +.. code-block:: python + + >>> Band.select(Band.name, *Band.manager.all_columns()).output(nested=True).run_sync() + [ + {'name': 'Pythonistas', 'manager': {'id': 1, 'name': 'Guido'}}, + {'name': 'Rustaceans', 'manager': {'id': 2, 'manager.name': 'Graydon'}} + ] String syntax ------------- @@ -183,29 +200,29 @@ By default all columns are returned from the queried table. .. code-block:: python - b = Band # Equivalent to SELECT * from band - b.select().run_sync() + Band.select().run_sync() To restrict the returned columns, either pass in the columns into the ``select`` method, or use the ``columns`` method. .. code-block:: python - b = Band # Equivalent to SELECT name from band - b.select().columns(b.name).run_sync() + Band.select(Band.name).run_sync() + + # Or alternatively: + Band.select().columns(Band.name).run_sync() The ``columns`` method is additive, meaning you can chain it to add additional columns. .. code-block:: python - b = Band - b.select().columns(b.name).columns(b.manager).run_sync() + Band.select().columns(Band.name).columns(Band.manager).run_sync() # Or just define it one go: - b.select().columns(b.name, b.manager).run_sync() + Band.select().columns(Band.name, Band.manager).run_sync() first diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 98fe620ce..92ad42cbb 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -19,6 +19,7 @@ WhereDelegate, ) from piccolo.querystring import QueryString +from piccolo.utils.dictionary import make_nested if t.TYPE_CHECKING: # pragma: no cover from piccolo.custom_types import Combinable @@ -203,13 +204,23 @@ def offset(self, number: int) -> Select: return self async def response_handler(self, response): + # If no columns were specified, it's a select *, so we know that + # no columns were selected from related tables. + was_select_star = len(self.columns_delegate.selected_columns) == 0 + if self.limit_delegate._first: if len(response) == 0: return None + + if self.output_delegate._output.nested and not was_select_star: + return make_nested(response[0]) else: return response[0] else: - return response + if self.output_delegate._output.nested and not was_select_star: + return [make_nested(i) for i in response] + else: + return response def order_by(self, *columns: Column, ascending=True) -> Select: _columns: t.List[Column] = [ @@ -225,9 +236,13 @@ def output( as_list: bool = False, as_json: bool = False, load_json: bool = False, + nested: bool = False, ) -> Select: self.output_delegate.output( - as_list=as_list, as_json=as_json, load_json=load_json + as_list=as_list, + as_json=as_json, + load_json=load_json, + nested=nested, ) return self diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index 838f52ff6..8b7a6309f 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -78,6 +78,7 @@ class Output: as_list: bool = False as_objects: bool = False load_json: bool = False + nested: bool = False def copy(self) -> Output: return self.__class__( @@ -85,6 +86,7 @@ def copy(self) -> Output: as_list=self.as_list, as_objects=self.as_objects, load_json=self.load_json, + nested=self.nested, ) @@ -195,6 +197,7 @@ def output( as_list: t.Optional[bool] = None, as_json: t.Optional[bool] = None, load_json: t.Optional[bool] = None, + nested: t.Optional[bool] = None, ): """ :param as_list: @@ -216,6 +219,9 @@ def output( if load_json is not None: self._output.load_json = bool(load_json) + if nested is not None: + self._output.nested = bool(nested) + def copy(self) -> OutputDelegate: _output = self._output.copy() if self._output is not None else None return self.__class__(_output=_output) diff --git a/piccolo/utils/dictionary.py b/piccolo/utils/dictionary.py new file mode 100644 index 000000000..fd3f93687 --- /dev/null +++ b/piccolo/utils/dictionary.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import typing as t + + +def make_nested(dictionary: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]: + """ + Rows are returned from the database as a flat dictionary, with keys such + as ``'manager.name'`` if the column belongs to a related table. + + This function puts any values from a related table into a sub dictionary. + + .. code-block:: + + response = Band.select(Band.name, Band.manager.name).run_sync() + >>> print(response) + [{'name': 'Pythonistas', 'band.name': 'Guido'}] + + >>> make_nested(response[0]) + {'name': 'Pythonistas', 'band': {'name': 'Guido'}} + + """ + output: t.Dict[str, t.Any] = {} + + for key, value in dictionary.items(): + path = key.split(".") + if len(path) == 1: + output[path[0]] = value + else: + dictionary = output.setdefault(path[0], {}) + for path_element in path[1:-1]: + dictionary = dictionary.setdefault(path_element, {}) + dictionary[path[-1]] = value + + return output diff --git a/tests/table/test_output.py b/tests/table/test_output.py index 2eeccaa2f..d19361d3d 100644 --- a/tests/table/test_output.py +++ b/tests/table/test_output.py @@ -59,3 +59,30 @@ def test_objects(self): self.assertEqual(results[0].facilities, json) self.assertEqual(results[0].facilities_b, json) + + +class TestOutputNested(DBTestCase): + def test_output_nested(self): + self.insert_row() + + response = ( + Band.select(Band.name, Band.manager.name) + .output(nested=True) + .run_sync() + ) + self.assertEqual( + response, [{"name": "Pythonistas", "manager": {"name": "Guido"}}] + ) + + def test_output_nested_with_first(self): + self.insert_row() + + response = ( + Band.select(Band.name, Band.manager.name) + .first() + .output(nested=True) + .run_sync() + ) + self.assertEqual( + response, {"name": "Pythonistas", "manager": {"name": "Guido"}} + ) diff --git a/tests/utils/test_dictionary.py b/tests/utils/test_dictionary.py new file mode 100644 index 000000000..c83e4d6c0 --- /dev/null +++ b/tests/utils/test_dictionary.py @@ -0,0 +1,28 @@ +from unittest import TestCase + +from piccolo.utils.dictionary import make_nested + + +class TestMakeNested(TestCase): + def test_nesting(self): + response = make_nested( + { + "id": 1, + "name": "Pythonistas", + "manager.id": 1, + "manager.name": "Guido", + "manager.car.colour": "green", + } + ) + self.assertEqual( + response, + { + "id": 1, + "name": "Pythonistas", + "manager": { + "id": 1, + "name": "Guido", + "car": {"colour": "green"}, + }, + }, + ) From 133e76dbb72eca39bdfffbbfc03af2054aee4db2 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 29 Aug 2021 23:29:23 +0100 Subject: [PATCH 034/727] bumped version --- CHANGES | 12 ++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index e92d6268b..d34c200cc 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,18 @@ Changes ======= +0.40.0 +------ +Added ``nested`` output option, which makes the response from a ``select`` +query use nested dictionaries: + +.. code-block:: python + + >>> await Band.select(Band.name, *Band.manager.all_columns()).output(nested=True).run() + [{'name': 'Pythonistas', 'manager': {'id': 1, 'name': 'Guido'}}] + +Thanks to @wmshort for the idea. + 0.39.0 ------ Added ``to_dict`` method to ``Table``. diff --git a/piccolo/__init__.py b/piccolo/__init__.py index eb6142e41..0015ceee8 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.39.0" +__VERSION__ = "0.40.0" From cb0dc394071a0f58cb7f9b967436ec5236533d1a Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 30 Aug 2021 11:06:24 +0100 Subject: [PATCH 035/727] loosen typing-extensions requirement --- requirements/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 56ca3c765..0838d37ea 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -3,4 +3,4 @@ colorama>=0.4.0 Jinja2>=2.11.0 targ>=0.3.3 inflection>=0.5.1 -typing-extensions==3.10.0.0 +typing-extensions>=3.10.0.0 From 0132ae12d454939d3a17ea66f75e17d79ef1316b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 30 Aug 2021 11:07:52 +0100 Subject: [PATCH 036/727] bumped version --- CHANGES | 5 +++++ piccolo/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index d34c200cc..0bca9549f 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,11 @@ Changes ======= +0.40.1 +------ +Loosen the ``typing-extensions`` requirement, as it was causing issues when +installing ``asyncpg``. + 0.40.0 ------ Added ``nested`` output option, which makes the response from a ``select`` diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 0015ceee8..0906044b0 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.40.0" +__VERSION__ = "0.40.1" From 7a937652c72d966228d72caa245fbe83f43f13c6 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 31 Aug 2021 13:48:27 +0100 Subject: [PATCH 037/727] `all_columns` fix (#210) * fixing bug with all_columns if called from too deep * ``ColumnsDelegate`` auto unpacks lists of columns This is useful if someone uses ``all_columns`` and forgets to unpack them. await Band.select(Band.name, Band.manager.all_columns()).run() The above now works - before this was required: await Band.select(Band.name, *Band.manager.all_columns()).run() * update docs, to mention that unpacking `all_columns` is now optional --- docs/src/piccolo/query_types/select.rst | 9 +++- piccolo/columns/base.py | 2 +- piccolo/columns/column_types.py | 32 +++++++------- piccolo/query/mixins.py | 18 +++++++- tests/columns/test_foreignkey.py | 51 ++++++++++++---------- tests/query/test_mixins.py | 28 ++++++++++++ tests/table/test_delete.py | 1 - tests/table/test_join.py | 58 ++++++++++++++++++++++--- tests/table/test_objects.py | 5 --- tests/table/test_select.py | 2 - 10 files changed, 147 insertions(+), 59 deletions(-) create mode 100644 tests/query/test_mixins.py diff --git a/docs/src/piccolo/query_types/select.rst b/docs/src/piccolo/query_types/select.rst index 3db3b2cfa..c23081eb7 100644 --- a/docs/src/piccolo/query_types/select.rst +++ b/docs/src/piccolo/query_types/select.rst @@ -61,8 +61,9 @@ The joins can go several layers deep. >>> Concert.select(Concert.id, Concert.band_1.manager.name).run_sync() [{'id': 1, 'band_1.manager.name': 'Guido'}] -If you want all of the columns from a related table, there's a useful shortcut -which saves you from typing them all out: +If you want all of the columns from a related table you can use +``all_columns``, which is a useful shortcut which saves you from typing them +all out: .. code-block:: python @@ -72,6 +73,10 @@ which saves you from typing them all out: {'name': 'Rustaceans', 'manager.id': 2, 'manager.name': 'Graydon'} ] + # In Piccolo > 0.41.0 you no longer need to explicitly unpack ``all_columns``. + # This is equivalent: + >>> Band.select(Band.name, Band.manager.all_columns()).run_sync() + You can also get the response as nested dictionaries, which can be very useful: .. code-block:: python diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 17d76eb20..f735bcd81 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -137,7 +137,7 @@ class ColumnMeta: _table: t.Optional[t.Type[Table]] = None # Used by Foreign Keys: - call_chain: t.List["ForeignKey"] = field(default_factory=lambda: []) + call_chain: t.List["ForeignKey"] = field(default_factory=list) table_alias: t.Optional[str] = None @property diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index bdcc424b3..8377d7be6 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -1220,6 +1220,22 @@ def copy(self) -> ForeignKey: column._foreign_key_meta = self._foreign_key_meta.copy() return column + def all_columns(self): + """ + Allow a user to access all of the columns on the related table. + + For example: + + Band.select(Band.name, *Band.manager.all_columns()).run_sync() + + """ + _fk_meta = object.__getattribute__(self, "_foreign_key_meta") + + return [ + getattr(self, column._meta.name) + for column in _fk_meta.resolved_references._meta.columns + ] + def set_proxy_columns(self): """ In order to allow a fluent interface, where tables can be traversed @@ -1233,22 +1249,6 @@ def set_proxy_columns(self): setattr(self, _column._meta.name, _column) _fk_meta.proxy_columns.append(_column) - def all_columns(): - """ - Allow a user to access all of the columns on the related table. - - For example: - - Band.select(Band.name, *Band.manager.all_columns()).run_sync() - - """ - return [ - getattr(self, column._meta.name) - for column in _fk_meta.resolved_references._meta.columns - ] - - setattr(self, "all_columns", all_columns) - def __getattribute__(self, name: str): """ Returns attributes unmodified unless they're Column instances, in which diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index 8b7a6309f..11673f4cd 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -237,8 +237,22 @@ class ColumnsDelegate: selected_columns: t.Sequence[Selectable] = field(default_factory=list) - def columns(self, *columns: Selectable): - combined = list(self.selected_columns) + list(columns) + def columns(self, *columns: t.Union[Selectable, t.List[Selectable]]): + """ + :param columns: + We accept ``Selectable`` and ``List[Selectable]`` here, in case + someone passes in a list by accident when using ``all_columns()``, + in which case we flatten the list. + + """ + _columns = [] + for column in columns: + if isinstance(column, list): + _columns.extend(column) + else: + _columns.append(column) + + combined = list(self.selected_columns) + _columns self.selected_columns = combined def remove_secret_columns(self): diff --git a/tests/columns/test_foreignkey.py b/tests/columns/test_foreignkey.py index 90a2c4d7e..f19ebf6b6 100644 --- a/tests/columns/test_foreignkey.py +++ b/tests/columns/test_foreignkey.py @@ -3,8 +3,7 @@ from piccolo.columns import Column, ForeignKey, LazyTableReference, Varchar from piccolo.table import Table -from tests.base import DBTestCase -from tests.example_app.tables import Band, Manager +from tests.example_app.tables import Band, Concert, Manager class Manager1(Table, tablename="manager"): @@ -172,32 +171,36 @@ def test_recursion_time(self): self.assertTrue(end - start < 1.0) -class TestAllColumns(DBTestCase): - def setUp(self): - Manager.create_table().run_sync() - manager = Manager(name="Guido") - manager.save().run_sync() - - Band.create_table().run_sync() - Band(manager=manager, name="Pythonistas").save().run_sync() - - def tearDown(self): - Band.alter().drop_table().run_sync() - Manager.alter().drop_table().run_sync() - +class TestAllColumns(TestCase): def test_all_columns(self): """ Make sure you can retrieve all columns from a related table, without explicitly specifying them. """ - result = Band.select(Band.name, *Band.manager.all_columns()).run_sync() + all_columns = Band.manager.all_columns() + self.assertEqual(all_columns, [Band.manager.id, Band.manager.name]) + + # Make sure the call chains are also correct. self.assertEqual( - result, - [ - { - "name": "Pythonistas", - "manager.id": 1, - "manager.name": "Guido", - } - ], + all_columns[0]._meta.call_chain, Band.manager.id._meta.call_chain + ) + self.assertEqual( + all_columns[1]._meta.call_chain, Band.manager.name._meta.call_chain + ) + + def test_all_columns_deep(self): + """ + Make sure ``all_columns`` works when the joins are several layers deep. + """ + all_columns = Concert.band_1.manager.all_columns() + self.assertEqual(all_columns, [Band.manager.id, Band.manager.name]) + + # Make sure the call chains are also correct. + self.assertEqual( + all_columns[0]._meta.call_chain, + Concert.band_1.manager.id._meta.call_chain, + ) + self.assertEqual( + all_columns[1]._meta.call_chain, + Concert.band_1.manager.name._meta.call_chain, ) diff --git a/tests/query/test_mixins.py b/tests/query/test_mixins.py new file mode 100644 index 000000000..f03dbefa4 --- /dev/null +++ b/tests/query/test_mixins.py @@ -0,0 +1,28 @@ +from unittest import TestCase + +from piccolo.query.mixins import ColumnsDelegate +from tests.example_app.tables import Band + + +class TestColumnsDelegate(TestCase): + def test_list_unpacking(self): + """ + The ``ColumnsDelegate`` should unpack a list of columns if passed in by + mistake, without the user unpacking them explicitly. + + .. code-block:: python + + # These two should both work the same: + await Band.select([Band.id, Band.name]).run() + await Band.select(Band.id, Band.name).run() + + """ + columns_delegate = ColumnsDelegate() + + columns_delegate.columns([Band.name]) + self.assertEqual(columns_delegate.selected_columns, [Band.name]) + + columns_delegate.columns([Band.id]) + self.assertEqual( + columns_delegate.selected_columns, [Band.name, Band.id] + ) diff --git a/tests/table/test_delete.py b/tests/table/test_delete.py index 2c5c6d35f..0fd8c24ba 100644 --- a/tests/table/test_delete.py +++ b/tests/table/test_delete.py @@ -11,7 +11,6 @@ def test_delete(self): Band.delete().where(Band.name == "CSharps").run_sync() response = Band.count().where(Band.name == "CSharps").run_sync() - print(f"response = {response}") self.assertEqual(response, 0) diff --git a/tests/table/test_join.py b/tests/table/test_join.py index e0d59e098..72afa9fb5 100644 --- a/tests/table/test_join.py +++ b/tests/table/test_join.py @@ -25,11 +25,6 @@ def setUp(self): for table in self.tables: table.create_table().run_sync() - def tearDown(self): - for table in reversed(self.tables): - table.alter().drop_table().run_sync() - - def test_join(self): manager_1 = Manager(name="Guido") manager_1.save().run_sync() @@ -42,7 +37,7 @@ def test_join(self): band_2 = Band(name="Rustaceans", manager=manager_2.id) band_2.save().run_sync() - venue = Venue(name="Grand Central") + venue = Venue(name="Grand Central", capacity=1000) venue.save().run_sync() save_query = Concert( @@ -50,6 +45,11 @@ def test_join(self): ).save() save_query.run_sync() + def tearDown(self): + for table in reversed(self.tables): + table.alter().drop_table().run_sync() + + def test_join(self): select_query = Concert.select( Concert.band_1.name, Concert.band_2.name, @@ -73,3 +73,49 @@ def test_join(self): select_query = Concert.select(Concert.band_1.manager.name) response = select_query.run_sync() self.assertEqual(response, [{"band_1.manager.name": "Guido"}]) + + def test_all_columns(self): + """ + Make sure you can retrieve all columns from a related table, without + explicitly specifying them. + """ + result = ( + Band.select(Band.name, *Band.manager.all_columns()) + .first() + .run_sync() + ) + self.assertEqual( + result, + { + "name": "Pythonistas", + "manager.id": 1, + "manager.name": "Guido", + }, + ) + + def test_all_columns_deep(self): + """ + Make sure that ``all_columns`` can be used several layers deep. + """ + result = ( + Concert.select( + *Concert.venue.all_columns(), + *Concert.band_1.manager.all_columns(), + *Concert.band_2.manager.all_columns(), + ) + .first() + .run_sync() + ) + + self.assertEqual( + result, + { + "venue.id": 1, + "venue.name": "Grand Central", + "venue.capacity": 1000, + "band_1.manager.id": 1, + "band_1.manager.name": "Guido", + "band_2.manager.id": 2, + "band_2.manager.name": "Graydon", + }, + ) diff --git a/tests/table/test_objects.py b/tests/table/test_objects.py index 30893b528..7766f989f 100644 --- a/tests/table/test_objects.py +++ b/tests/table/test_objects.py @@ -32,9 +32,6 @@ def test_offset_postgres(self): """ self.insert_rows() response = Band.objects().order_by(Band.name).offset(1).run_sync() - - print(f"response = {response}") - self.assertEqual( [i.name for i in response], ["Pythonistas", "Rustaceans"] ) @@ -54,8 +51,6 @@ def test_offset_sqlite(self): response = query.run_sync() - print(f"response = {response}") - self.assertEqual( [i.name for i in response], ["Pythonistas", "Rustaceans"] ) diff --git a/tests/table/test_select.py b/tests/table/test_select.py index 9ceba55bd..958997f50 100644 --- a/tests/table/test_select.py +++ b/tests/table/test_select.py @@ -13,7 +13,6 @@ def test_query_all_columns(self): self.insert_row() response = Band.select().run_sync() - print(f"response = {response}") self.assertDictEqual( response[0], @@ -24,7 +23,6 @@ def test_query_some_columns(self): self.insert_row() response = Band.select(Band.name).run_sync() - print(f"response = {response}") self.assertDictEqual(response[0], {"name": "Pythonistas"}) From ff831cfae23e1f4f048dc4612c20b4671be6c81b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 31 Aug 2021 13:53:40 +0100 Subject: [PATCH 038/727] bumped version --- CHANGES | 25 +++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 0bca9549f..7dc2215dc 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,31 @@ Changes ======= +0.41.0 +------ +Fixed a bug where if ``all_columns`` was used two or more levels deep, it would +fail. Thanks to @wmshort for reporting this issue. + +Here's an example: + +.. code-block:: python + + Concert.select( + Concert.venue.name, + *Concert.band_1.manager.all_columns() + ).run_sync() + +Also, the ``ColumnsDelegate`` has now been tweaked, so unpacking of +``all_columns`` is optional. + +.. code-block:: python + + # This now works the same as the code above (we have omitted the *) + Concert.select( + Concert.venue.name, + Concert.band_1.manager.all_columns() + ).run_sync() + 0.40.1 ------ Loosen the ``typing-extensions`` requirement, as it was causing issues when diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 0906044b0..44739926a 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.40.1" +__VERSION__ = "0.41.0" From cbd30f7aeea896ccfda7530a3bbb249778e12895 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 31 Aug 2021 20:24:03 +0100 Subject: [PATCH 039/727] fix table creation ordering in migrations (#211) * fix table creation ordering * put a recursion limit on `_compare_tables` --- .../apps/migrations/auto/migration_manager.py | 82 +++++++++++++++---- .../migrations/auto/test_migration_manager.py | 25 +++++- .../piccolo_migrations/2020-12-17T18-44-30.py | 2 +- 3 files changed, 89 insertions(+), 20 deletions(-) diff --git a/piccolo/apps/migrations/auto/migration_manager.py b/piccolo/apps/migrations/auto/migration_manager.py index 704c81d76..0f6c01478 100644 --- a/piccolo/apps/migrations/auto/migration_manager.py +++ b/piccolo/apps/migrations/auto/migration_manager.py @@ -3,6 +3,7 @@ import inspect import typing as t from dataclasses import dataclass, field +from functools import cmp_to_key from piccolo.apps.migrations.auto.diffable_table import DiffableTable from piccolo.apps.migrations.auto.operations import ( @@ -117,6 +118,49 @@ def table_class_names(self) -> t.List[str]: return list(set([i.table_class_name for i in self.alter_columns])) +def _compare_tables( + table_a: t.Type[Table], + table_b: t.Type[Table], + iterations: int = 0, + max_iterations=5, +) -> int: + """ + A comparison function, for sorting Table classes, based on their foreign + keys. + + :param iterations: + As this function is called recursively, we use this to limit the depth, + to prevent an infinite loop. + + """ + if iterations >= max_iterations: + return 0 + + for fk_column in table_a._meta.foreign_key_columns: + references = fk_column._foreign_key_meta.resolved_references + if references._meta.tablename == table_b._meta.tablename: + return 1 + else: + for _fk_column in references._meta.foreign_key_columns: + _references = _fk_column._foreign_key_meta.resolved_references + if _compare_tables( + _references, table_b, iterations=iterations + 1 + ): + return 1 + + return -1 + + +def sort_table_classes( + table_classes: t.List[t.Type[Table]], +) -> t.List[t.Type[Table]]: + """ + Sort the table classes based on their foreign keys, so they can be created + in the correct order. + """ + return sorted(table_classes, key=cmp_to_key(_compare_tables)) + + @dataclass class MigrationManager: """ @@ -563,25 +607,29 @@ async def _run_rename_columns(self, backwards=False): ).run() async def _run_add_tables(self, backwards=False): + table_classes: t.List[t.Type[Table]] = [] + for add_table in self.add_tables: + add_columns: t.List[ + AddColumnClass + ] = self.add_columns.for_table_class_name(add_table.class_name) + _Table: t.Type[Table] = create_table_class( + class_name=add_table.class_name, + class_kwargs={"tablename": add_table.tablename}, + class_members={ + add_column.column._meta.name: add_column.column + for add_column in add_columns + }, + ) + table_classes.append(_Table) + + # Sort by foreign key, so they're created in the right order. + sorted_table_classes = sort_table_classes(table_classes) + if backwards: - for add_table in self.add_tables: - await add_table.to_table_class().alter().drop_table( - cascade=True - ).run() + for _Table in reversed(sorted_table_classes): + await _Table.alter().drop_table(cascade=True).run() else: - for add_table in self.add_tables: - add_columns: t.List[ - AddColumnClass - ] = self.add_columns.for_table_class_name(add_table.class_name) - _Table: t.Type[Table] = create_table_class( - class_name=add_table.class_name, - class_kwargs={"tablename": add_table.tablename}, - class_members={ - add_column.column._meta.name: add_column.column - for add_column in add_columns - }, - ) - + for _Table in sorted_table_classes: await _Table.create_table().run() async def _run_add_columns(self, backwards=False): diff --git a/tests/apps/migrations/auto/test_migration_manager.py b/tests/apps/migrations/auto/test_migration_manager.py index 965473613..0eabe7651 100644 --- a/tests/apps/migrations/auto/test_migration_manager.py +++ b/tests/apps/migrations/auto/test_migration_manager.py @@ -1,7 +1,11 @@ import asyncio +from unittest import TestCase from unittest.mock import MagicMock, patch -from piccolo.apps.migrations.auto import MigrationManager +from piccolo.apps.migrations.auto.migration_manager import ( + MigrationManager, + sort_table_classes, +) from piccolo.apps.migrations.commands.base import BaseMigrationManager from piccolo.columns import Text, Varchar from piccolo.columns.base import OnDelete, OnUpdate @@ -9,11 +13,28 @@ from piccolo.conf.apps import AppConfig from piccolo.utils.lazy_loader import LazyLoader from tests.base import DBTestCase, postgres_only, set_mock_return_value -from tests.example_app.tables import Manager +from tests.example_app.tables import Band, Concert, Manager, Venue asyncpg = LazyLoader("asyncpg", globals(), "asyncpg") +class TestSortTableClasses(TestCase): + def test_sort_table_classes(self): + self.assertEqual(sort_table_classes([Manager, Band]), [Manager, Band]) + self.assertEqual(sort_table_classes([Band, Manager]), [Manager, Band]) + + sorted_tables = sort_table_classes([Manager, Venue, Concert, Band]) + self.assertTrue( + sorted_tables.index(Manager) < sorted_tables.index(Band) + ) + self.assertTrue( + sorted_tables.index(Venue) < sorted_tables.index(Concert) + ) + self.assertTrue( + sorted_tables.index(Band) < sorted_tables.index(Concert) + ) + + class TestMigrationManager(DBTestCase): @postgres_only def test_rename_column(self): diff --git a/tests/example_app/piccolo_migrations/2020-12-17T18-44-30.py b/tests/example_app/piccolo_migrations/2020-12-17T18-44-30.py index 94943796a..8d037f7e6 100644 --- a/tests/example_app/piccolo_migrations/2020-12-17T18-44-30.py +++ b/tests/example_app/piccolo_migrations/2020-12-17T18-44-30.py @@ -14,8 +14,8 @@ class Manager(Table, tablename="manager"): async def forwards(): manager = MigrationManager(migration_id=ID, app_name="example_app") - manager.add_table("Manager", tablename="manager") manager.add_table("Band", tablename="band") + manager.add_table("Manager", tablename="manager") manager.add_column( table_class_name="Band", From 240ef5d64f501585c53fa33643b6a1bd8165e1e0 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 31 Aug 2021 20:26:25 +0100 Subject: [PATCH 040/727] bumped version --- CHANGES | 5 +++++ piccolo/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 7dc2215dc..aeda59d8e 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,11 @@ Changes ======= +0.41.1 +------ +Fix a regression where if multiple tables are created in a single migration +file, it could potentially fail by applying them in the wrong order. + 0.41.0 ------ Fixed a bug where if ``all_columns`` was used two or more levels deep, it would diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 44739926a..aa70a7ec0 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.41.0" +__VERSION__ = "0.41.1" From f9c7acde8f509aff70c871554fc2ddba15bb823b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 1 Sep 2021 07:35:42 +0100 Subject: [PATCH 041/727] allow `all_columns` at the root, and `exclude` (#212) * allow all_columns at the root, and exclude * add an extra test for make_nested --- docs/src/piccolo/query_types/select.rst | 77 ++++++++++++++++++-- piccolo/columns/column_types.py | 28 ++++++- piccolo/table.py | 33 +++++++++ piccolo/utils/dictionary.py | 17 ++++- tests/columns/test_foreignkey.py | 13 ++++ tests/table/test_all_columns.py | 23 ++++++ tests/table/test_join.py | 97 +++++++++++++++++++++++-- tests/utils/test_dictionary.py | 28 +++++++ 8 files changed, 299 insertions(+), 17 deletions(-) create mode 100644 tests/table/test_all_columns.py diff --git a/docs/src/piccolo/query_types/select.rst b/docs/src/piccolo/query_types/select.rst index c23081eb7..6989bb990 100644 --- a/docs/src/piccolo/query_types/select.rst +++ b/docs/src/piccolo/query_types/select.rst @@ -31,6 +31,8 @@ Or use an alias to make it shorter: .. hint:: All of these examples also work with async by using .run() inside coroutines - see :ref:`SyncAndAsync`. +------------------------------------------------------------------------------- + as_alias -------- @@ -43,6 +45,8 @@ By using ``as_alias``, the name of the row can be overriden in the response. This is equivalent to ``SELECT name AS title FROM band`` in SQL. +------------------------------------------------------------------------------- + Joins ----- @@ -51,7 +55,10 @@ One of the most powerful things about ``select`` is it's support for joins. .. code-block:: python >>> Band.select(Band.name, Band.manager.name).run_sync() - [{'name': 'Pythonistas', 'manager.name': 'Guido'}, {'name': 'Rustaceans', 'manager.name': 'Graydon'}] + [ + {'name': 'Pythonistas', 'manager.name': 'Guido'}, + {'name': 'Rustaceans', 'manager.name': 'Graydon'} + ] The joins can go several layers deep. @@ -61,36 +68,90 @@ The joins can go several layers deep. >>> Concert.select(Concert.id, Concert.band_1.manager.name).run_sync() [{'id': 1, 'band_1.manager.name': 'Guido'}] +all_columns +~~~~~~~~~~~ + If you want all of the columns from a related table you can use ``all_columns``, which is a useful shortcut which saves you from typing them all out: .. code-block:: python - >>> Band.select(Band.name, *Band.manager.all_columns()).run_sync() + >>> Band.select(Band.name, Band.manager.all_columns()).run_sync() [ {'name': 'Pythonistas', 'manager.id': 1, 'manager.name': 'Guido'}, {'name': 'Rustaceans', 'manager.id': 2, 'manager.name': 'Graydon'} ] - # In Piccolo > 0.41.0 you no longer need to explicitly unpack ``all_columns``. - # This is equivalent: - >>> Band.select(Band.name, Band.manager.all_columns()).run_sync() + +In Piccolo < 0.41.0 you had to explicitly unpack ``all_columns``. This is +equivalent to the code above: + +.. code-block:: python + + >>> Band.select(Band.name, *Band.manager.all_columns()).run_sync() + + +You can exclude some columns if you like: + +.. code-block:: python + + >>> Band.select( + >>> Band.name, + >>> Band.manager.all_columns(exclude=[Band.manager.id) + >>> ).run_sync() + [ + {'name': 'Pythonistas', 'manager.name': 'Guido'}, + {'name': 'Rustaceans', 'manager.name': 'Graydon'} + ] + + +Strings are supported too if you prefer: + +.. code-block:: python + + >>> Band.select( + >>> Band.name, + >>> Band.manager.all_columns(exclude=['id']) + >>> ).run_sync() + [ + {'name': 'Pythonistas', 'manager.name': 'Guido'}, + {'name': 'Rustaceans', 'manager.name': 'Graydon'} + ] + +You can also use ``all_columns`` on the root table, which saves you time if +you have lots of columns. It works identically to related tables: + +.. code-block:: python + + >>> Band.select( + >>> Band.all_columns(exclude=[Band.id]), + >>> Band.manager.all_columns(exclude=[Band.manager.id]) + >>> ).run_sync() + [ + {'name': 'Pythonistas', 'popularity': 1000, 'manager.name': 'Guido'}, + {'name': 'Rustaceans', 'popularity': 500, 'manager.name': 'Graydon'} + ] + +Nested +~~~~~~ You can also get the response as nested dictionaries, which can be very useful: .. code-block:: python - >>> Band.select(Band.name, *Band.manager.all_columns()).output(nested=True).run_sync() + >>> Band.select(Band.name, Band.manager.all_columns()).output(nested=True).run_sync() [ {'name': 'Pythonistas', 'manager': {'id': 1, 'name': 'Guido'}}, {'name': 'Rustaceans', 'manager': {'id': 2, 'manager.name': 'Graydon'}} ] +------------------------------------------------------------------------------- + String syntax ------------- -Alternatively, you can specify the column names using a string. The +You can specify the column names using a string if you prefer. The disadvantage is you won't have tab completion, but sometimes it's more convenient. @@ -101,6 +162,7 @@ convenient. # For joins: Band.select('manager.name').run_sync() +------------------------------------------------------------------------------- Aggregate functions ------------------- @@ -189,6 +251,7 @@ And can use aliases for aggregate functions like this: >>> response["popularity_avg"] 750.0 +------------------------------------------------------------------------------- Query clauses ------------- diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 8377d7be6..3de7fd21e 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -1220,20 +1220,44 @@ def copy(self) -> ForeignKey: column._foreign_key_meta = self._foreign_key_meta.copy() return column - def all_columns(self): + def all_columns( + self, exclude: t.List[t.Union[Column, str]] = [] + ) -> t.List[Column]: """ Allow a user to access all of the columns on the related table. For example: - Band.select(Band.name, *Band.manager.all_columns()).run_sync() + .. code-block:: python + + Band.select(Band.name, Band.manager.all_columns()).run_sync() + + To exclude certain columns: + + .. code-block:: python + + Band.select( + Band.name, + Band.manager.all_columns( + exclude=[Band.manager.id] + ) + ).run_sync() + + :param exclude: + Columns to exclude - can be the name of a column, or a column + instance. For example ``['id']`` or ``[Band.manager.id]``. """ _fk_meta = object.__getattribute__(self, "_foreign_key_meta") + excluded_column_names = [ + i._meta.name if isinstance(i, Column) else i for i in exclude + ] + return [ getattr(self, column._meta.name) for column in _fk_meta.resolved_references._meta.columns + if column._meta.name not in excluded_column_names ] def set_proxy_columns(self): diff --git a/piccolo/table.py b/piccolo/table.py index eadb53330..a000125a2 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -517,6 +517,39 @@ def __repr__(self) -> str: ########################################################################### # Classmethods + @classmethod + def all_columns( + cls, exclude: t.List[t.Union[str, Column]] = [] + ) -> t.List[Column]: + """ + Just as we can use ``all_columns`` to retrieve all of the columns from + a related table, we can also use it at the root of our query to get + all of the columns for the root table. For example: + + .. code-block:: python + + await Band.select( + Band.all_columns(), + Band.manager.all_columns() + ).run() + + This is mostly useful when the table has a lot of columns, and typing + them out by hand would be tedious. + + :param exclude: + You can request all columns, except these. + + """ + excluded_column_names = [ + i._meta.name if isinstance(i, Column) else i for i in exclude + ] + + return [ + i + for i in cls._meta.columns + if i._meta.name not in excluded_column_names + ] + @classmethod def ref(cls, column_name: str) -> Column: """ diff --git a/piccolo/utils/dictionary.py b/piccolo/utils/dictionary.py index fd3f93687..cd10dc8ce 100644 --- a/piccolo/utils/dictionary.py +++ b/piccolo/utils/dictionary.py @@ -27,7 +27,22 @@ def make_nested(dictionary: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]: if len(path) == 1: output[path[0]] = value else: - dictionary = output.setdefault(path[0], {}) + # Force the root element to be an empty dictionary, if it's some + # other value (most likely an integer). This is because there are + # situations where a query can have `band` and `band.id`. + # For example: + # await Band.select( + # Band.all_columns(), + # Band.manager.all_columns() + # ).run() + # In this situation nesting takes precendence. + root = output.get(path[0], None) + if isinstance(root, dict): + dictionary = root + else: + dictionary = {} + output[path[0]] = dictionary + for path_element in path[1:-1]: dictionary = dictionary.setdefault(path_element, {}) dictionary[path[-1]] = value diff --git a/tests/columns/test_foreignkey.py b/tests/columns/test_foreignkey.py index f19ebf6b6..21efc491f 100644 --- a/tests/columns/test_foreignkey.py +++ b/tests/columns/test_foreignkey.py @@ -204,3 +204,16 @@ def test_all_columns_deep(self): all_columns[1]._meta.call_chain, Concert.band_1.manager.name._meta.call_chain, ) + + def test_all_columns_exclude(self): + """ + Make sure you can exclude some columns. + """ + self.assertEqual( + Band.manager.all_columns(exclude=["id"]), [Band.manager.name] + ) + + self.assertEqual( + Band.manager.all_columns(exclude=[Band.manager.id]), + [Band.manager.name], + ) diff --git a/tests/table/test_all_columns.py b/tests/table/test_all_columns.py new file mode 100644 index 000000000..1ff7d1283 --- /dev/null +++ b/tests/table/test_all_columns.py @@ -0,0 +1,23 @@ +from unittest import TestCase + +from tests.example_app.tables import Band + + +class TestAllColumns(TestCase): + def test_all_columns(self): + self.assertEqual( + Band.all_columns(), + [Band.id, Band.name, Band.manager, Band.popularity], + ) + self.assertEqual(Band.all_columns(), Band._meta.columns) + + def test_all_columns_excluding(self): + self.assertEqual( + Band.all_columns(exclude=[Band.id]), + [Band.name, Band.manager, Band.popularity], + ) + + self.assertEqual( + Band.all_columns(exclude=["id"]), + [Band.name, Band.manager, Band.popularity], + ) diff --git a/tests/table/test_join.py b/tests/table/test_join.py index 72afa9fb5..f325261fd 100644 --- a/tests/table/test_join.py +++ b/tests/table/test_join.py @@ -28,7 +28,9 @@ def setUp(self): manager_1 = Manager(name="Guido") manager_1.save().run_sync() - band_1 = Band(name="Pythonistas", manager=manager_1.id) + band_1 = Band( + name="Pythonistas", manager=manager_1.id, popularity=1000 + ) band_1.save().run_sync() manager_2 = Manager(name="Graydon") @@ -80,11 +82,11 @@ def test_all_columns(self): explicitly specifying them. """ result = ( - Band.select(Band.name, *Band.manager.all_columns()) + Band.select(Band.name, Band.manager.all_columns()) .first() .run_sync() ) - self.assertEqual( + self.assertDictEqual( result, { "name": "Pythonistas", @@ -99,15 +101,15 @@ def test_all_columns_deep(self): """ result = ( Concert.select( - *Concert.venue.all_columns(), - *Concert.band_1.manager.all_columns(), - *Concert.band_2.manager.all_columns(), + Concert.venue.all_columns(), + Concert.band_1.manager.all_columns(), + Concert.band_2.manager.all_columns(), ) .first() .run_sync() ) - self.assertEqual( + self.assertDictEqual( result, { "venue.id": 1, @@ -119,3 +121,84 @@ def test_all_columns_deep(self): "band_2.manager.name": "Graydon", }, ) + + def test_all_columns_root(self): + """ + Make sure that using ``all_columns`` at the root doesn't interfere + with using it for referenced tables. + """ + result = ( + Band.select( + Band.all_columns(), + Band.manager.all_columns(), + ) + .first() + .run_sync() + ) + self.assertDictEqual( + result, + { + "id": 1, + "name": "Pythonistas", + "manager": 1, + "popularity": 1000, + "manager.id": 1, + "manager.name": "Guido", + }, + ) + + def test_all_columns_root_nested(self): + """ + Make sure that using ``all_columns`` at the root doesn't interfere + with using it for referenced tables. + """ + result = ( + Band.select(Band.all_columns(), Band.manager.all_columns()) + .output(nested=True) + .first() + .run_sync() + ) + + self.assertDictEqual( + result, + { + "id": 1, + "name": "Pythonistas", + "manager": {"id": 1, "name": "Guido"}, + "popularity": 1000, + }, + ) + + def test_all_columns_exclude(self): + """ + Make sure we can get all columns, except the ones we specify. + """ + result = ( + Band.select( + Band.all_columns(exclude=[Band.id]), + Band.manager.all_columns(exclude=[Band.manager.id]), + ) + .output(nested=True) + .first() + .run_sync() + ) + + result_str_args = ( + Band.select( + Band.all_columns(exclude=["id"]), + Band.manager.all_columns(exclude=["id"]), + ) + .output(nested=True) + .first() + .run_sync() + ) + + for data in (result, result_str_args): + self.assertDictEqual( + data, + { + "name": "Pythonistas", + "manager": {"name": "Guido"}, + "popularity": 1000, + }, + ) diff --git a/tests/utils/test_dictionary.py b/tests/utils/test_dictionary.py index c83e4d6c0..5817d007b 100644 --- a/tests/utils/test_dictionary.py +++ b/tests/utils/test_dictionary.py @@ -26,3 +26,31 @@ def test_nesting(self): }, }, ) + + def test_name_clash(self): + """ + In this example, `manager` and `manager.*` could potentially clash. + Nesting should take precedence. + """ + response = make_nested( + { + "id": 1, + "name": "Pythonistas", + "manager": 1, + "manager.id": 1, + "manager.name": "Guido", + "manager.car.colour": "green", + } + ) + self.assertEqual( + response, + { + "id": 1, + "name": "Pythonistas", + "manager": { + "id": 1, + "name": "Guido", + "car": {"colour": "green"}, + }, + }, + ) From 1c42d6a8cda2262e089df2610d29986bf6a13700 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 1 Sep 2021 07:39:04 +0100 Subject: [PATCH 042/727] bumped version --- CHANGES | 20 ++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index aeda59d8e..9bcffd815 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,26 @@ Changes ======= +0.42.0 +------ +You can now use ``all_columns`` at the root. For example: + +.. code-block:: python + + await Band.select( + Band.all_columns(), + Band.manager.all_columns() + ).run() + +You can also exclude certain columns if you like: + +.. code-block:: python + + await Band.select( + Band.all_columns(exclude=[Band.id]), + Band.manager.all_columns(exclude=[Band.manager.id]) + ).run() + 0.41.1 ------ Fix a regression where if multiple tables are created in a single migration diff --git a/piccolo/__init__.py b/piccolo/__init__.py index aa70a7ec0..6c4ec24ce 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.41.1" +__VERSION__ = "0.42.0" From 5e7d9a612e4ba68fa1a395aa01aefd57ed5f6e47 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 2 Sep 2021 10:00:29 +0100 Subject: [PATCH 043/727] `Array` and `JSON` column migration fixes (#213) * create DDL class * some test fixes * fix mypy warnings * add tests for JSON and JSONB * add tests for arrays of varchars * reduce code repetition --- piccolo/apps/migrations/auto/serialisation.py | 23 +++-- piccolo/columns/base.py | 25 ++--- piccolo/engine/base.py | 4 + piccolo/engine/postgres.py | 29 ++++-- piccolo/engine/sqlite.py | 34 +++++-- piccolo/query/base.py | 89 +++++++++++++++++ piccolo/query/methods/alter.py | 98 ++++++++----------- piccolo/query/methods/create.py | 18 ++-- piccolo/query/methods/create_index.py | 13 ++- .../auto/integration/test_migrations.py | 82 ++++++++++++++++ tests/table/test_create.py | 8 +- 11 files changed, 312 insertions(+), 111 deletions(-) diff --git a/piccolo/apps/migrations/auto/serialisation.py b/piccolo/apps/migrations/auto/serialisation.py index bc97ca44c..ecca2d79b 100644 --- a/piccolo/apps/migrations/auto/serialisation.py +++ b/piccolo/apps/migrations/auto/serialisation.py @@ -21,6 +21,13 @@ ############################################################################### +def check_equality(self, other): + if getattr(other, "__hash__", None) is not None: + return self.__hash__() == other.__hash__() + else: + return False + + @dataclass class SerialisedBuiltin: builtin: t.Any @@ -29,7 +36,7 @@ def __hash__(self): return hash(self.builtin.__name__) def __eq__(self, other): - return self.__hash__() == other.__hash__() + return check_equality(self, other) def __repr__(self): return self.builtin.__name__ @@ -43,7 +50,7 @@ def __hash__(self): return self.instance.__hash__() def __eq__(self, other): - return self.__hash__() == other.__hash__() + return check_equality(self, other) def __repr__(self): return repr_class_instance(self.instance) @@ -58,7 +65,7 @@ def __hash__(self): return self.instance.__hash__() def __eq__(self, other): - return self.__hash__() == other.__hash__() + return check_equality(self, other) def __repr__(self): args = ", ".join( @@ -78,7 +85,7 @@ def __hash__(self): return hash(self.__repr__()) def __eq__(self, other): - return self.__hash__() == other.__hash__() + return check_equality(self, other) def __repr__(self): return f"{self.instance.__class__.__name__}.{self.instance.name}" @@ -94,7 +101,7 @@ def __hash__(self): ) def __eq__(self, other): - return self.__hash__() == other.__hash__() + return check_equality(self, other) def __repr__(self): tablename = self.table_type._meta.tablename @@ -126,7 +133,7 @@ def __hash__(self): return hash(self.__repr__()) def __eq__(self, other): - return self.__hash__() == other.__hash__() + return check_equality(self, other) def __repr__(self): class_name = self.enum_type.__name__ @@ -142,7 +149,7 @@ def __hash__(self): return hash(self.callable_.__name__) def __eq__(self, other): - return self.__hash__() == other.__hash__() + return check_equality(self, other) def __repr__(self): return self.callable_.__name__ @@ -156,7 +163,7 @@ def __hash__(self): return self.instance.int def __eq__(self, other): - return self.__hash__() == other.__hash__() + return check_equality(self, other) def __repr__(self): return f"UUID('{str(self.instance)}')" diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index f735bcd81..b6992513c 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -32,7 +32,6 @@ NotLike, ) from piccolo.columns.reference import LazyTableReference -from piccolo.querystring import QueryString from piccolo.utils.warnings import colored_warning if t.TYPE_CHECKING: # pragma: no cover @@ -579,7 +578,16 @@ def get_sql_value(self, value: t.Any) -> t.Any: elif isinstance(value, list): # Convert to the array syntax. output = ( - "'{" + ", ".join([self.get_sql_value(i) for i in value]) + "}'" + "'{" + + ", ".join( + [ + f'"{i}"' + if isinstance(i, str) + else str(self.get_sql_value(i)) + for i in value + ] + ) + + "}'" ) else: output = value @@ -591,7 +599,7 @@ def column_type(self): return self.__class__.__name__.upper() @property - def querystring(self) -> QueryString: + def ddl(self) -> str: """ Used when creating tables. """ @@ -621,16 +629,9 @@ def querystring(self) -> QueryString: if not self._meta.primary_key: default = self.get_default_value() sql_value = self.get_sql_value(value=default) - # Escape the value if it contains a pair of curly braces, otherwise - # an empty value will appear in the compiled querystring. - sql_value = ( - sql_value.replace("{}", "{{}}") - if isinstance(sql_value, str) - else sql_value - ) query += f" DEFAULT {sql_value}" - return QueryString(query) + return query def copy(self) -> Column: column: Column = copy.copy(self) @@ -645,7 +646,7 @@ def __deepcopy__(self, memo) -> Column: return self.copy() def __str__(self): - return self.querystring.__str__() + return self.ddl.__str__() def __repr__(self): try: diff --git a/piccolo/engine/base.py b/piccolo/engine/base.py index 968fe049d..2db51692c 100644 --- a/piccolo/engine/base.py +++ b/piccolo/engine/base.py @@ -50,6 +50,10 @@ async def batch(self, query: Query, batch_size: int = 100) -> Batch: async def run_querystring(self, querystring: QueryString, in_pool: bool): pass + @abstractmethod + async def run_ddl(self, ddl: str, in_pool: bool = True): + pass + async def check_version(self): """ Warn if the database version isn't supported. diff --git a/piccolo/engine/postgres.py b/piccolo/engine/postgres.py index 7d815bd0a..dce9b1b81 100644 --- a/piccolo/engine/postgres.py +++ b/piccolo/engine/postgres.py @@ -6,7 +6,7 @@ from piccolo.engine.base import Batch, Engine from piccolo.engine.exceptions import TransactionError -from piccolo.query.base import Query +from piccolo.query.base import DDL, Query from piccolo.querystring import QueryString from piccolo.utils.lazy_loader import LazyLoader from piccolo.utils.sync import run_sync @@ -100,11 +100,15 @@ def add(self, *query: Query): async def _run_queries(self, connection): async with connection.transaction(): for query in self.queries: - for querystring in query.querystrings: - _query, args = querystring.compile_string( - engine_type=self.engine.engine_type - ) - await connection.execute(_query, *args) + if isinstance(query, Query): + for querystring in query.querystrings: + _query, args = querystring.compile_string( + engine_type=self.engine.engine_type + ) + await connection.execute(_query, *args) + elif isinstance(query, DDL): + for ddl in query.ddl: + await connection.execute(ddl) self.queries = [] @@ -391,6 +395,19 @@ async def run_querystring( else: return await self._run_in_new_connection(query, query_args) + async def run_ddl(self, ddl: str, in_pool: bool = True): + if self.log_queries: + print(ddl) + + # If running inside a transaction: + connection = self.transaction_connection.get() + if connection: + return await connection.fetch(ddl) + elif in_pool and self.pool: + return await self._run_in_pool(ddl) + else: + return await self._run_in_new_connection(ddl) + def atomic(self) -> Atomic: return Atomic(engine=self) diff --git a/piccolo/engine/sqlite.py b/piccolo/engine/sqlite.py index 8b70e825f..cb84f3148 100644 --- a/piccolo/engine/sqlite.py +++ b/piccolo/engine/sqlite.py @@ -11,7 +11,7 @@ from piccolo.engine.base import Batch, Engine from piccolo.engine.exceptions import TransactionError -from piccolo.query.base import Query +from piccolo.query.base import DDL, Query from piccolo.querystring import QueryString from piccolo.utils.encoding import dump_json, load_json from piccolo.utils.lazy_loader import LazyLoader @@ -250,12 +250,17 @@ async def run(self): try: for query in self.queries: - for querystring in query.querystrings: - await connection.execute( - *querystring.compile_string( - engine_type=self.engine.engine_type + if isinstance(query, Query): + for querystring in query.querystrings: + await connection.execute( + *querystring.compile_string( + engine_type=self.engine.engine_type + ) ) - ) + elif isinstance(query, DDL): + for ddl in query.ddl: + await connection.execute(ddl) + except Exception as exception: await connection.execute("ROLLBACK") await connection.close() @@ -513,6 +518,23 @@ async def run_querystring( table=querystring.table, ) + async def run_ddl(self, ddl: str, in_pool: bool = False): + """ + Connection pools aren't currently supported - the argument is there + for consistency with other engines. + """ + # If running inside a transaction: + connection = self.transaction_connection.get() + if connection: + return await self._run_in_existing_connection( + connection=connection, + query=ddl, + ) + + return await self._run_in_new_connection( + query=ddl, + ) + def atomic(self) -> Atomic: return Atomic(engine=self) diff --git a/piccolo/query/base.py b/piccolo/query/base.py index 5ec36f2de..7c094f71a 100644 --- a/piccolo/query/base.py +++ b/piccolo/query/base.py @@ -317,3 +317,92 @@ def __getattr__(self, name: str): def __str__(self) -> str: return self.query.__str__() + + +class DDL: + + __slots__ = ("table",) + + def __init__(self, table: t.Type[Table], **kwargs): + self.table = table + + @property + def engine_type(self) -> str: + engine = self.table._meta.db + if engine: + return engine.engine_type + else: + raise ValueError("Engine isn't defined.") + + @property + def sqlite_ddl(self) -> t.Sequence[str]: + raise NotImplementedError + + @property + def postgres_ddl(self) -> t.Sequence[str]: + raise NotImplementedError + + @property + def default_ddl(self) -> t.Sequence[str]: + raise NotImplementedError + + @property + def ddl(self) -> t.Sequence[str]: + """ + Calls the correct underlying method, depending on the current engine. + """ + engine_type = self.engine_type + if engine_type == "postgres": + try: + return self.postgres_ddl + except NotImplementedError: + return self.default_ddl + elif engine_type == "sqlite": + try: + return self.sqlite_ddl + except NotImplementedError: + return self.default_ddl + else: + raise Exception( + f"No querystring found for the {engine_type} engine." + ) + + def __await__(self): + """ + If the user doesn't explicity call .run(), proxy to it as a + convenience. + """ + return self.run().__await__() + + async def run(self, in_pool=True): + engine = self.table._meta.db + if not engine: + raise ValueError( + f"Table {self.table._meta.tablename} has no db defined in " + "_meta" + ) + + if len(self.ddl) == 1: + return await engine.run_ddl(self.ddl[0], in_pool=in_pool) + else: + responses = [] + # TODO - run in a transaction + for ddl in self.ddl: + response = await engine.run_ddl(ddl, in_pool=in_pool) + responses.append(response) + return responses + + def run_sync(self, timed=False, *args, **kwargs): + """ + A convenience method for running the coroutine synchronously. + """ + coroutine = self.run(*args, **kwargs, in_pool=False) + + if timed: + with Timer(): + return run_sync(coroutine) + else: + return run_sync(coroutine) + + def __str__(self) -> str: + return self.ddl.__str__() diff --git a/piccolo/query/methods/alter.py b/piccolo/query/methods/alter.py index 5e9b3b5be..9a5cac5ff 100644 --- a/piccolo/query/methods/alter.py +++ b/piccolo/query/methods/alter.py @@ -6,8 +6,7 @@ from piccolo.columns.base import Column from piccolo.columns.column_types import ForeignKey, Numeric, Varchar -from piccolo.query.base import Query -from piccolo.querystring import QueryString +from piccolo.query.base import DDL from piccolo.utils.warnings import Level, colored_warning if t.TYPE_CHECKING: # pragma: no cover @@ -15,15 +14,15 @@ from piccolo.table import Table -@dataclass class AlterStatement: __slots__ = tuple() # type: ignore - def querystring(self) -> QueryString: + @property + def ddl(self) -> str: raise NotImplementedError() def __str__(self) -> str: - return self.querystring.__str__() + return self.ddl @dataclass @@ -33,8 +32,8 @@ class RenameTable(AlterStatement): new_name: str @property - def querystring(self) -> QueryString: - return QueryString(f"RENAME TO {self.new_name}") + def ddl(self) -> str: + return f"RENAME TO {self.new_name}" @dataclass @@ -60,17 +59,15 @@ class RenameColumn(AlterColumnStatement): new_name: str @property - def querystring(self) -> QueryString: - return QueryString( - f"RENAME COLUMN {self.column_name} TO {self.new_name}" - ) + def ddl(self) -> str: + return f"RENAME COLUMN {self.column_name} TO {self.new_name}" @dataclass class DropColumn(AlterColumnStatement): @property - def querystring(self) -> QueryString: - return QueryString(f"DROP COLUMN {self.column_name}") + def ddl(self) -> str: + return f"DROP COLUMN {self.column_name}" @dataclass @@ -81,16 +78,16 @@ class AddColumn(AlterColumnStatement): name: str @property - def querystring(self) -> QueryString: + def ddl(self) -> str: self.column._meta.name = self.name - return QueryString("ADD COLUMN {}", self.column.querystring) + return f"ADD COLUMN {self.column.ddl}" @dataclass class DropDefault(AlterColumnStatement): @property - def querystring(self) -> QueryString: - return QueryString(f"ALTER COLUMN {self.column_name} DROP DEFAULT") + def ddl(self) -> str: + return f"ALTER COLUMN {self.column_name} DROP DEFAULT" @dataclass @@ -108,7 +105,7 @@ class SetColumnType(AlterStatement): using_expression: t.Optional[str] = None @property - def querystring(self) -> QueryString: + def ddl(self) -> str: if self.new_column._meta._table is None: self.new_column._meta._table = self.old_column._meta.table @@ -118,7 +115,7 @@ def querystring(self) -> QueryString: ) if self.using_expression is not None: query += f" USING {self.using_expression}" - return QueryString(query) + return query @dataclass @@ -129,11 +126,9 @@ class SetDefault(AlterColumnStatement): value: t.Any @property - def querystring(self) -> QueryString: + def ddl(self) -> str: sql_value = self.column.get_sql_value(self.value) - return QueryString( - f"ALTER COLUMN {self.column_name} SET DEFAULT {sql_value}" - ) + return f"ALTER COLUMN {self.column_name} SET DEFAULT {sql_value}" @dataclass @@ -143,9 +138,9 @@ class SetUnique(AlterColumnStatement): boolean: bool @property - def querystring(self) -> QueryString: + def ddl(self) -> str: if self.boolean: - return QueryString(f"ADD UNIQUE ({self.column_name})") + return f"ADD UNIQUE ({self.column_name})" else: if isinstance(self.column, str): raise ValueError( @@ -155,7 +150,7 @@ def querystring(self) -> QueryString: tablename = self.column._meta.table._meta.tablename column_name = self.column_name key = f"{tablename}_{column_name}_key" - return QueryString(f'DROP CONSTRAINT "{key}"') + return f'DROP CONSTRAINT "{key}"' @dataclass @@ -165,13 +160,11 @@ class SetNull(AlterColumnStatement): boolean: bool @property - def querystring(self) -> QueryString: + def ddl(self) -> str: if self.boolean: - return QueryString( - f"ALTER COLUMN {self.column_name} DROP NOT NULL" - ) + return f"ALTER COLUMN {self.column_name} DROP NOT NULL" else: - return QueryString(f"ALTER COLUMN {self.column_name} SET NOT NULL") + return f"ALTER COLUMN {self.column_name} SET NOT NULL" @dataclass @@ -182,10 +175,8 @@ class SetLength(AlterColumnStatement): length: int @property - def querystring(self) -> QueryString: - return QueryString( - f"ALTER COLUMN {self.column_name} TYPE VARCHAR({self.length})" - ) + def ddl(self) -> str: + return f"ALTER COLUMN {self.column_name} TYPE VARCHAR({self.length})" @dataclass @@ -195,10 +186,8 @@ class DropConstraint(AlterStatement): constraint_name: str @property - def querystring(self) -> QueryString: - return QueryString( - f"DROP CONSTRAINT IF EXISTS {self.constraint_name}", - ) + def ddl(self) -> str: + return f"DROP CONSTRAINT IF EXISTS {self.constraint_name}" @dataclass @@ -219,7 +208,7 @@ class AddForeignKeyConstraint(AlterStatement): referenced_column_name: str = "id" @property - def querystring(self) -> QueryString: + def ddl(self) -> str: query = ( f"ADD CONSTRAINT {self.constraint_name} FOREIGN KEY " f"({self.foreign_key_column_name}) REFERENCES " @@ -229,7 +218,7 @@ def querystring(self) -> QueryString: query += f" ON DELETE {self.on_delete.value}" if self.on_update: query += f" ON UPDATE {self.on_update.value}" - return QueryString(query) + return query @dataclass @@ -241,18 +230,16 @@ class SetDigits(AlterColumnStatement): column_type: str @property - def querystring(self) -> QueryString: + def ddl(self) -> str: if self.digits is not None: precision = self.digits[0] scale = self.digits[1] - return QueryString( + return ( f"ALTER COLUMN {self.column_name} TYPE " f"{self.column_type}({precision}, {scale})" ) else: - return QueryString( - f"ALTER COLUMN {self.column_name} TYPE {self.column_type}", - ) + return f"ALTER COLUMN {self.column_name} TYPE {self.column_type}" @dataclass @@ -262,7 +249,7 @@ class DropTable: if_exists: bool @property - def querystring(self) -> QueryString: + def ddl(self) -> str: query = "DROP TABLE" if self.if_exists: @@ -273,10 +260,10 @@ def querystring(self) -> QueryString: if self.cascade: query += " CASCADE" - return QueryString(query) + return query -class Alter(Query): +class Alter(DDL): __slots__ = ( "_add_foreign_key_constraint", @@ -512,14 +499,14 @@ def set_digits( return self @property - def default_querystrings(self) -> t.Sequence[QueryString]: + def default_ddl(self) -> t.Sequence[str]: if self._drop_table is not None: - return [self._drop_table.querystring] + return [self._drop_table.ddl] query = f"ALTER TABLE {self.table._meta.tablename}" alterations = [ - i.querystring + i.ddl for i in itertools.chain( self._add, self._rename_columns, @@ -537,10 +524,9 @@ def default_querystrings(self) -> t.Sequence[QueryString]: if self.engine_type == "sqlite": # Can only perform one alter statement at a time. - query += " {}" - return [QueryString(query, i) for i in alterations] + return [f"{query} {i}" for i in alterations] # Postgres can perform them all at once: - query += ",".join([" {}" for i in alterations]) + query += ",".join([f" {i}" for i in alterations]) - return [QueryString(query, *alterations)] + return [query] diff --git a/piccolo/query/methods/create.py b/piccolo/query/methods/create.py index 1b0108bda..e84d1465d 100644 --- a/piccolo/query/methods/create.py +++ b/piccolo/query/methods/create.py @@ -2,15 +2,14 @@ import typing as t -from piccolo.query.base import Query +from piccolo.query.base import DDL from piccolo.query.methods.create_index import CreateIndex -from piccolo.querystring import QueryString if t.TYPE_CHECKING: # pragma: no cover from piccolo.table import Table -class Create(Query): +class Create(DDL): """ Creates a database table. """ @@ -29,7 +28,7 @@ def __init__( self.only_default_columns = only_default_columns @property - def default_querystrings(self) -> t.Sequence[QueryString]: + def default_ddl(self) -> t.Sequence[str]: prefix = "CREATE TABLE" if self.if_not_exists: prefix += " IF NOT EXISTS" @@ -40,11 +39,10 @@ def default_querystrings(self) -> t.Sequence[QueryString]: columns = self.table._meta.columns base = f"{prefix} {self.table._meta.tablename}" - columns_sql = ", ".join(["{}" for i in columns]) - query = f"{base} ({columns_sql})" - create_table = QueryString(query, *[i.querystring for i in columns]) + columns_sql = ", ".join([i.ddl for i in columns]) + create_table_ddl = f"{base} ({columns_sql})" - create_indexes: t.List[QueryString] = [] + create_indexes: t.List[str] = [] for column in columns: if column._meta.index is True: create_indexes.extend( @@ -53,7 +51,7 @@ def default_querystrings(self) -> t.Sequence[QueryString]: columns=[column], method=column._meta.index_method, if_not_exists=self.if_not_exists, - ).querystrings + ).ddl ) - return [create_table] + create_indexes + return [create_table_ddl] + create_indexes diff --git a/piccolo/query/methods/create_index.py b/piccolo/query/methods/create_index.py index 3fc4685aa..7b6e93d99 100644 --- a/piccolo/query/methods/create_index.py +++ b/piccolo/query/methods/create_index.py @@ -4,14 +4,13 @@ from piccolo.columns import Column from piccolo.columns.indexes import IndexMethod -from piccolo.query.base import Query -from piccolo.querystring import QueryString +from piccolo.query.base import DDL if t.TYPE_CHECKING: # pragma: no cover from piccolo.table import Table -class CreateIndex(Query): +class CreateIndex(DDL): def __init__( self, table: t.Type[Table], @@ -39,21 +38,21 @@ def prefix(self) -> str: return prefix @property - def postgres_querystrings(self) -> t.Sequence[QueryString]: + def postgres_ddl(self) -> t.Sequence[str]: column_names = self.column_names index_name = self.table._get_index_name(column_names) tablename = self.table._meta.tablename method_name = self.method.value column_names_str = ", ".join(column_names) return [ - QueryString( + ( f"{self.prefix} {index_name} ON {tablename} USING " f"{method_name} ({column_names_str})" ) ] @property - def sqlite_querystrings(self) -> t.Sequence[QueryString]: + def sqlite_ddl(self) -> t.Sequence[str]: column_names = self.column_names index_name = self.table._get_index_name(column_names) tablename = self.table._meta.tablename @@ -64,7 +63,7 @@ def sqlite_querystrings(self) -> t.Sequence[QueryString]: column_names_str = ", ".join(column_names) return [ - QueryString( + ( f"{self.prefix} {index_name} ON {tablename} " f"({column_names_str})" ) diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index 0cd336502..8738bd461 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -16,7 +16,10 @@ ) from piccolo.apps.migrations.tables import Migration from piccolo.columns.column_types import ( + JSON, + JSONB, UUID, + Array, BigInt, Boolean, Date, @@ -70,6 +73,14 @@ def boolean_default(): return True +def array_default_integer(): + return [4, 5, 6] + + +def array_default_varchar(): + return ['x', 'y', 'z'] + + @postgres_only class TestMigrations(TestCase): def tearDown(self): @@ -310,3 +321,74 @@ def test_boolean_column(self): ] ] ) + + def test_array_column_integer(self): + self._test_migrations( + table_classes=[ + self.table(column) + for column in [ + Array(base_column=Integer()), + Array(base_column=Integer(), default=[1, 2, 3]), + Array( + base_column=Integer(), default=array_default_integer + ), + Array(base_column=Integer(), null=True, default=None), + Array(base_column=Integer(), null=False), + Array(base_column=Integer(), index=True), + Array(base_column=Integer(), index=False), + ] + ] + ) + + def test_array_column_varchar(self): + self._test_migrations( + table_classes=[ + self.table(column) + for column in [ + Array(base_column=Varchar()), + Array(base_column=Varchar(), default=["a", "b", "c"]), + Array( + base_column=Varchar(), default=array_default_varchar + ), + Array(base_column=Varchar(), null=True, default=None), + Array(base_column=Varchar(), null=False), + Array(base_column=Varchar(), index=True), + Array(base_column=Varchar(), index=False), + ] + ] + ) + + ########################################################################### + + # We deliberately don't test setting JSON or JSONB columns as indexes, as + # we know it'll fail. + + def test_json_column(self): + self._test_migrations( + table_classes=[ + self.table(column) + for column in [ + JSON(), + JSON(default=["a", "b", "c"]), + JSON(default={"name": "bob"}), + JSON(default='{"name": "Sally"}'), + JSON(null=True, default=None), + JSON(null=False), + ] + ] + ) + + def test_jsonb_column(self): + self._test_migrations( + table_classes=[ + self.table(column) + for column in [ + JSONB(), + JSONB(default=["a", "b", "c"]), + JSONB(default={"name": "bob"}), + JSONB(default='{"name": "Sally"}'), + JSONB(null=True, default=None), + JSONB(null=False), + ] + ] + ) diff --git a/tests/table/test_create.py b/tests/table/test_create.py index b8cd90034..5ef01335e 100644 --- a/tests/table/test_create.py +++ b/tests/table/test_create.py @@ -42,10 +42,6 @@ def test_create_if_not_exists_with_indexes(self): query.run_sync() self.assertTrue( - query.querystrings[0] - .__str__() - .startswith("CREATE TABLE IF NOT EXISTS"), - query.querystrings[1] - .__str__() - .startswith("CREATE INDEX IF NOT EXISTS"), + query.ddl[0].__str__().startswith("CREATE TABLE IF NOT EXISTS"), + query.ddl[1].__str__().startswith("CREATE INDEX IF NOT EXISTS"), ) From fd72c6064188a4139238cf70b416a0b3862e0da0 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 2 Sep 2021 10:02:32 +0100 Subject: [PATCH 044/727] bumped version --- CHANGES | 5 +++++ piccolo/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 9bcffd815..07eb41b2f 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,11 @@ Changes ======= +0.43.0 +------ +Migrations containing ``Array``, ``JSON`` and ``JSONB`` columns should be +more reliable now. More unit tests were added to cover edge cases. + 0.42.0 ------ You can now use ``all_columns`` at the root. For example: diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 6c4ec24ce..68ec08fd1 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.42.0" +__VERSION__ = "0.43.0" From 0717f8db8d052c19b33bfae33e04ca8f2207ea1e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 7 Sep 2021 19:27:15 +0100 Subject: [PATCH 045/727] Nested objects prototype (#217) * prototype for nested objects * fix typo * finish prototype * fix linting errors * added `all_related` method * more tests * adds more tests and docs * add notes about the `prefetch` clause * make sure `prefetch` work with `get` and `get_or_create` * load all intermediate objects * another test * update docs - declaring intermediate objects no longer required --- docs/src/piccolo/query_types/objects.rst | 100 ++++++++-- piccolo/columns/column_types.py | 66 ++++++- piccolo/query/base.py | 19 +- piccolo/query/methods/objects.py | 48 ++++- piccolo/query/mixins.py | 32 +++ piccolo/table.py | 94 +++++++-- piccolo/utils/dictionary.py | 12 +- piccolo/utils/objects.py | 64 ++++++ .../auto/integration/test_migrations.py | 2 +- .../commands/test_forwards_backwards.py | 1 + tests/columns/test_foreignkey.py | 61 +++++- tests/conftest.py | 4 +- .../2021-09-06T13-58-23-024723.py | 48 +++++ tests/example_app/tables.py | 1 + tests/table/test_join.py | 184 ++++++++++++++++-- tests/table/test_objects.py | 47 ++++- tests/testing/test_model_builder.py | 4 + 17 files changed, 732 insertions(+), 55 deletions(-) create mode 100644 piccolo/utils/objects.py create mode 100644 tests/example_app/piccolo_migrations/2021-09-06T13-58-23-024723.py diff --git a/docs/src/piccolo/query_types/objects.rst b/docs/src/piccolo/query_types/objects.rst index d5430d6cb..80ba9c4e7 100644 --- a/docs/src/piccolo/query_types/objects.rst +++ b/docs/src/piccolo/query_types/objects.rst @@ -11,6 +11,8 @@ can manipulate them, and save the changes back to the database. In Piccolo, an instance of a ``Table`` class represents a row. Let's do some examples. +------------------------------------------------------------------------------- + Fetching objects ---------------- @@ -45,6 +47,8 @@ To get the first row: You'll notice that the API is similar to :ref:`Select` - except it returns all columns. +------------------------------------------------------------------------------- + Creating objects ---------------- @@ -53,6 +57,8 @@ Creating objects >>> band = Band(name="C-Sharps", popularity=100) >>> band.save().run_sync() +------------------------------------------------------------------------------- + Updating objects ---------------- @@ -60,12 +66,14 @@ Objects have a ``save`` method, which is convenient for updating values: .. code-block:: python - pythonistas = Band.objects().where( + band = Band.objects().where( Band.name == 'Pythonistas' ).first().run_sync() - pythonistas.popularity = 100000 - pythonistas.save().run_sync() + band.popularity = 100000 + band.save().run_sync() + +------------------------------------------------------------------------------- Deleting objects ---------------- @@ -74,28 +82,95 @@ Similarly, we can delete objects, using the ``remove`` method. .. code-block:: python - pythonistas = Band.objects().where( + band = Band.objects().where( Band.name == 'Pythonistas' ).first().run_sync() - pythonistas.remove().run_sync() + band.remove().run_sync() + +------------------------------------------------------------------------------- + +Fetching related objects +------------------------ get_related ------------ +~~~~~~~~~~~ + +If you have an object from a table with a ``ForeignKey`` column, and you want +to fetch the related row as an object, you can do so using ``get_related``. + +.. code-block:: python -If you have an object with a foreign key, and you want to fetch the related -object, you can do so using ``get_related``. + band = Band.objects().where( + Band.name == 'Pythonistas' + ).first().run_sync() + + manager = band.get_related(Band.manager).run_sync() + >>> manager + + >>> manager.name + 'Guido' + +Prefetching related objects +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can also prefetch the rows from related tables, and store them as child +objects. To do this, pass ``ForeignKey`` columns into ``objects``, which +refer to the related rows you want to load. .. code-block:: python - pythonistas = Band.objects().where( + band = Band.objects(Band.manager).where( Band.name == 'Pythonistas' ).first().run_sync() - manager = pythonistas.get_related(Band.manager).run_sync() - >>> print(manager.name) + >>> band.manager + + >>> band.manager.name 'Guido' +If you have a table containing lots of ``ForeignKey`` columns, and want to +prefetch them all you can do so using ``all_related``. + +.. code-block:: python + + ticket = Ticket.objects( + Ticket.concert.all_related() + ).first().run_sync() + + # Any intermediate objects will also be loaded: + >>> ticket.concert + + + >>> ticket.concert.band_1 + + >>> ticket.concert.band_2 + + +You can manipulate these nested objects, and save the values back to the +database, just as you would expect: + +.. code-block:: python + + ticket.concert.band_1.name = 'Pythonistas 2' + ticket.concert.band_1.save().run_sync() + +Instead of passing the ``ForeignKey`` columns into the ``objects`` method, you +can use the ``prefetch`` clause if you prefer. + +.. code-block:: python + + # These are equivalent: + ticket = Ticket.objects( + Ticket.concert.all_related() + ).first().run_sync() + + ticket = Ticket.objects().prefetch( + Ticket.concert.all_related() + ).run_sync() + +------------------------------------------------------------------------------- + get_or_create ------------- @@ -138,6 +213,8 @@ Complex where clauses are supported, but only within reason. For example: defaults={'popularity': 100} ).run_sync() +------------------------------------------------------------------------------- + to_dict ------- @@ -161,6 +238,7 @@ the columns: >>> band.to_dict(Band.id, Band.name.as_alias('title')) {'id': 1, 'title': 'Pythonistas'} +------------------------------------------------------------------------------- Query clauses ------------- diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 3de7fd21e..782eda04c 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -1224,7 +1224,9 @@ def all_columns( self, exclude: t.List[t.Union[Column, str]] = [] ) -> t.List[Column]: """ - Allow a user to access all of the columns on the related table. + Allow a user to access all of the columns on the related table. This is + intended for use with ``select`` queries, and saves the user from + typing out all of the columns by hand. For example: @@ -1232,6 +1234,13 @@ def all_columns( Band.select(Band.name, Band.manager.all_columns()).run_sync() + # Equivalent to: + Band.select( + Band.name, + Band.manager.id, + Band.manager.name + ).run_sync() + To exclude certain columns: .. code-block:: python @@ -1260,6 +1269,60 @@ def all_columns( if column._meta.name not in excluded_column_names ] + def all_related( + self, exclude: t.List[t.Union[ForeignKey, str]] = [] + ) -> t.List[ForeignKey]: + """ + Returns each ``ForeignKey`` column on the related table. This is + intended for use with ``objects`` queries, where you want to return + all of the related tables as nested objects. + + For example: + + .. code-block:: python + + class Band(Table): + name = Varchar() + + class Concert(Table): + name = Varchar() + band_1 = ForeignKey(Band) + band_2 = ForeignKey(Band) + + class Tour(Table): + name = Varchar() + concert = ForeignKey(Concert) + + Tour.objects(Tour.concert, Tour.concert.all_related()).run_sync() + + # Equivalent to + Tour.objects( + Tour.concert, + Tour.concert.band_1, + Tour.concert.band_2 + ).run_sync() + + :param exclude: + Columns to exclude - can be the name of a column, or a + ``ForeignKey`` instance. For example ``['band_1']`` or + ``[Tour.concert.band_1]``. + + """ + _fk_meta: ForeignKeyMeta = object.__getattribute__( + self, "_foreign_key_meta" + ) + related_fk_columns = ( + _fk_meta.resolved_references._meta.foreign_key_columns + ) + excluded_column_names = [ + i._meta.name if isinstance(i, ForeignKey) else i for i in exclude + ] + return [ + getattr(self, fk_column._meta.name) + for fk_column in related_fk_columns + if fk_column._meta.name not in excluded_column_names + ] + def set_proxy_columns(self): """ In order to allow a fluent interface, where tables can be traversed @@ -1334,7 +1397,6 @@ def __getattribute__(self, name: str): _column._meta.call_chain = [ i for i in new_column._meta.call_chain ] - _column._meta.call_chain.append(new_column) setattr(new_column, _column._meta.name, _column) foreign_key_meta.proxy_columns.append(_column) diff --git a/piccolo/query/base.py b/piccolo/query/base.py index 7c094f71a..fc6382d9b 100644 --- a/piccolo/query/base.py +++ b/piccolo/query/base.py @@ -8,6 +8,7 @@ from piccolo.query.mixins import ColumnsDelegate from piccolo.querystring import QueryString from piccolo.utils.encoding import dump_json, load_json +from piccolo.utils.objects import make_nested_object from piccolo.utils.sync import run_sync if t.TYPE_CHECKING: # pragma: no cover @@ -109,14 +110,22 @@ async def _process_results(self, results): # noqa: C901 # When using .first() we get a single row, not a list # of rows. if type(raw) is list: - raw = [ - self.table(**columns, exists_in_db=True) - for columns in raw - ] + if output._output.nested: + raw = [ + make_nested_object(row, self.table) for row in raw + ] + else: + raw = [ + self.table(**columns, exists_in_db=True) + for columns in raw + ] elif raw is None: pass else: - raw = self.table(**raw, exists_in_db=True) + if output._output.nested: + raw = make_nested_object(raw, self.table) + else: + raw = self.table(**raw, exists_in_db=True) elif type(raw) is list: if output._output.as_list: if len(raw) == 0: diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index 426a4934c..b5ef82a9d 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -3,6 +3,7 @@ import typing as t from dataclasses import dataclass +from piccolo.columns.column_types import ForeignKey from piccolo.columns.combination import And, Where from piccolo.custom_types import Combinable from piccolo.engine.base import Batch @@ -12,9 +13,11 @@ OffsetDelegate, OrderByDelegate, OutputDelegate, + PrefetchDelegate, WhereDelegate, ) from piccolo.querystring import QueryString +from piccolo.utils.dictionary import make_nested from piccolo.utils.sync import run_sync from .select import Select @@ -74,6 +77,10 @@ def __await__(self): def run_sync(self): return run_sync(self.run()) + def prefetch(self, *fk_columns) -> GetOrCreate: + self.query.prefetch(*fk_columns) + return self + @dataclass class Objects(Query): @@ -83,20 +90,29 @@ class Objects(Query): """ __slots__ = ( + "nested", "limit_delegate", "offset_delegate", "order_by_delegate", "output_delegate", + "prefetch_delegate", "where_delegate", ) - def __init__(self, table: t.Type[Table], **kwargs): + def __init__( + self, + table: t.Type[Table], + prefetch: t.Sequence[t.Union[ForeignKey, t.List[ForeignKey]]] = (), + **kwargs, + ): super().__init__(table, **kwargs) self.limit_delegate = LimitDelegate() self.offset_delegate = OffsetDelegate() self.order_by_delegate = OrderByDelegate() self.output_delegate = OutputDelegate() self.output_delegate._output.as_objects = True + self.prefetch_delegate = PrefetchDelegate() + self.prefetch(*prefetch) self.where_delegate = WhereDelegate() def output(self, load_json: bool = False) -> Objects: @@ -113,6 +129,12 @@ def first(self) -> Objects: self.limit_delegate.first() return self + def prefetch( + self, *fk_columns: t.Union[ForeignKey, t.List[ForeignKey]] + ) -> Objects: + self.prefetch_delegate.prefetch(*fk_columns) + return self + def get(self, where: Combinable) -> Objects: self.where_delegate.where(where) self.limit_delegate.first() @@ -149,9 +171,15 @@ async def response_handler(self, response): if len(response) == 0: return None else: - return response[0] + if self.output_delegate._output.nested: + return make_nested(response[0]) + else: + return response[0] else: - return response + if self.output_delegate._output.nested: + return [make_nested(i) for i in response] + else: + return response @property def default_querystrings(self) -> t.Sequence[QueryString]: @@ -166,4 +194,18 @@ def default_querystrings(self) -> t.Sequence[QueryString]: ): setattr(select, attr, getattr(self, attr)) + if self.prefetch_delegate.fk_columns: + select.columns(*self.table.all_columns()) + for fk in self.prefetch_delegate.fk_columns: + if isinstance(fk, ForeignKey): + select.columns(*fk.all_columns()) + else: + raise ValueError(f"{fk} doesn't seem to be a ForeignKey.") + + # Make sure that all intermediate objects are fully loaded. + for parent_fk in fk._meta.call_chain: + select.columns(*parent_fk.all_columns()) + + select.output_delegate.output(nested=True) + return select.querystrings diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index 11673f4cd..178434821 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -4,6 +4,7 @@ from dataclasses import dataclass, field from piccolo.columns import And, Column, Or, Secret, Where +from piccolo.columns.column_types import ForeignKey from piccolo.custom_types import Combinable from piccolo.querystring import QueryString from piccolo.utils.sql_values import convert_to_sql_value @@ -210,6 +211,8 @@ def output( If True, any JSON fields will have the JSON values returned from the database loaded as Python objects. """ + # We do it like this, so output can be called multiple times, without + # overriding any existing values if they're not specified. if as_list is not None: self._output.as_list = bool(as_list) @@ -227,6 +230,35 @@ def copy(self) -> OutputDelegate: return self.__class__(_output=_output) +@dataclass +class PrefetchDelegate: + """ + Example usage: + + .prefetch(MyTable.column_a, MyTable.column_b) + """ + + fk_columns: t.List[ForeignKey] = field(default_factory=list) + + def prefetch(self, *fk_columns: t.Union[ForeignKey, t.List[ForeignKey]]): + """ + :param columns: + We accept ``ForeignKey`` and ``List[ForeignKey]`` here, in case + someone passes in a list by accident when using ``all_related()``, + in which case we flatten the list. + + """ + _fk_columns: t.List[ForeignKey] = [] + for column in fk_columns: + if isinstance(column, list): + _fk_columns.extend(column) + else: + _fk_columns.append(column) + + combined = self.fk_columns + _fk_columns + self.fk_columns = combined + + @dataclass class ColumnsDelegate: """ diff --git a/piccolo/table.py b/piccolo/table.py index a000125a2..ce66c55c2 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -517,14 +517,63 @@ def __repr__(self) -> str: ########################################################################### # Classmethods + @classmethod + def all_related( + cls, exclude: t.List[t.Union[str, ForeignKey]] = [] + ) -> t.List[Column]: + """ + Used in conjunction with ``objects`` queries. Just as we can use + ``all_related`` on a ``ForeignKey``, you can also use it for the table + at the root of the query, which will return each related row as a + nested object. For example: + + .. code-block:: python + + concert = await Concert.objects( + Concert.all_related() + ).run() + + >>> concert.band_1 + + >>> concert.band_2 + + >>> concert.venue + + + This is mostly useful when the table has a lot of foreign keys, and + typing them out by hand would be tedious. It's equivalent to: + + .. code-block:: python + + concert = await Concert.objects( + Concert.venue, + Concert.band_1, + Concert.band_2 + ).run() + + :param exclude: + You can request all columns, except these. + + """ + excluded_column_names = [ + i._meta.name if isinstance(i, ForeignKey) else i for i in exclude + ] + + return [ + i + for i in cls._meta.foreign_key_columns + if i._meta.name not in excluded_column_names + ] + @classmethod def all_columns( cls, exclude: t.List[t.Union[str, Column]] = [] ) -> t.List[Column]: """ - Just as we can use ``all_columns`` to retrieve all of the columns from - a related table, we can also use it at the root of our query to get - all of the columns for the root table. For example: + Used in conjunction with ``select`` queries. Just as we can use + ``all_columns`` to retrieve all of the columns from a related table, + we can also use it at the root of our query to get all of the columns + for the root table. For example: .. code-block:: python @@ -680,23 +729,44 @@ def alter(cls) -> Alter: return Alter(table=cls) @classmethod - def objects(cls) -> Objects: + def objects( + cls, *prefetch: t.Union[ForeignKey, t.List[ForeignKey]] + ) -> Objects: """ Returns a list of table instances (each representing a row), which you can modify and then call 'save' on, or can delete by calling 'remove'. - pythonistas = await Band.objects().where( - Band.name == 'Pythonistas' - ).first().run() + .. code-block:: python + + pythonistas = await Band.objects().where( + Band.name == 'Pythonistas' + ).first().run() + + pythonistas.name = 'Pythonistas Reborn' + + await pythonistas.save().run() + + # Or to remove it from the database: + await pythonistas.remove() + + :param prefetch: + Rather than returning the primary key value of this related table, + a nested object will be returned for the row on the related table. + + .. code-block:: python - pythonistas.name = 'Pythonistas Reborn' + # Without nested + band = await Band.objects().first().run() + >>> band.manager + 1 - await pythonistas.save().run() + # With nested + band = await Band.objects(Band.manager).first().run() + >>> band.manager + - # Or to remove it from the database: - await pythonistas.remove() """ - return Objects(table=cls) + return Objects(table=cls, prefetch=prefetch) @classmethod def count(cls) -> Count: diff --git a/piccolo/utils/dictionary.py b/piccolo/utils/dictionary.py index cd10dc8ce..6259e395e 100644 --- a/piccolo/utils/dictionary.py +++ b/piccolo/utils/dictionary.py @@ -22,7 +22,10 @@ def make_nested(dictionary: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]: """ output: t.Dict[str, t.Any] = {} - for key, value in dictionary.items(): + items = [i for i in dictionary.items()] + items.sort(key=lambda x: x[0]) + + for key, value in items: path = key.split(".") if len(path) == 1: output[path[0]] = value @@ -44,7 +47,12 @@ def make_nested(dictionary: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]: output[path[0]] = dictionary for path_element in path[1:-1]: - dictionary = dictionary.setdefault(path_element, {}) + root = dictionary.setdefault(path_element, {}) + if not isinstance(root, dict): + root = {} + dictionary[path_element] = root + dictionary = root + dictionary[path[-1]] = value return output diff --git a/piccolo/utils/objects.py b/piccolo/utils/objects.py new file mode 100644 index 000000000..312bcd362 --- /dev/null +++ b/piccolo/utils/objects.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import typing as t + +from piccolo.columns.column_types import ForeignKey + +if t.TYPE_CHECKING: + from piccolo.table import Table + + +def make_nested_object( + row: t.Dict[str, t.Any], table_class: t.Type[Table] +) -> Table: + """ + Takes a nested dictionary such as this: + + .. code-block:: python + + row = { + 'id': 1, + 'name': 'Pythonistas', + 'manager': {'id': 1, 'name': 'Guido'} + } + + And returns a ``Table`` instance, with nested table instances for related + tables. + + For example: + + .. code-block:: + + band = make_nested(row, Band) + >>> band + + >>> band.manager + + >>> band.manager.id + 1 + + """ + table_params: t.Dict[str, t.Any] = {} + + for key, value in row.items(): + if isinstance(value, dict): + # This is probably a related table. + fk_column = table_class._meta.get_column_by_name(key) + + if isinstance(fk_column, ForeignKey): + related_table_class = ( + fk_column._foreign_key_meta.resolved_references + ) + table_params[key] = make_nested_object( + value, related_table_class + ) + else: + # The value doesn't belong to a foreign key, so just append it. + table_params[key] = value + + else: + table_params[key] = value + + table_instance = table_class(**table_params) + table_instance._exists_in_db = True + return table_instance diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index 8738bd461..9682e888d 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -78,7 +78,7 @@ def array_default_integer(): def array_default_varchar(): - return ['x', 'y', 'z'] + return ["x", "y", "z"] @postgres_only diff --git a/tests/apps/migrations/commands/test_forwards_backwards.py b/tests/apps/migrations/commands/test_forwards_backwards.py index dd5c8e5b3..fa205eaf9 100644 --- a/tests/apps/migrations/commands/test_forwards_backwards.py +++ b/tests/apps/migrations/commands/test_forwards_backwards.py @@ -179,6 +179,7 @@ def test_forwards_fake(self): "2020-12-17T18:44:39", "2020-12-17T18:44:44", "2021-07-25T22:38:48:009306", + "2021-09-06T13:58:23:024723", ], ) diff --git a/tests/columns/test_foreignkey.py b/tests/columns/test_foreignkey.py index 21efc491f..ac26c24b4 100644 --- a/tests/columns/test_foreignkey.py +++ b/tests/columns/test_foreignkey.py @@ -3,7 +3,7 @@ from piccolo.columns import Column, ForeignKey, LazyTableReference, Varchar from piccolo.table import Table -from tests.example_app.tables import Band, Concert, Manager +from tests.example_app.tables import Band, Concert, Manager, Ticket class Manager1(Table, tablename="manager"): @@ -217,3 +217,62 @@ def test_all_columns_exclude(self): Band.manager.all_columns(exclude=[Band.manager.id]), [Band.manager.name], ) + + +class TestAllRelated(TestCase): + def test_all_related(self): + """ + Make sure you can retrieve all foreign keys from a related table, + without explicitly specifying them. + """ + all_related = Ticket.concert.all_related() + + self.assertEqual( + all_related, + [ + Ticket.concert.band_1, + Ticket.concert.band_2, + Ticket.concert.venue, + ], + ) + + # Make sure the call chains are also correct. + self.assertEqual( + all_related[0]._meta.call_chain, + Ticket.concert.band_1._meta.call_chain, + ) + self.assertEqual( + all_related[1]._meta.call_chain, + Ticket.concert.band_2._meta.call_chain, + ) + self.assertEqual( + all_related[2]._meta.call_chain, + Ticket.concert.venue._meta.call_chain, + ) + + def test_all_related_deep(self): + """ + Make sure ``all_related`` works when the joins are several layers deep. + """ + all_related = Ticket.concert.band_1.all_related() + self.assertEqual(all_related, [Ticket.concert.band_1.manager]) + + # Make sure the call chains are also correct. + self.assertEqual( + all_related[0]._meta.call_chain, + Ticket.concert.band_1.manager._meta.call_chain, + ) + + def test_all_related_exclude(self): + """ + Make sure you can exclude some columns. + """ + self.assertEqual( + Ticket.concert.all_related(exclude=["venue"]), + [Ticket.concert.band_1, Ticket.concert.band_2], + ) + + self.assertEqual( + Ticket.concert.all_related(exclude=[Ticket.concert.venue]), + [Ticket.concert.band_1, Ticket.concert.band_2], + ) diff --git a/tests/conftest.py b/tests/conftest.py index fe425df69..9ee3ff0c6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,11 +7,11 @@ async def drop_tables(): for table in [ - "venue", + "ticket", "concert", + "venue", "band", "manager", - "ticket", "poster", "migration", "musician", diff --git a/tests/example_app/piccolo_migrations/2021-09-06T13-58-23-024723.py b/tests/example_app/piccolo_migrations/2021-09-06T13-58-23-024723.py new file mode 100644 index 000000000..f20846128 --- /dev/null +++ b/tests/example_app/piccolo_migrations/2021-09-06T13-58-23-024723.py @@ -0,0 +1,48 @@ +from piccolo.apps.migrations.auto import MigrationManager +from piccolo.columns.base import OnDelete, OnUpdate +from piccolo.columns.column_types import ForeignKey, Serial +from piccolo.columns.indexes import IndexMethod +from piccolo.table import Table + + +class Concert(Table, tablename="concert"): + id = Serial( + null=False, + primary_key=True, + unique=False, + index=False, + index_method=IndexMethod.btree, + choices=None, + ) + + +ID = "2021-09-06T13:58:23:024723" +VERSION = "0.43.0" +DESCRIPTION = "" + + +async def forwards(): + manager = MigrationManager( + migration_id=ID, app_name="example_app", description=DESCRIPTION + ) + + manager.add_column( + table_class_name="Ticket", + tablename="ticket", + column_name="concert", + column_class_name="ForeignKey", + column_class=ForeignKey, + params={ + "references": Concert, + "on_delete": OnDelete.cascade, + "on_update": OnUpdate.cascade, + "null": True, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + }, + ) + + return manager diff --git a/tests/example_app/tables.py b/tests/example_app/tables.py index dd16984ba..331fab081 100644 --- a/tests/example_app/tables.py +++ b/tests/example_app/tables.py @@ -46,6 +46,7 @@ class Concert(Table): class Ticket(Table): + concert = ForeignKey(Concert) price = Numeric(digits=(5, 2)) diff --git a/tests/table/test_join.py b/tests/table/test_join.py index f325261fd..2468b0c57 100644 --- a/tests/table/test_join.py +++ b/tests/table/test_join.py @@ -1,6 +1,7 @@ +import decimal from unittest import TestCase -from ..example_app.tables import Band, Concert, Manager, Venue +from tests.example_app.tables import Band, Concert, Manager, Ticket, Venue TABLES = [Manager, Band, Venue, Concert] @@ -15,11 +16,8 @@ def test_create_join(self): class TestJoin(TestCase): - """ - Test instantiating Table instances - """ - tables = [Manager, Band, Venue, Concert] + tables = [Manager, Band, Venue, Concert, Ticket] def setUp(self): for table in self.tables: @@ -42,15 +40,18 @@ def setUp(self): venue = Venue(name="Grand Central", capacity=1000) venue.save().run_sync() - save_query = Concert( - band_1=band_1.id, band_2=band_2.id, venue=venue.id - ).save() - save_query.run_sync() + concert = Concert(band_1=band_1.id, band_2=band_2.id, venue=venue.id) + concert.save().run_sync() + + ticket = Ticket(concert=concert, price=decimal.Decimal(50.0)) + ticket.save().run_sync() def tearDown(self): for table in reversed(self.tables): table.alter().drop_table().run_sync() + ########################################################################### + def test_join(self): select_query = Concert.select( Concert.band_1.name, @@ -76,7 +77,7 @@ def test_join(self): response = select_query.run_sync() self.assertEqual(response, [{"band_1.manager.name": "Guido"}]) - def test_all_columns(self): + def test_select_all_columns(self): """ Make sure you can retrieve all columns from a related table, without explicitly specifying them. @@ -95,7 +96,7 @@ def test_all_columns(self): }, ) - def test_all_columns_deep(self): + def test_select_all_columns_deep(self): """ Make sure that ``all_columns`` can be used several layers deep. """ @@ -122,7 +123,7 @@ def test_all_columns_deep(self): }, ) - def test_all_columns_root(self): + def test_select_all_columns_root(self): """ Make sure that using ``all_columns`` at the root doesn't interfere with using it for referenced tables. @@ -147,7 +148,7 @@ def test_all_columns_root(self): }, ) - def test_all_columns_root_nested(self): + def test_select_all_columns_root_nested(self): """ Make sure that using ``all_columns`` at the root doesn't interfere with using it for referenced tables. @@ -169,7 +170,7 @@ def test_all_columns_root_nested(self): }, ) - def test_all_columns_exclude(self): + def test_select_all_columns_exclude(self): """ Make sure we can get all columns, except the ones we specify. """ @@ -202,3 +203,158 @@ def test_all_columns_exclude(self): "popularity": 1000, }, ) + + ########################################################################### + + def test_objects_nested(self): + """ + Make sure the prefetch argument works correctly for objects. + """ + band = Band.objects(Band.manager).first().run_sync() + self.assertIsInstance(band.manager, Manager) + + def test_objects__all_related__root(self): + """ + Make sure that ``all_related`` works correctly when called from the + root table of the query. + """ + concert = Concert.objects(Concert.all_related()).first().run_sync() + self.assertIsInstance(concert.band_1, Band) + self.assertIsInstance(concert.band_2, Band) + self.assertIsInstance(concert.venue, Venue) + + def test_objects_nested_deep(self): + """ + Make sure that ``prefetch`` works correctly with deeply nested tables. + """ + ticket = ( + Ticket.objects( + Ticket.concert, + Ticket.concert.band_1, + Ticket.concert.band_2, + Ticket.concert.venue, + Ticket.concert.band_1.manager, + Ticket.concert.band_2.manager, + ) + .first() + .run_sync() + ) + + self.assertIsInstance(ticket.concert, Concert) + self.assertIsInstance(ticket.concert.band_1, Band) + self.assertIsInstance(ticket.concert.band_2, Band) + self.assertIsInstance(ticket.concert.venue, Venue) + self.assertIsInstance(ticket.concert.band_1.manager, Manager) + self.assertIsInstance(ticket.concert.band_2.manager, Manager) + + def test_objects__all_related__deep(self): + """ + Make sure that ``all_related`` works correctly when called on a deeply + nested table. + """ + ticket = ( + Ticket.objects( + Ticket.all_related(), + Ticket.concert.all_related(), + Ticket.concert.band_1.all_related(), + Ticket.concert.band_2.all_related(), + ) + .first() + .run_sync() + ) + + self.assertIsInstance(ticket.concert, Concert) + self.assertIsInstance(ticket.concert.band_1, Band) + self.assertIsInstance(ticket.concert.band_2, Band) + self.assertIsInstance(ticket.concert.venue, Venue) + self.assertIsInstance(ticket.concert.band_1.manager, Manager) + self.assertIsInstance(ticket.concert.band_2.manager, Manager) + + def test_objects_prefetch_clause(self): + """ + Make sure that ``prefetch`` clause works correctly. + """ + ticket = ( + Ticket.objects() + .prefetch( + Ticket.all_related(), + Ticket.concert.all_related(), + Ticket.concert.band_1.all_related(), + Ticket.concert.band_2.all_related(), + ) + .first() + .run_sync() + ) + + self.assertIsInstance(ticket.concert, Concert) + self.assertIsInstance(ticket.concert.band_1, Band) + self.assertIsInstance(ticket.concert.band_2, Band) + self.assertIsInstance(ticket.concert.venue, Venue) + self.assertIsInstance(ticket.concert.band_1.manager, Manager) + self.assertIsInstance(ticket.concert.band_2.manager, Manager) + + def test_objects_prefetch_intermediate(self): + """ + Make sure when using ``prefetch`` on a deeply nested table, all of the + intermediate objects are also retrieved properly. + """ + ticket = ( + Ticket.objects() + .prefetch( + Ticket.concert.band_1.manager, + ) + .first() + .run_sync() + ) + + self.assertIsInstance(ticket.price, decimal.Decimal) + self.assertIsInstance(ticket.concert, Concert) + + self.assertIsInstance(ticket.concert.id, int) + self.assertIsInstance(ticket.concert.band_1, Band) + self.assertIsInstance(ticket.concert.band_2, int) + self.assertIsInstance(ticket.concert.venue, int) + + self.assertIsInstance(ticket.concert.band_1.id, int) + self.assertIsInstance(ticket.concert.band_1.name, str) + self.assertIsInstance(ticket.concert.band_1.manager, Manager) + + self.assertIsInstance(ticket.concert.band_1.manager.id, int) + self.assertIsInstance(ticket.concert.band_1.manager.name, str) + + def test_objects_prefetch_multiple_intermediate(self): + """ + Make sure that if we're fetching multiple deeply nested tables, the + intermediate tables are still created correctly. + """ + ticket = ( + Ticket.objects() + .prefetch( + Ticket.concert.band_1.manager, + Ticket.concert.band_2.manager, + ) + .first() + .run_sync() + ) + + self.assertIsInstance(ticket.price, decimal.Decimal) + self.assertIsInstance(ticket.concert, Concert) + + self.assertIsInstance(ticket.concert.id, int) + self.assertIsInstance(ticket.concert.band_1, Band) + self.assertIsInstance(ticket.concert.band_2, Band) + self.assertIsInstance(ticket.concert.venue, int) + + self.assertIsInstance(ticket.concert.band_1.id, int) + self.assertIsInstance(ticket.concert.band_1.name, str) + self.assertIsInstance(ticket.concert.band_1.manager, Manager) + + self.assertIsInstance(ticket.concert.band_1.manager.id, int) + self.assertIsInstance(ticket.concert.band_1.manager.name, str) + + self.assertIsInstance(ticket.concert.band_2.id, int) + self.assertIsInstance(ticket.concert.band_2.name, str) + self.assertIsInstance(ticket.concert.band_2.manager, Manager) + + self.assertIsInstance(ticket.concert.band_2.manager.id, int) + self.assertIsInstance(ticket.concert.band_2.manager.name, str) diff --git a/tests/table/test_objects.py b/tests/table/test_objects.py index 7766f989f..2f602620a 100644 --- a/tests/table/test_objects.py +++ b/tests/table/test_objects.py @@ -1,5 +1,5 @@ -from ..base import DBTestCase, postgres_only, sqlite_only -from ..example_app.tables import Band +from tests.base import DBTestCase, postgres_only, sqlite_only +from tests.example_app.tables import Band, Manager class TestObjects(DBTestCase): @@ -62,6 +62,26 @@ def test_get(self): self.assertTrue(band.name == "Pythonistas") + def test_get__prefetch(self): + self.insert_rows() + + # With prefetch clause + band = ( + Band.objects() + .get((Band.name == "Pythonistas")) + .prefetch(Band.manager) + .run_sync() + ) + self.assertIsInstance(band.manager, Manager) + + # Just passing it straight into objects + band = ( + Band.objects(Band.manager) + .get((Band.name == "Pythonistas")) + .run_sync() + ) + self.assertIsInstance(band.manager, Manager) + def test_get_or_create(self): """ Make sure `get_or_create` works for simple where clauses. @@ -175,3 +195,26 @@ def test_get_or_create_with_joins(self): # We want to make sure the band name isn't 'Excellent manager' by # mistake. self.assertEqual(Band.name, "My new band") + + def test_get_or_create__prefetch(self): + """ + Make sure that that `get_or_create` works with the `prefetch` clause. + """ + self.insert_rows() + + # With prefetch clause + band = ( + Band.objects() + .get_or_create((Band.name == "Pythonistas")) + .prefetch(Band.manager) + .run_sync() + ) + self.assertIsInstance(band.manager, Manager) + + # Just passing it straight into objects + band = ( + Band.objects(Band.manager) + .get_or_create((Band.name == "Pythonistas")) + .run_sync() + ) + self.assertIsInstance(band.manager, Manager) diff --git a/tests/testing/test_model_builder.py b/tests/testing/test_model_builder.py index 83b797b40..bb13b534f 100644 --- a/tests/testing/test_model_builder.py +++ b/tests/testing/test_model_builder.py @@ -5,11 +5,13 @@ from ..example_app.tables import ( Band, + Concert, Manager, Poster, RecordingStudio, Shirt, Ticket, + Venue, ) @@ -21,6 +23,8 @@ def setUpClass(cls): Poster.create_table().run_sync() RecordingStudio.create_table().run_sync() Shirt.create_table().run_sync() + Venue.create_table().run_sync() + Concert.create_table().run_sync() Ticket.create_table().run_sync() def test_model_builder_async(self): From ce5ed42b4849306f7d2d13faf86f62ca8f663d11 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 7 Sep 2021 19:37:00 +0100 Subject: [PATCH 046/727] bumped version --- CHANGES | 25 +++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 07eb41b2f..d79e1010b 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,31 @@ Changes ======= +0.44.0 +------ +Added the ability to prefetch related objects. Here's an example: + +.. code-block:: python + + band = await Band.objects(Band.manager).run() + >>> band.manager + + +If a table has a lot of ``ForeignKey`` columns, there's a useful shortcut, +which will return all of the related rows as objects. + +.. code-block:: python + + concert = await Concert.objects(Concert.all_related()).run() + >>> concert.band_1 + + >>> concert.band_2 + + >>> concert.venue + + +Thanks to @wmshort for all the input. + 0.43.0 ------ Migrations containing ``Array``, ``JSON`` and ``JSONB`` columns should be diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 68ec08fd1..2ce827b4d 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.43.0" +__VERSION__ = "0.44.0" From 1f305f2f413d6625f66e2b780e17e12f0d17a620 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 8 Sep 2021 11:20:53 +0100 Subject: [PATCH 047/727] add help section to docs --- docs/src/index.rst | 1 + docs/src/piccolo/help/index.rst | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 docs/src/piccolo/help/index.rst diff --git a/docs/src/index.rst b/docs/src/index.rst index 2614ca6cb..69d1d63d8 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -24,6 +24,7 @@ batteries included. piccolo/ecosystem/index piccolo/contributing/index piccolo/changes/index + piccolo/help/index ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/help/index.rst b/docs/src/piccolo/help/index.rst new file mode 100644 index 000000000..29669a097 --- /dev/null +++ b/docs/src/piccolo/help/index.rst @@ -0,0 +1,5 @@ +Help +==== + +If you have any questions then the best place to ask them is the +`discussions section on our GitHub page `_. From 1daccfa7d042cf5c3396a14ab4ece812b1045544 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 8 Sep 2021 12:36:28 +0100 Subject: [PATCH 048/727] add `select_related`, `get`, and `get_or_create` comparisons (#223) --- .../piccolo/query_types/django_comparison.rst | 76 ++++++++++++++++++- 1 file changed, 73 insertions(+), 3 deletions(-) diff --git a/docs/src/piccolo/query_types/django_comparison.rst b/docs/src/piccolo/query_types/django_comparison.rst index be30c80be..7f5b3df69 100644 --- a/docs/src/piccolo/query_types/django_comparison.rst +++ b/docs/src/piccolo/query_types/django_comparison.rst @@ -6,9 +6,51 @@ Django Comparison Here are some common queries, showing how they're done in Django vs Piccolo. All of the Piccolo examples can also be run :ref:`asynchronously`. +------------------------------------------------------------------------------- + Queries ------- +get +~~~ + +They are very similar, except Django raises an ``ObjectDoesNotExist`` exception +if no match is found, whilst Piccolo returns ``None``. + +.. code-block:: python + + # Django + >>> Band.objects.get(name="Pythonistas") + + >>> Band.objects.get(name="DOESN'T EXIST") # ObjectDoesNotExist! + + # Piccolo + >>> Band.objects().get(Band.name == 'Pythonistas').run_sync() + + >>> Band.objects().get(Band.name == "DOESN'T EXIST").run_sync() + None + + +get_or_create +~~~~~~~~~~~~~ + +.. code-block:: python + + # Django + band, created = Band.objects.get_or_create(name="Pythonistas") + >>> band + + >>> created + True + + # Piccolo + >>> band = Band.objects().get_or_create(Band.name == 'Pythonistas').run_sync() + >>> band + + >>> band._was_created + True + + create ~~~~~~ @@ -39,7 +81,7 @@ update >>> band.save() # Piccolo - >>> band = Band.objects().where(Band.name == 'Pythonistas').first().run_sync() + >>> band = Band.objects().get(Band.name == 'Pythonistas').run_sync() >>> band >>> band.name = "Amazing Band" @@ -57,7 +99,7 @@ Individual rows: >>> band.delete() # Piccolo - >>> band = Band.objects().where(Band.name == 'Pythonistas').first().run_sync() + >>> band = Band.objects().get(Band.name == 'Pythonistas').run_sync() >>> band.remove().run_sync() In bulk: @@ -68,7 +110,7 @@ In bulk: >>> Band.objects.filter(popularity__lt=1000).delete() # Piccolo - >>> Band.delete().where(Band.popularity < 1000).delete().run_sync() + >>> Band.delete().where(Band.popularity < 1000).run_sync() filter ~~~~~~ @@ -108,6 +150,34 @@ With ``flat=True``: >>> Band.select(Band.name).output(as_list=True).run_sync() ['Pythonistas', 'Rustaceans'] +select_related +~~~~~~~~~~~~~~ + +Django has an optimisation called ``select_related`` which reduces the number +of SQL queries required when accessing related objects. + +.. code-block:: python + + # Django + band = Band.objects.get(name='Pythonistas') + >>> band.manager # This triggers another db query + + + # Django, with select_related + band = Band.objects.select_related('manager').get(name='Pythonistas') + >>> band.manager # Manager is pre-cached, so there's no extra db query + + +Piccolo has something similar: + +.. code-block:: python + + # Piccolo + band = Band.objects(Band.manager).get(name='Pythonistas') + >>> band.manager + + + ------------------------------------------------------------------------------- Database Settings From cad6447f80b1dad1dc7b1bf564e4b1fbcd544279 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 8 Sep 2021 12:45:27 +0100 Subject: [PATCH 049/727] improvements to `to_dict` to support nested objects (#222) --- piccolo/table.py | 24 +++++++++++++-- tests/table/instance/test_to_dict.py | 46 +++++++++++++++++++++++++--- 2 files changed, 62 insertions(+), 8 deletions(-) diff --git a/piccolo/table.py b/piccolo/table.py index ce66c55c2..d20e4a1cc 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -430,16 +430,34 @@ def to_dict(self, *columns: Column) -> t.Dict[str, t.Any]: {'id': 1, 'title': 'Guido'} """ + # Make sure we're only looking at columns for the current table. If + # someone passes in a column for a sub table (for example + # `Band.manager.name`), we need to add `Band.manager` so the nested + # value appears in the output. + filtered_columns = [] + for column in columns: + if column._meta.table == self.__class__: + filtered_columns.append(column) + else: + for parent_column in column._meta.call_chain: + if parent_column._meta.table == self.__class__: + filtered_columns.append(parent_column) + break + alias_names = { column._meta.name: getattr(column, "alias", None) - for column in columns + for column in filtered_columns } output = {} - for column in columns or self._meta.columns: + for column in filtered_columns if columns else self._meta.columns: + value = getattr(self, column._meta.name) + if isinstance(value, Table): + value = value.to_dict(*columns) + output[ alias_names.get(column._meta.name) or column._meta.name - ] = getattr(self, column._meta.name) + ] = value return output def __setitem__(self, key: str, value: t.Any): diff --git a/tests/table/instance/test_to_dict.py b/tests/table/instance/test_to_dict.py index 9a597c909..cbafa8812 100644 --- a/tests/table/instance/test_to_dict.py +++ b/tests/table/instance/test_to_dict.py @@ -1,5 +1,5 @@ from tests.base import DBTestCase -from tests.example_app.tables import Manager +from tests.example_app.tables import Band, Manager class TestToDict(DBTestCase): @@ -11,7 +11,26 @@ def test_to_dict(self): instance = Manager.objects().first().run_sync() dictionary = instance.to_dict() - self.assertEqual(dictionary, {"id": 1, "name": "Guido"}) + self.assertDictEqual(dictionary, {"id": 1, "name": "Guido"}) + + def test_nested(self): + """ + Make sure that `to_dict` works correctly, when the object contains + nested objects. + """ + self.insert_row() + + instance = Band.objects(Band.manager).first().run_sync() + dictionary = instance.to_dict() + self.assertDictEqual( + dictionary, + { + "id": 1, + "name": "Pythonistas", + "manager": {"id": 1, "name": "Guido"}, + "popularity": 1000, + }, + ) def test_filter_rows(self): """ @@ -21,9 +40,26 @@ def test_filter_rows(self): instance = Manager.objects().first().run_sync() dictionary = instance.to_dict(Manager.name) - self.assertEqual(dictionary, {"name": "Guido"}) + self.assertDictEqual(dictionary, {"name": "Guido"}) + + def test_nested_filter(self): + """ + Make sure that `to_dict` works correctly with nested objects and + filtering. + """ + self.insert_row() + + instance = Band.objects(Band.manager).first().run_sync() + dictionary = instance.to_dict(Band.name, Band.manager.id) + self.assertDictEqual( + dictionary, + { + "name": "Pythonistas", + "manager": {"id": 1}, + }, + ) - def test_to_dict_aliases(self): + def test_aliases(self): """ Make sure that `to_dict` works correctly with aliases. """ @@ -33,4 +69,4 @@ def test_to_dict_aliases(self): dictionary = instance.to_dict( Manager.id, Manager.name.as_alias("title") ) - self.assertEqual(dictionary, {"id": 1, "title": "Guido"}) + self.assertDictEqual(dictionary, {"id": 1, "title": "Guido"}) From 3d35c2b3f57c3d1e7178fd2bb987fb4254587415 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 8 Sep 2021 12:49:31 +0100 Subject: [PATCH 050/727] bumped version --- CHANGES | 21 +++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index d79e1010b..53c4743f3 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,27 @@ Changes ======= +0.44.1 +------ +Updated ``to_dict`` so it works with nested objects, as introduced by the +``prefech`` functionality. + +For example: + +.. code-block:: python + + band = Band.objects(Band.manager).first().run_sync() + + >>> band.to_dict() + {'id': 1, 'name': 'Pythonistas', 'manager': {'id': 1, 'name': 'Guido'}} + +It also works with filtering: + +.. code-block:: python + + >>> band.to_dict(Band.name, Band.manager.name) + {'name': 'Pythonistas', 'manager': {'name': 'Guido'}} + 0.44.0 ------ Added the ability to prefetch related objects. Here's an example: diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 2ce827b4d..0e1d75596 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.44.0" +__VERSION__ = "0.44.1" From 621c2aab235c5735381d1e938e8c61edd90fc6bd Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 8 Sep 2021 12:50:49 +0100 Subject: [PATCH 051/727] fix typo in CHANGES --- CHANGES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 53c4743f3..9ae2a2e3f 100644 --- a/CHANGES +++ b/CHANGES @@ -4,7 +4,7 @@ Changes 0.44.1 ------ Updated ``to_dict`` so it works with nested objects, as introduced by the -``prefech`` functionality. +``prefetch`` functionality. For example: From 20486977bc978ad2327782014cc97e16f5443a67 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 8 Sep 2021 17:10:32 +0100 Subject: [PATCH 052/727] ASGI template MyPy fixes (#224) * add __init__.py file to asgi template * fix mypy warning with `create_pydantic_model` --- piccolo/apps/asgi/commands/new.py | 2 +- .../asgi/commands/templates/app/_blacksheep_app.py.jinja | 8 ++++---- .../asgi/commands/templates/app/_fastapi_app.py.jinja | 7 +++++-- .../asgi/commands/templates/app/home/__init__.py.jinja | 0 4 files changed, 10 insertions(+), 7 deletions(-) create mode 100644 piccolo/apps/asgi/commands/templates/app/home/__init__.py.jinja diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index 9e70346ef..7fdd20da0 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -75,7 +75,7 @@ def new(root: str = ".", name: str = "piccolo_project"): os.mkdir(sub_dir_path) for file_name in file_names: - if file_name.startswith("_"): + if file_name.startswith("_") and file_name != "__init__.py.jinja": continue extension = file_name.rsplit(".")[0] diff --git a/piccolo/apps/asgi/commands/templates/app/_blacksheep_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_blacksheep_app.py.jinja index 2d668f595..fd4db076f 100644 --- a/piccolo/apps/asgi/commands/templates/app/_blacksheep_app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/_blacksheep_app.py.jinja @@ -11,7 +11,7 @@ from blacksheep.server.openapi.v3 import OpenAPIHandler from openapidocs.v3 import Info from home.endpoints import home -from home.piccolo_app import APP_CONFIG +from home.piccolo_app import APP_CONFIG from home.tables import Task @@ -36,11 +36,11 @@ app.serve_files("static", root_path="/static") app.router.add_get("/", home) -TaskModelIn = create_pydantic_model(table=Task, model_name="TaskModelIn") -TaskModelOut = create_pydantic_model( +TaskModelIn: t.Any = create_pydantic_model(table=Task, model_name="TaskModelIn") +TaskModelOut: t.Any = create_pydantic_model( table=Task, include_default_columns=True, model_name="TaskModelOut" ) -TaskModelPartial = create_pydantic_model( +TaskModelPartial: t.Any = create_pydantic_model( table=Task, model_name="TaskModelPartial", all_optional=True ) diff --git a/piccolo/apps/asgi/commands/templates/app/_fastapi_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_fastapi_app.py.jinja index 4053c7cde..53b8dc20e 100644 --- a/piccolo/apps/asgi/commands/templates/app/_fastapi_app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/_fastapi_app.py.jinja @@ -29,8 +29,11 @@ app = FastAPI( ) -TaskModelIn = create_pydantic_model(table=Task, model_name='TaskModelIn') -TaskModelOut = create_pydantic_model( +TaskModelIn: t.Any = create_pydantic_model( + table=Task, + model_name='TaskModelIn' +) +TaskModelOut: t.Any = create_pydantic_model( table=Task, include_default_columns=True, model_name='TaskModelOut' diff --git a/piccolo/apps/asgi/commands/templates/app/home/__init__.py.jinja b/piccolo/apps/asgi/commands/templates/app/home/__init__.py.jinja new file mode 100644 index 000000000..e69de29bb From 14ad250558700140c4b17706571f88ecbce85ca9 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 9 Sep 2021 12:03:09 +0100 Subject: [PATCH 053/727] Schema visualise (#225) * basic prototype * render data from apps * allow the user to specify which apps to output * make direction optional * more aliases * added docs * fix lgtm warning * be able to write output to file * don't print out database version error if the proper version can't be retrieved * use warnings instead of print, to prevent polluting output when piping to clipboard or file * prevent more stdout pollution * add example image * add tests * use a random filename for the test * tweak test - mocks have a slightly different API on Python 3.7 --- .../images/schema_graph_output.png | Bin 0 -> 127048 bytes .../projects_and_apps/included_apps.rst | 49 ++++++++ piccolo/apps/schema/commands/graph.py | 113 ++++++++++++++++++ .../commands/templates/graphviz.dot.jinja | 53 ++++++++ piccolo/apps/schema/piccolo_app.py | 9 +- piccolo/engine/base.py | 2 +- piccolo/engine/postgres.py | 21 ++-- piccolo/main.py | 25 ++-- tests/apps/schema/commands/test_graph.py | 47 ++++++++ 9 files changed, 292 insertions(+), 27 deletions(-) create mode 100644 docs/src/piccolo/projects_and_apps/images/schema_graph_output.png create mode 100644 piccolo/apps/schema/commands/graph.py create mode 100644 piccolo/apps/schema/commands/templates/graphviz.dot.jinja create mode 100644 tests/apps/schema/commands/test_graph.py diff --git a/docs/src/piccolo/projects_and_apps/images/schema_graph_output.png b/docs/src/piccolo/projects_and_apps/images/schema_graph_output.png new file mode 100644 index 0000000000000000000000000000000000000000..6bd8deb49a2e75cc5cf5f8f8f473b742049ba015 GIT binary patch literal 127048 zcmeFZWk8l$*ETE&DkumDqO?j00xI1YbcqT`Bb_23-L@bh3?Pl7fOK~mba#V*(%sFw zPH-Fnz2AAC@Ba5a^JC_k>%8_p_g;I&v5vJiFK$Q)V__0u9yo9SOH}0g%>xJU+728* z89Z_bz8T8r^??7NnBNq>dLW^OWaPjBiUXq8uiUm)9q+aYma}N*pElG{KhS;~|Hh4D z_~-=q=vI=As``xDAI^Vxt;A=b$f%%LTgy_7CCXB*Oj{`;-o=_(MT&uXTjIuTffgJz zt+0UJF_N(C0J~G_&Udr?msUzShwS~w)(Wtn7W8Y1%E~sM?mH{sd;kR%?FbIVf9t~y z3REgt_y^tWWa)ox(w~dNLCvs7`)@9HduDVh3ia6Vj**N1^_gM02mkAn{_k>ssO$eq z7X|fymEnJt;irfBUuF28C_}hk5&71#F!fI{{`lY$jK4f-RT$)Bao%-*loUw7IMP;; z?h*ZL|FhqJ!M+P#mE5Z!c5ph%9_>)4r} z!SFwSaOuaxNs?9GuvFX~-@TYnQ8Q-W-x{UdH@*xV1w|>l4xRPi!T|v(jHY94L~KTP zq?#j7?S79y_%hnlW2f;vd1gh9w9ysxa9`SoG>USL9%)mw;%3J9mv0OgKsPvKBpJlU z&=b^{CLld}XOE}^5{{BIMX1;h5clFF`Hs2YB;WT^fg0rCvi)eE|NfZvgCVvIfeGh$ z;T}~IPtU;Q|T@$&q+9+(&}4W`z9(9BE(5buU85`ut%Z(2vy5i5xs4} zUD)%EetfWiUZtB)36)V?zv{~fOjEdmmb+a5wVIuQHzj?3+nF`b@x2TZzx(pb-ybLH(lHP%=H$$ z!*sFx!+zBz5`sscITZIO`!5UMBScIQ7=XgsD6@M}0%YJ&Dh3A6hwPJpdIUV`7-qP! z@5$~}{xjjXKu}%fvFEn`e1$AXz!VewSxo^q>ps;L_qoO&8Dm{T68S03H$NqhtO_i}s$Ni}DFDHA7OFYk(elcru9&m0&u>`IQI>)#j&D@_brrC|D$fo2q+ z{Ql6ks<<42QO)7OC(rGZG3{konqbTQQmTEA{;&d8B&epo!rf7XqEYlw;w0I8G1=?N zm*%@qaqFu}HDb0^j4aZ6Af|wM1Fl!o6Z>@DfS@Bis;S))|sqr$Wkn_TiMq-0;oF?wRov^sVBcZz)7~AK% z@9}>7`sM=z>_`0+x1Ec?JD zeTst5YgQA{JpO^XQ>z^`uO3@9=V6bwGum3|2ieRo)iY~$bQPOwg?S`bvc*TJjLpt( zHZSqH>z39iQhmLfd+)TJWhYCHQL<>2Y7)!l!pAOa-HP?Bk#8ZXgCrIk(_KanW44c6 z00o>DoijBT>ONT)34!ZKuOBv+_>+&^dl1=l5W?iSBN(^k;25plT4m+)v1QKNxuBO@5PYAr`STm zk6P{R{fqYEhUjM&MM(Fr!#)Y%1mAP+Hx6yfrcBdi70rmWEkZ@vtZ7jpDsQ;JN@G0@ zeIzwBM5#1UYWy@6{cNV(d9(J7f&P&5bB#;s2Qyl}yhgL7`l#C6N~$~WAr-`~)|ggh zhoZC)dD&}QY2wmBbuS`!Gi(#$SdK0JHz<95_=n(Z9)mlrzNv8PcI^VVF zt0hY?+7x>ei|w}$Kgj?rpaAjry^oaTd1-T?-zEmrr}e{y1+dD#%)!oF{Pnu z!)GfLHZ&|*8 zQ>m?rKRwR6oTj3g%fMcq7aJYtO1`D9oX2i_MX_?KW|DM5(N~vLxTqyCXN-4S8}3-a z2FLE{Qtz{y1IW&Q^X+eBfjEyF;&95+4|mS=j+#q4R$CyCoy_{wqpcSTbACD*J#REC zY0sZ8?{eZWzk4AzT4KtoD@L}KQn&4yS>7V)Q_S<%nfDbUAIen+T~`=!osRn^i1%Q@hu_o9Dzd6J=D`*z zyXU}S4_Ag>;{~?!Ol~PDOoQ6B9(O?X+C=C2XNgdh(yyg>cX$+Rxpv=Bp+vR7Ngv zHLM;oVr8Irf5tc(l-{aPNPMDk@InpGi6q%Xxl`A-S4<=UD}E%SxIf^AA3+C^-fbie z*g33U&*%5w$T3lhucZ$>K{DigUrar6u4TF4qETnexBOw_^d^AQi(Q zZI=DI?h**D*8cc!W8?1@AMOX3Fbr*HqND=U1+2E!l}yW*1>J1bM)I-*rl86Z=D8sR-K2Bf1Wes!;w~~%d z(*ZT3W~;-$KQ_NZH=euyQT9xiQmV#~v0|x=-2>$hI)JX0@3wFAY_3v>-KKZB5MTLP z%D-tU5yTdcYng(+N6Ea7kXSUyn0A?#8si^>l;}x!Y>qxtt<9d+L!mDGH3XoXyp-GkbxpJLPoI!Q+r4QW|#dg-(B5ybCKN zIg00A{a_4>h|mC5$G%OWyJt7}+1heEuBu{VX^Nz`J@Zz6yvfGevzet!%&qk5^zGxF zb|$1zWLc8=O9V->CUgh4Jz&g$bGMCM28RW#1AJ}MDspau(ZuIHSbPn-*6pPc#5jhzZ#`k|anDK&A zIbt=&^?9kVq^Co!2^kV&Lp|5;-&-#{-0K$W(R<#qGr@RpWjg}AnbMiGdtgA?x-^xo zkstFs!sJwF`VtCvy4z9gDWd8?Kb=q}$TfR;=I#bt3|rQ(Z-vsNlQ=jmDsTK?XNVBZ z`mCV%LLgnB+ByEk2^yhN57NH9CxLZgx=quw7=tF*D}hcu*2R=T_z(Nu?;To%5lIXO zi0_OrI^ts7k^X#ADg9-6Dq-oe{!z9sCoK}PZDJ!4JF#|J9lB(*_*SU1RcPn@iwF_F ziGfC>iWnY0KcgGxQGeETA-O#Xs32WkVyDhb`RgHw3;J86LBEdz`xj`Y3^MRw7qQu` zZ{Z>c-zXYEnks`nirKc>8PrIWekf?S0=ry2nKk~J6n<#-D8&8WzHwFw-srd_y|Cwv zH;^}q(S)4yM*#ax_NLvO)Dv3JLY&-DiQU8gd=fVhG7K})7kl?SXtRp&e$`Cr8BQcQ z5hjk5m6RYW*_R?iuzhhL#N0UfgI%vxjcTh6qFNpBs^34@`w`V#qD_UGDgs8#(e5M-+`WKr!7x#<;fsfu0<0+zG*!Kr^`2X4>i-Ew2Z7V!5 zw1=V1J_8$KUk!KtgfP2zHXab|5>Xj1a)zxZ2zFb*u1)Xu8!kYNGm!6PwY-iZq2|p| z16mfC@JWlz&@MIm6FIlXBmZgw(XjB!p})2F?;n192y#;%en^CgkT!2DnrwE$zjr4nI|T2M`99U+a~FyhWOQ3wU|9F>5oV6au(4|Ei5bLT_#Q86@l1d+NFvLL$F1gZoXn9n*$fx zM3ChBmt#%0re?*sU@T9u8f46ngDQ>WL#<6e@A3|8+5&txd;V9Q+HEkqC33)p1SxTS zu9hL%t^KrT;pKu&M#9F3M{pN1K8LK5R>MUW-DqPVWUHizhA;Y-PHDViu{+I_@(?%t zdi@gTcA@OsdyrzDQPA6e?Pl*F<|7^z;g>Q;X>r#1Ir)Z_PR~^#8=nTQW$tI+XbsA@jlC&6B z4kufT>8Yv`Avstm88fq={O#gaOT(~$rBNq7_gCaK@u{2FEPF0z7~LWx@RgmZk9%^- zvLTPlP;zOci7T~aBT+(hs<6YZQm;2sfoH2F&ibqS2f_6xT-yg~qzBR=R%~1SqcZPS z9tr_S=IM>?m#-;q^`SDM=kDN{1Kq2>I7-GxcsN-W&>Eaq};gbs{F8a!ff}HWoNF|6C>0*ynWE<69GH z7O`B{^1W^RH}#cUIy%z~aa%*0m=$5*P-rE6tVWrePyZ_?Nwtv;_O*KiQKWgsO|lcZ zNBspmB?EN^dA1gvH0ylGwI+QilDTV!eOV(152X@ba1UEw>3+?oYdSbHwG}08)MHQ_ zxD}8?!E3l6+q%NGE`Rs%wrNmf=YdN3S~1+ZxLbI%A}|qpyWF#mGH?WJ zW(SS7(`UgmH<;C9>CV#8#hhKqH#Ym58u@;Md);8B!4*#5@ANHZ^|5ywT%|A8EEFRK zfDARQFpkjZYjAyFG{M`FR>UAJ`4Y%&U+l5=Y>V~LFQGCwtwQBnZu)ogGxh|eoNXI> z&C6YK`q=}8M$3RD)`J;=e!e9?XvIWJN4&}AEx%U*sV&%TF}y1i<#P=;7zwvm-*y98 z>tAse|JLN41rCA&7#z4RzSk_r7@(GwkHH|IP6dMC!`$|cNqtC51y!zlFbDhMaE`vt zfFXG(PNMr3OTP5gSYTAtSk7;p%{Q1ysd$~Kr;ujSc!9YG?ivfwm>9?CgJ4FA>p zC7;?1Qgsw+eNbvWLI^T>crTIW#yH1QK99}Q*Kew+3FOi>4c^GTG}09-)pp);tbr|6 z7mtWdd-1Ja%R82iOcfd3%j&CLRHlO=5zM-FUq2Pp!@!fPz&u39#kkH=x7Yey0jV7d z%P=&`{7$W4?vsBjQ_+}&@2-?GuaW)g14$I69U=v@tz)_nZyfO^{YWh?FW<^+cO2M@ zo&J$i*+CaDi>**q&n*%QvS9HSdyQ-pgQP&|*>VYc0WQ+m@|VAVsW+AQ>=X_iwbwNy zxk8e-OzGWUX3yx*^swvzs$(<)^M9|T0d!S$ZksNgW= zbLXlT@*hU%6&H6PA`J(Y1km@$^3Nee-4_+M%4In&vO^&Mb3PPF_2B)CS!{_3!y$m7 z8pWJL!V0wXK)H`=BQs*a4<%QqQb$`qUNozj{oeZ7W23E6lXxoJX;r~}2#1NaVJ8$j zD0#Uy=3s_5MVxVGdKr(Jssh0YUHi~%{00 zWKe$=v6FZ1Rz`@O=-ozaKy>;Rp1nm|f|AZ?)<(N$e|Dx$v6@p8drU;^pnbswschrh zJj}F%nuK%?VuQn(&wJwlS_FV zA7lzqwmA#|Q^2QmlbhT-rJH{tyKh+_91;_++1;>C;1$xbk@A##5-HiIU$J43+LrZA zL}>hLl};L?P?cS3&((g+w>k9|A*yHcnM{Ig7VCl<`+Ep!imwqaxJPPR95z}QZJ59b483Giwb-RDm9`$DPNuC|>wLr*32pykCBai$Pskyhxd7MKFNM% z_7tDJ_27NG@nm5Zlb*bb$H#7?x!wi@{fez8nf-@3$!td(FDf9F$NZvbN>Vj-i$&Kg zJWec)-HvL3B1%6vqR1(-{0E75a1#k+NRyhKwrXZV98!nW7V4I*=QcP;@;b)YuNTP; zSu*r5Hri=7udjEeYSj6hXw)U`U~|i@>0uai00mh9Z{{f#&pg!=aG9Sm*Gajhjk|W7 zRLMA_KddjzX{`{7B**&+OQ-O2OG}oTYT8O^L3}V9_T>;PHa9jH&pA%X567QNV%{y>bvpGsny`hBMh7} zPS9P>Ry_3Yzcx)$n4lRzmbtogd|*mbr-b^u_NDnxys4j3V~$%4&5@UCo{Ngggf!@9 z`H90mM_nO!HAp64Ew!M5%A+Gfa}cRE6{J-9nfBqTFdNDVeo1C8I9*Ba6L!{exb5AQd3G5m(BOUu+GxAS>risPL89s(4cjVaV%al^w z6KPeCIL!yYLHcq<}KjiPsdOL}Gh=(L`W@R6v zCWw$Us>R0V>jzB}MO;P1gZZMyp6zdycEs5VTvh-ywZ@^pRin_Sg0ym+FgYDCX;E!u z`z4_>;Ybu2G%G3d&38+X{wP>ibW_N$CP2gLfzCk@RhLWq3zh*BgxV{L9cN8uu-~6=Y>fw z!_4S_B^4P30_g6=-C9+@_n@)xAC#{;<{tx?GjIyl_~)yLbC>vOH9=Ckr`7hErCmc%U%Lxbn{%KK8= zo@4q+(NLHZnM$sudjPX;f}d95#-bacs`Z(Q%1YwD?u|6OPp)R`)LPv7<|C~!7N4ma z%L^GHf&WLe*_-eZ)eOn{T}(GEtxGey5h>)d%O6{FbV8Q8r3XvjI}!QYWWaIcTeV$a z+|pUHRiZj0FWM$+7H?Q2=hNVAJ>2i8@KME(?(3SB`wIXe!Pvysvy1^aajeziJ@SW* zsQ8z93(f6yd!-fvPKr{rK{E+~o4?$Vwiy4{R}ie1bOyA;SVYBeyJo&S75jYvc3A*s zJw4Pp@G<|d;V`aSO9h2M(-4 zL$#p+Hgifb^6M_H2LENzC~Ah7*jXg|q5LS@+$wwA-IKjH%qLq76#KHe@xfTBJZ-HD zck=8*z{q)9sfX!KJB{`7zby1zIa`(s<#df{XO##W8-@H;C1!Q{fN8e@E7$H!!`zN+ zSy^0mo<+yJ^|1Dltlll@ur}b~s)FyDNV(am2H#HHz}jg)!vdZ+_6&afZ>qYx(A__< znhoi-#zHexKqVGc|MVDR1_Z#_ou8M-!&viFE6}CxjaPUtZLw>eg}$hd=87`sjEKo!!?Wmsc+( z6PRl_&XHMvb$QU!TCkYT@%^t`{ZEPSJ=d=Edp}jnH5NDAmaM6>Io zM#TxTxHHSqBgrO9)e$POvsG=t4Y_YEeYFc%^Q*4t&)MWI%}`1!i!q)Z7fc?ZmW+T7 z*?{EQ0#^EaHU=7g!i5v;+l5SHgO}omEsS040 zN9CHUVjjroyyq4v85B7sX0ZO@Yl2E{q`43MX{$y_cEKPv(4zitM9ex8M4{6bo2szCwTu6*_ zMH-?i#N;1ac0=4i=r{|;fc3E_kj%Lv?Icc-HJ-o+Cg>YZZOgbw2y`r6rp>b$K_J2TYq}gyUyUMF_dOc3bs2xDWhYb)UUOvC_K=8h$Qz*++`& zd=2MNa2xz0A}#H~`$tr76h*V-EN;lD&=fATC=GO(*Gj52A8U)@WNRM)6OKi^3!tJQ z-KSQQU7V2;0p97&V~K@ic2oIQMA(Y*BJz%w*MZ$_f=jNIR#(FNAc1BLe($@p4Hwp< z&@kh^Rx@@23g{(BnvVdHkS1tmN#J}VJf0+587IhiYF17nIQxF}QQ{qMWQJT9z-AZL z|Ht|KU&I2vWhTx7cQNAV)D5>bmOSOv99VgcmL|K;VRUsErB?TYud?ZKT$?zJ24%7Z zUJk77fDtYJHM4NT*nVrRS8YTV&%pQb{bQ#tQYfYy_}XmwgsoFLF2t(6Uz+I5^gMIz z>Gi3e{DH`zd#)}A4@cH2W!@tNFXRDgn1G+;i)KBMQpuP+mXKR=KP z9-B+n(_de+o2ZT_ea3!;W>3}1+Nyj)lmV(zt!S8Pu+hGyHVk6uaF82N=-i1uIM1`T zy~9L@4lPhJeiss7>PU)3yrWdo8`Kr=pYQc3*@0@asxYi4vkv9|BA%ZfJ=_bai<()5 z#GgXy@G;TRH?wK$$)lM>Mc-@j@bh9;k8oZEjq7t+v@#F79+MZ-^_KgA+GNvLLT zDiO?K`hLJ_qJwpFwY9KhGIuJpfE$`l2EO~NNjVkFR?s&(*L(})wz@~urr?mjwYgRw zuap_*9Dn7}p<;AGWv|nzG56yrug9jOTqgBj@6qb}^dwpR<3o|j07M3IaV4}{<{W3!K&Q03TpS+-2P?G$k z=Uo|mxbGP0Z?0(lAz{Tw8fD+!EBs6tEttcsXsSqx0Dek+U9=Cz&G_Fx*y|ox? z39yO~_Vnn9|6yc}PwgG;obQG;Z@!e8sB^>_E#;iN43bMyYYDAC&7eqk% zk-+iif3l)yEtm8E?giK@6R$$Z31%~`=xA>EBhnyuS~oSy?6wT(1I6XuS%NjXACp=r zr@exL0`)P&$*yeEg=YCuN%ro)Y{J&<0qmEwD4D1*CEfk}R1nYg*3FcD&pM>(@3jA8 z^X5EnuS5v-?_CtjsnlatjW0eIxB@Pj?l~LRbr1MZ@k)sH(e|qF0|Wpt4mZD-9B(&n z98M{wzBE$krqT4wTytJ{4hgKU^f?$2d&Wnus9l{(n%i9r`ZG2ZP(|vBV~giQi^NbZ zAhv(R4>C#-)fw;IT+HNHY}0I9LqB@*`dAC(z(`7&Z?{DKqXPmzd>cl`E2dWjkJoTd zy=1QCVKxtYY+^CmbY7<>IDn6P<(rRYnb(8Yfm5PIEdU{j3sej5b?d3lq``Ez}yhsFx{qpp1YoeO1 z8fl8p`UQ1*Pd=9Scz2{>TVoh{TrR&W->ZLy^r$1LKyFXZ#ct8;62cgSC&T<@32HBM zqz1WK%;bo>wN|(l^kM0pkCdw)gHlP|n`_Kxh{`KvB7}+&zh*vk6K6+r;2PYcLs+il zo72w}+#KeDdS7N3cb51_^95jhlQ`7nw6$T1C_CtRKQwk#SBat~Bjf}D0l`e!InQ_I z-mGNy(;sQ+>D{39y5sIM(#w6y<h4{*>3|76Uvho(4WeuXpx*Ai2MKoNfdY3C?s0sRi}cTWzT3LLWy3r?CgHOf zO-L%;GTxpVp<5fO@aD8dA4c1(=nfJ5>11?V+{@U+0xo*%)PxhQ$Ew77!c0p}n=W-* zCE^W1b|}@$Zrt%89#WM1ajCU@+>?28HO>79PN0?!y#P3iAHi90RkWUT z+`G5;^5^4yAO0!#-Z|`o_*-o5@uUYjV4#{kc{TyE-U6i^#1G=pVDgzDTOzyX&d07#ywRi{I z#KnZHe%Dj{j`o8#cpesm0lFnNLm~b)ayM^!udJ*Lg?V1X=A<;|ciLRnYmAf}ELdG4 zJ4(p-GHhd9_tO2ZMRDmq;feH#d6(zLK|Im5jvph&JBVL=*_o~EYZ z?LNF{q-THXH{J5PPx7ZCd`p69NbY$w95JGBp={F_ZTf%&UcH8Q<%c?oQ+6yvpx_8J zyE0&X9|yxtGio<;*QU99C(M3U)F^+UWr3)$1l{o1ofIum`}um|`k$X`C}dR4;~;ZdpRt}uZ`G*y z>wGe>ZKda^43h=O;NQ?C}zuL^0qgd<2{jS>Kt90iBErwWU+x#<(#KCW|E9Gu=~;?18L? z88;cG3b!2B7ZRN$4~8Mx!`fu735o6S8w7*tb!GsA^ATOa+>cQu@X$4CJ4%!WaOfVI z9-P#WpT0WHZnrp|HIdQz7E;3rkGIP z)}Kcln7u>@r^O$lLIV@z*K+1+xCg43YpZyqP;(II!2u|Mc*E8*%$4uteW{? z)r&?cZU!=&Z?AkyET?*Sd+g_`Sy*CMZ9$v+J0a4ErO$&`#~Kp5O74RbKT3~rn)|Es zK`00&@^8*~))zW$&Cfi>IB~X}DcifygPfmlV}-DAS#W(^m%lT-*EoTZ@s~?kf14~V za;5D_gXqxjxtQJRj{L^uA=3BgWo$vaJDPcALdcR!OW;zcx0$U}+q{g;GhK8F$?jrU zP5Zx86;V!)zSi`30jI?@mtfLcpr0gbCK%Qet-pSwqx;GBBpsN;%*k?IEXzuC- z1t)>UkOTr@pZ9J#=e2}Dxq>Iye;>_{6TuV%O?0)C1GX9_F|8LQ)K;Z@a|evn-qvaK zLsW^^nQND)wljlJYYkgy)L_+bpd^Fp1k1QVka5Nf_$7~npE9Ch)$F}I8=N+qV0ye8 z*>F@CMYi9470BQTak5(K1;)RKNMe)tvR&%Ekln zd=9H$%3hsoEG>j-20k5Sl4uR)u`x?C?qp-$q~Tu?U58N>Nb_*-=1BygHC|%Ysd~OR z(WzT9l{MgT$#H!doHc!aOl$)DEC=?lS>%j@zF?@Sf_+ZiP^GDAS92r^>H?$)vQG5p ze#do0M-@>A8F_CK;;Zb%6)x$x97|p10|>eeI$wfEiwUYOSg-SJPZPDxIRG7kvB1Tv zI&CaOuaF-{@<8g>F^jn)jZvj?O2)p7mjNzT)w1$C169Iufqumh&x6&VDLVSJh%A)C zz>h|LRAx;)c%0{=s^iM{!Y?_+DKD{7)&Tfw~F z+AU3vy#8hcmoRMRx=o_5q8l00Dva$`EEguo?^p2taa%tl5RTq`kOG#J!Ij&85krFdl0iTA|5$_1>&m6AeB6H)3>MU zmO^|8J)|#*1zh#$E=lE}1aTaw~4zn(=p4_7VXqe((y~PuashZE5%;?w`Y0yHa)Jm*PkAGt=*OJ0(v{!5LCt`ii|jDTeSy50 z65@KG)HnuTYrchKwLV3=G7I8?CInsu7x}1QGqElR&8DD3CEETfezF#j#!2%o+$J&o zf;*B`;ILNGV>d-Q(UI;Z>O)=3yVT`#NV6JBH4cTPWOiSoWg_}r4wF3}@Ze@NoXA1V z$Y|FIN-KCbd6Ge~>3mzZ`Oqm&vv(l22T&dO&cyY3w-FU7BDYG!FYftz2fo}%Meawy zTP}oLbnJM@D$TMN>Ckeyv~PtFtSycPhAJV~b$VUKkWd?X& zk_vnlJH&i1KD8j{cqHe8c%U&@6`)VsYE`(_UDbgw{6yoj(8`rlnAu}%xH@HIs0t|~ zDMXZ+emADbD=JYTmK_EYG!n0Ix^u0hOK;1`35kk|Ml8_Dv*RWC#)io3v|Rli5b=>q z=vPPHng3%$e?P)0JL3W(zvtNh$ht3NYw(8ecH7UdLDTF?xI*#za;NlhadEhwe3Vq? z8Y5CQDo%+0K_dM<8GfRG<{=}fbsC$NQ~7^+E9f?gRe&5ZFb)H7Unp)V*n+eT+5?&+ zMcWz@f%1~zgxp1cxCQ9)R!;S?-mOim93e9YpfB`k?<_1#e?DHy4p}PDI&Mq_9QSfY zBDZlzfNp5{+Ye;Dpc}8jaj#CU&((TTz8o7<+p^^Wj*?!68bu8fX0b51Hn$>vA}Kwg zaJ6br+sFdaT+2@r@sD-gzP2%+U@*x!(R;HBLm=|5h>i@SvJ7=dD@j{3-`coOJ2gfY5x1pL zO9m*((!gS;iF8=JBowg-L6Ie z?%h;A+XHU4)kokFq{asKZywzf#2z9INlEb*NrwP(Haoxe+bN0tBh(fp^=kl&04UEy zkZ=vZyZEFh3q zv5VqGn}B2s;XLnVh0I(*Zu7TX(aWV3iz{zFnW6WhF%5KyG-d@1Zai*cwpde32%GtKXoi&L-?+EqeDxj3vu$hksemh#IV|@* zt4}v65Wd0qktbSF(u#(qRx%{3(@`UucOo-Hlkp&$EiDf{Wh1_%E5=X3Gne{T<5Xz z1iX4EDN*pemvTHOlY*|MluzoUuJN{h*S1`&Iv*C0=BfNFJ@ujHzc64C4e)N4c`f>&=De{Xm%elzCw`4_IPcvw$} zE@?cjC9*ct5IvY-MnF4KJ~?s}1r0~wz%TzQ#93ZhGkFJzkWRwZlG7FqOjPM@bC_~8 zfDm)<8g@3SWru>Ak+MOpwePXD1nk~Zrcs9NB-Ff#O?aMq?AfUEam*~ZHOj~+H9J(x zbm`J1MtQ^Df_$jym42UC)p!KF!L!>5%Iy*peb31KARfyvPIl7+slr@2J6KJB7@NdP zqu6z&f1Z~w>K%&G_e%ezHY{^HW=P6vN9K&U~Ek7cKg!Fh?l9 z-UMMQSBPrveY~6oqxkY|0)pMpj zaIe0NyGw_DNO<}&tSIzGqFA9DEsVF{0^mE| zrJj;xS?o%Xnq-wDl~@Fbtx<`!+7|d%Z!p+Mh1vj2G$YOkL6yC4x=anHSJDL#Vegae zQ@n{wq3)(~aNolQdKhN=K6&zmY$U$MQJ?(BtMa`;+0?R^XEQXoWMnB$nO)p%RDBOxKz)N zuB3waH}e((xPRKEGe5=!#WWGAFh1i*l2fN>5kzXbO3Y^D1;tFIDe%Nnv)?O+Ok>C$ zm#;w21WIi0moL{nJzcPI1t<&=<$;C>rT`ZGn=KM_A^9!wO6(!59|qk~l`@RVo07m~ z;gi#!`$~v9R1@M;6T)4YShz{YRJfjkU>^36B`{AaXknBK0T$BrsgcYNQ=%|OwG9<} z)CVN-dEn!i-Qg|L_i`uLKHN!ui-v}#mGCGit~*hUKY-0Rkkf?CZqWz>1H&{XF*=$S z6*7odkS*ScXd*sf*b*0D)CBme zaD92evt@R8Sg(F+gjWSjA&q0>%vWB-8`uI?X*hPa{3Yres(^Y;s9df?vH2Cip~A?O zvNDnLY6W?@Id4H>r=5$MKHMUGXvU2zJx44x#JVN^sdi>A<^+(^Z1xA@n4 zasNn;rF-2u<&c+C0)Sx5%bmzS8p%fl(LkR0U zsIH70ia7gI34VHD^%Jl89CBFhb)CXN!x1?vP;@dEZQrln$wTm;7n7+C19h7nim&bZ z$5z)x)hqoNZYHYo8dd-;`xlgRIdO)uRv}YL^Bci(xfLlH!g(Ok=k1}q`LpBD7f?2D+Ufd<4)DxP!}fD0-{$x6TI%#Q06u!Zf5CE_ov zD}e8E>RGKK{bzVD8l7Am?WbUJJ!dO_d&1uj}mDnkQPe!9eGJu={$FWdzWD&97N zkCKRUU%p6AL*p$z?G1s=%{3GM+PN1rpUzIKu*tUW0JQ5fyErO-A3mVc?J99 zKhQ$dj0QIV>-iuI!`}XCtWrvokWt+W^gPd}4yj`=SAsjouj$gk%=z!zKf_!79#~UGcBAu5V?C3v^ zP?>2Oa+Jy-lY)`y71ehlxMLPj_NgAq^~)L%pvp;PkY&HaEnaUu2if$XnmOnDR-42R(bt^3s`I5z%yvHeEr>aJe-|!)bw$I+S}_G z*-%RO2GxVVl(WqOxB!T;;OGK3lXCa-p%avK(K6>o>LZ?^p`&vOWSa#i^rEIcKY%)d z3o7Hc$g5XJahlcOw(Zlt+aXZXPzB!lN>t06%c!9^Al?%d^DfRqXxJni6woMjne^%7 z$7NqXJ|g8(E3nVzq_J9_zWYHXH-q!|o%dn{u^wcV=83sO8HBvs@MAetAdJr;ZmPod zBm*6h{pnMu^r(r5Gbt3W|J@5fOb-G01XfbRWvwzC8^}{HJYHV5ZimXmu$7IiR2=MQ zFgF$0->$y(mf`vc!9xDV28b5*bz~ci6R(8HaCxoR&Ay4eaC2jx;)iyvn|aozv7QTK zt!_PTkzp%}QD)yO8X_f$F^(V490k+0Tccv0#Rdf$N6la{P69;QldOFTeV>Q2e(xN5 z#9`#Ue^ncJs0zt1;dug}+DvUnj-l)v@NtZ^XnBbow(|_xS2wc{oY8+#7)opIxF)W% zwd(&&=sNq~>e>xzUU5(9<%|yeQBWm?H&S6ovZU1k7hQfZe529jsVph4ZBvd{Z_60u z=+_Hk?QTxp8?MEf< z_dG+Fo+#P^staz1tf&~kMALwJrI8DNw)REu^v-g(@aitoZyd|?1y9m|rUi;S$%0mC zdE)C_EkBcHsR!$nezYva$?w3#loyHl(orF;2rf#BMjts=&7WWrNIZ``86QR?XuICH~{Ic3HA&8|R4t4NpnyLa!(5hJjmw{oO6i)h$){!WqH z4D_?5mq-?!T)T`qiJ&-VhNW?ZG}k?D%-Ivb(Dw*pKmF|6pcDRPfXTKGTa~_Us~*$L zDgNvTQE$C@@@&_;iIJGGTHnFrMzk)Ro7v?e;=Sda-xKHL*8?7BZ)bJO@Hg?49FtxJ zXhb5}8Os?gim$S$izrI$#IK(!>6HmK)LqCQk#gH`w_2q%%D=*-ucAwnaF-0?&gEGB zXuqe1?WpWG4GbZRY{)cG)6ISJcUJiB%niVV?FAcQ=A)$CmVbdb0>Ty!6+#!*9mBD) z+{qkzg=s&zx`88WN%7DjAE17L0p7cKt)3Y5I2n%~nQ|gK@q0Ozx?3hv>%d@r1CCb2 z+zu+QcNEj~`escd+b= zLL$7nXpNbG_lO&O_G*BMT* z?+hJ#`#jFW>_F1rR)Kan1~sRRbemp2^-4$diGfGQ`9v%ezLQU(^BxmG8jtoxlpU20M+vVsPN;p}WUOxo$ro zWw5=L#W$iTzP31_{qZ6C;1DoUHwvJ;O$$ZIByZ;FF%pg&P+DbJePFycJK)JLhUDZv zc?&4a8F-{HI#ey}Ew9FxBdJ+>^>3l%bGtt2xWjBle8E$P5N<1}hmRiRAM}^mfg)ee zp|0A*_61~^EsV<6JHLh$C@5&&D6F>Blia_kNI(Gs9w!<6*+^jzk$}tET5!8!@D7DO zHw2i;{3TU$o}1A(&gi!#yF`ufK8 zD@95OLTq99oP|^NTEGBBiRdM0zG9=EFP-f5gPEB4v=k-mNk?1&fc9p4M`O{+s z&uU8bgKBVd9)?o@MNcxl@+8V+n`hyifWzOfa?Yq()1hOK~kYXp0rReOkNH} z4ZUV*1`_dWX?S>e61*GZt#exbOlp3RvIjhy)*ES0je3Q5env}?WOj4Rh1I~;26Dtx$_tz}<( zKO6Tm)oe3Usn%km6K7LnQ3AHRxY)u?oA>II>MwHen4Z;sDO=@dta!@LNExL0L?ipX zV7^Antcq~Nw^9||cQ)M`H#3#_>K&P`mzIZW%3Rj0z9cI0{R3CW)aO3IcdPL>?~VGD z%^3KuZ1ND&8!miS9zeh@FbDaF=P zaNO6w$mw%g!WR|)7yRqzJ=y8bxYHB$9I{AkU2~F>)mA)ZU!rovo zT;y8SDKCk-wUdc;Q6ScEL%(4Nv4(8PipT596zR?k+Ap3z_i`s@CwhZ%f?5Q+tgCC` z0+SyQJ{9XLbJc$MF;#`lm=wb%x+8GG9< zj4C89Fgrk(*x;$FCT-#$^uGEz)^r{=!XjpqfF1|~+18*{&}&^)1klbxi_Lqk7T`|K zG{dFYYftZ>s6#sI4>#cMl#(T!(hYRJ4RlQ=rZt_qIL#aOi-piC^MoY&2wmYIRl@6cC}en|glRx3JD+#8haj1-G8IY6Ht zOA^mt?ZgPsE0#Kg?-jn6HA{}sAL(Hu-MA-Nc%t!HMH~OSL7m3tXZ6F^^vC}XXWt#q zb>F_7k*G)mS9S_zXJmv#N=8cdmXSS@y&F`@9@!%!@wN9>DSL0CvbW3-p7V{a?qB!s zdH#6*xL>dCtJ3%T{=7fuIL_lZPMrMZU&?@>VmjrH^t2szjO1`!1_Yx^j!DRI*$<|^ zP{6%fxfg$G3}1?zJaY^xo~TGKLB)ktM}Wl~CsUIaTqLV1zG?ajU6(IP0v?mdkB@RR%5bZ3{sIc~liFNSZW zXknCdg7@z0T(a6vK8GN0{g;J1&mBf2AKg4K4dAvmO%fn4m zpTG&B}Baq;j}b(G22HG@EZ@Y(UeE5$Z#1DEqMo$nM-Lu+)nuPIpn zoHp~M1NW?S#gO-_S9#?)@c$cSz(#?S9^fG?wXi-9<^3+@4~*h=5)N-DpKqxVJY>~Q z-_fPo`@L-6%5HX^ps$q_Rq`mH93P^Tu)<#JWMMfTd{}v&>Iw}rSJy9VE?N69j+2VP zlKCNdKAs9T)CDT(>LunO^n^WQ`p#}2NK0`9fYDdH)M9I| z;d4^INji%wNC(eY5ztC~fNTv+72e7FgD&hQbh`S$TyHCPampF!g(*wsOPEVa`qPQ0 z6-_rzkxwM@QuVyRGfNdae=@E#ke3e%i=viA=x3KS z2 z7$Df`aR?_h&yJsMFcC5Ufu>L4gGk}*urNIAUd76G=mc(~&iQc^XA8T|~2W^T7uS1o=xN!Ok%8w!j8X;OAg(+~vN zA`aoH8>;2b4ycD0Ey)eQyUznn=DUpX`Q>?ewcdQQ2!e%o@+ko%PcpVZ5#}|vSq7Ao zI+&Wu*kI=mls;}L9fGGMUKuj9bK=0Cx@^BF;ev>SV176Bb+ifumv;Mt1u_qDDCqI4 zHyl3@gk^8YCMRX+C=3^Ohg4Gf{7}v6PM4Cn?W0$3N97$*#bp;E3~}#;tOWNoJ>g8- zrUR6NgXBYLWK&YnEwo-gjO~>SYhF=!nOQV#uduIAh3W?ID)~vf=BGcw8^R|AXD3^c z!vs2SLaBokN}9qA+79=G{~SKx;WodF!k@V74)gr$i$mI+$+0PW z13r;U0Msy3n4a@|Jqlk%f1N@50Dhr-Wv@F126bv=1Jl*3z@ZH|-3+b0_>N1<_bGSQ zvKB2den6e>_jPNtmtoOzFUM4<@hs?$)cB9x99rc)RD9~ir+p+{1hB?4Dp~i$IiCY$ zERkp2nKlZ7-vEf}Vcm@pyrBR=ypDmE{+{{a++etOjnP7CP*UkBwo+`JJ$p70#H=UC z$QU6+Wt)rq0`jOdZ88@$M3jsrhoBd!Vz!gKmM3~(@UurrEEEEP}sd|`5==EgkF#M{^XQ`ZvZ-q}h$$yE^Gd~!b&j>$5*8GZ$ z6~N1#b&lx~4YKUUsKHV<&P#uy)84O2Z=p2Rf9`0_5!9u;fA!vP@5n6u-95D$X?j}X z@_r1_1dOWfIK(SC)<>A7N`82;j+r8t=W*T^UKd)7CrBf2P4VnFZ!=5~rJB|Mll8&e zV%{m7{IOrFp#-sQHk(4~SE-M(f9D8d{A3>Rw#VLx>W--6fZC^TWou;Aep+iWq3<2U zMk=CAB>gkp*pgZv{FHvV9FZ;Nc;)sj)edXt?D;aR>g|D5xbgAob^}kgV zmv-fW(?X89f2ycAfO45cCzNSdVH>q4z3TMR`~{Sr`>n3!>thtwV%n(Ohmgsj{`k>; zp_@+D(rgYe7Q3Tvtm{}m+2-&O4e%$Rey$Y?vmg~H!KW>UqX%jE0Aoun zf^=v7A!c^W%|*;&uAO4b#zn2w5sLDNyXY1sPv;pO$HI*S#Hao=>lB@(HKgkvG2#VQ zj}9QFP#?MSX9cD^evT6vo{y{IPqXQwDx{rf>#-;! zB5F8eeTdtyk`k*c`}q}We+$$90$Xq{Iq`&IN<|N4+4(XfNrDs}JnIK4xv z22YBTaLmeJT>Hhv_Z?JS)0@VmAs&s)su{W&8lkH+TWsyosf-7gaMkL4wF>FVDhjP| z?%MZqI24H4OmF>sa?NP#8{U!t03z9}$s_F~UU!!zRYToblWm!qb{k{8yYM8kCp~q0 zs8Mi!Gw{&Uz5ie&Y!%=!37H_YGYW?GacRU%>7c2R+kQw&?6V1F(`eYNs^Z-?xMdPq z>brkblVp5u+oh+N!8#S5}<77%5cg<@3x;)BroU4`G{#Q$vU3Kdm|mYF$& zH~P|F?_*$yK&zcCn{@|>D)Ck65b7ZZ?o5<|ttfpEWNl~?G z{(8$0=fk-{Zk+swPoF-fqkLavI_tLmkm|x6+74r`#LQIk#6{Kl&((}I9)YddUlZ{O z4wqj1Zt{cR!MbGW2m2#q1XlK6Y?rUMI^Gn^*^ucy1N~#d1SHFlD!J(gyFBDdSoPZgoy2XS9_Dm@@Ur!vmp9X2pWePbb>TS0`N18Euwb7`;@0S z9hhTNy16!l|EY@tE82C#$N`j*LUT8~_5hoo5$>!XKV6#t;B`z22A_c!fhx{E)>F03 zgi-;wZgY$=B8baZx4!ewZUxqf4tbyEdGd^KR>Odm^QRni(oMj ze$bI<9`bfq1o`V$h=qXKzRH^cgwqc?zGrwQM!n@OB^~J+6mt97s3xHj{&=IeP zX<|zim}3GLOiB8+qT$3Vlbd;$w*=>gDk?tQ&-2urEaDJqPF$-sBxpHew8Kg+GL&m{ zcRJ5ClnmlbS-wmD=Z_z2C@f0Pfb2AXuzkKicNQT!s;e-Vj##~Gd^Yf0nveVcS24AY zLm??~o5-Ztn>Q+`*^Mt?ZgS|viSyPQyq0fwWTHmj042wc~U zEItcwU0=qk1#`Qr0Mk+zjjZ*|O`NJW^{dx3BlGXl$%?}gCTf z0AyoH##5cd3sxQg2}dL=nzY^~-1J@(x@c3jumd-ZQ+5oP1O<%63BP$HHSU$r_>O$9 z_KxR0f_Lc9{n&$_p-C&EBcD5HO#+sA8-Z*T*$yg5uyZKK3?a zY!w9pO<#~OOWwZiw}Zu@yZ3kmSuDSQBsl4qqN6go?!zoBthxPbN%mxj3Je58eO9MB zQ^(XJgnxbNG%I-+%k2rwrQJgH*j$%1!CrpZf6Vamtf3GutfsmO^1FQJ`)Zsg_kK_A z#On=pjSE4HA018F!}yMIpGVRMre4R|FO05^e+pCgtQ*3PjRmju4>ljM+V3$Q8vj}cD+u|OOX39 zpm-(KHY&JP>X>Q<4)!&2`5KF|a`I5x{+-fuTv~709mFgId6sK>mBUJ>YGkVHs9yy@ zq|5$E@|h#c^{#ErH?E5p(j#)sC98C!d+V0>DioNN!uO8k5z^N}scSx8cjRw8_-8!a z)AB-vu)B)zK!X()mv@8ppT6X)EZi@^NMtuJTkn1EKL@obKfGUP9C@XuMj9#? zNH7@(>6l3Tc>Cn7I5MAV^|F^%DC$h^7>av2YW=djMrtN-;iS_LL-pcl?%-uxHpLC; zKJzj*HNMc7)5qsB=Wf&!)g8OsY`~;Fok&VDI!OK$WBVrkWBtjUe!VjD&e+0bW7RH4 zmkXCP1$@7pqq(9W#dNCVcth~(^NS;eGV$~Ml0NPS9P(`R*KMbbP+6NN$OTaS^`tnx zQ+s#R85j0hU);-(!qqNEmKDbf3kPl97sY`d} zzfKMe=hey!WWw%rp_8*&UH*&W`1Q)($z5@}%=P!lRlyPhg+^djwxn4R+sthbr%SrSFM0*9a!9EKYl^t^FdQ z*RtR{BmUumWPy{El%+P&^j6_CK^t_O0Ct{%prs%=;(tfOSYSC-cYACtP!6LwL3QQ*BMPTg~G<@0ufxOnhl%E-S?is!r0|f)va=Qscp1@4Y2G2C5ziowI}dh-`+Y(ZwPCTSFkP}w6K`ZXRj4kii|(EpI8GV^mOz;tIx z?otfNhZ}qEFE4sKUd`693;-D}#K|Rtp}Oif7r z4x(VzXKjs9%ighN${f_|A)BTMyvJ z+nWsTc33G(=AWldXpVgLAy z`1@go(uUI}=jFYB*c5xEZalDCRmtBkmXcTe<|+SCwOh5x%tV(vQg1Jlp8j!-u$JV6 zeftXUFRQDA+m00qj5)A6%A6o1>x+pL+|QI%xZK~p&Su-U-%P!woKCx5sAhH{a&0V5 zyHzQpK9=3CHp@E8Km)GsUN_1dVycd7`uoz>?_S#C?Gu?RRflM5U_v;$JV|jT?cG%% z=({|E#hTknentPqW8Fd-!L|S8ai|Xx*YJ0`5grdGL?q zbpKevzV<2vUfh} zpB8qDA^2Vp3|Q5P%`hXsa=fWz`G@Kjd8rIG^`qAIY9ERr)<@6Hitba?vcW%Ht+pVf z0@qXlqpGkvKg15>kPvDi#~RQu>2g5jR4xm_HELTyH|@Uya5?mrjy-)HpzLNEf$AkWgs@o+yF9N(wp&brS;FNdN|ago)8CX)mP>Y)2X^P)O{BI)VqL;==12SnXl z`S$>ZMMODpYdq(5j3xvIl(kT$p9iNw>$jU=`c)4FW+Dt;5pbYVXc?!Hd^=9`L9@`B z1k5wfi3DDml@t|y)DU%93DlNkKt0SI0NvCg_ykD6gu9_upno7j_B%X z2GfBMgI+-KnR6c$UjHM0z}*~Rg_v6M)MzLMMcHIbqxv6K{@ zE)}gLf$bpHcy&R-HuOpov0?4+H16v!w>=IE*HXkaC0ut49`#{5h_Rv(pgPMg{l!1U zA(M0OeJe);XLfUiu@PekEb6S1PXBdX(s%C~1_rnG_JyWHmBG z;05}1^c!vuttV?_)tF-&ZRy$xxlvdBn$JuBPS%g?wV(PX zD#0flMm|+GjN4*BR(h*dgq5&9oZBcs<%X1=UP2JN*5EHe{GY5EXR6U^S6 zh+n!8$$O1W<9TS!-HVIUg1J%{fEBEPpoLXkIZH<;EmU^X1TYDMB35l?K|yu&R-a5y90g8g z2#l5^V#ruk&ml4fvr@VQ%$Vt!nCfs{dLQNIHUZZn5eQggLU3y~Oz8ga>ILll$ffv8 z*t`e_-1;wt!P?%CV{?tuaqnT#Mi2l%yJGm|UfX!pgD#egPxLjzx|akRinnG9>H_V8 z1Ki!j3IhVtzSwCBD?h9}oK>t+sdz0Bd(kn1DPNGiT^@3!gYEZA{3~k;OT`8$rHi+q zmI^vMO0+)_4TO$p(cBB_t4lLUYAGvT6z7CJo?uM#nYq+=UqoKivoa`$uUtUIhp{#1 z>A&C40TgZ@Xl}Y=o;<-NkXWQ2*0B3R4;+bLps9$$Tr&9kYSF{VfREMHwE(R!0yGhV z`07Cy?n-N@Rd&Jjk{tw9w#pSSkH|J?Bwq*C&j_f#{6UBiLCnfZ#-^T$U1-YOi(LvA zdpd&v?(S%!&EKEn|D!^yF~ClTjip&C2OVN<{kchiT5(HQebFl=RJ9U~tMyFR`c4>< zMJ8q`J-$B|^5t`ULvi9ZCQ!97#Ef=-x_G@(Fm#||17Fd zn*k6sP@YKf3(?r4Tczzc+u`5lKLsAPcqM5C)qa@syH*(tp4*F~LF|r03|O`$wj66{*sY;(M6ycoOhCK2^pEb7qYa6vNrO2j3#1-h68-%)pV9V?I`CKR@b9O@ zr(N(xd1>tl1RD@&>tbhul>xi%rGx2o|M(bw{JVLl zZh-n@-`=e6Ef=6 zO#f!LZD3;^E6u4jZpy<~m7xto>cOWL+eU$tLWpQrpgo z3Zwt~Mx>*;W6?dhi)MYf{)nmWMdNxNLA0iQxi`tV4YkSJh*=0fWCue!lSQ7YM-Q1| zv??^odCgfImd1jmLfPmkSO8c5$tqDo2&Dc?@pym#Blz~{wl%A?2hO9};Aa2p(|=K) z^vd7^LgBupS%be{xpQJE;h8o%$CZx|5@s9lS!`K_t+u}sN$XZq^1mK@M0tK*(D9~c zLth{aIOyY74zbl-KC*hblxMccQI114GjHbSQ|7i!Z?mrHg=`%Z-@2xMPb2=tQy zFfI4?G z2Fuiz>2l#B>Kj`e!T&j4b!}gEXYh_yOAT(NR)2iMp}&emj~S>D6;}scUk;X_A~Aio z%5d+@{!Q=BcXx|KXX|DI#l2gpCyHKU7$cn9(L7{+eR(moL~s5_eb>Z;@uNZ6;^tYJ zJ5(nbQSSrWRH`TVCKqeR^m5b}{pz*bD^38N5qu^JShUL3AB;D#B6{9c)#1;+p-55zOE4Q2 z9c;U=0NKu>tZr0VQodJ{HdT=GB>|@lfFRN(%|EOGcPPR-70-@QAftr$% z1*SgRUX?G8FJ$hXlY@Qen|-vRRI5?}Jn*rxM2V?yPtgg^G|JF6qA`Ikz4Ku%>R99RSI>X5+hjdqnP6e!A zNni+K7FY-i$zg5{W(xH>)!=3x<_P6UUPyt(kS2It9Y^9C6;dELfNC3!s=@>vvP*%n z&1t`&?PUvG`7;pcg5i3GJLbd05&E1&?};5VeK2gy1}d#ernW~z%oQzFptZxaLe{#Q zP5!MPG<^Iju!q2nnjV>l0QW^Uhsy08^b=Ri%*Osm628u3gM*8IAfs$jttI%;O2G`e zJ#9Sw@c*WkoSUkP`vQy&@9pG7<*a*c+4@pmYrd5Hbj*rrnIR-B%D2G}aM(Yql>mDi zAo$&XY6{7%23Vyx_meu{)Z4U~Fr%9;egPbDV0t^fXdi@O*aw?qXod~DB^)9rAQSk3 z-+AQ}Z|nATD_>tvqT>t4cGKOWuTYfAUI_pC1l|4I`u*8Z>e{!Zwc1aQ$uQjrWn!X# zpqA0BUd#YPL-y(6?V?rPUGqA)DJJIfoj)%4~L zBAg;#*Ml3uj}*U%CF_nnO}QND_l9XNxon?-ujXcY-yjSnV9xQ+``PzTYW^800$x?X zhm!^mGB%KH1!iV)q}+~sgec_Gmplg5sVFFz0Rud#Myw$DSPy@NRojEzfmpE{trgWIwXRDSnHAi~)d-h2T3vbL1Fj0L{lM8pfQ z$29+!nKze~9m-0#H2Qu6fFK7Y!*UVn0j5{lB*eA5#z<_sL9V(3`T^DYH*emE&%6D2 z2qMq+@5^59uT6pl9!_YVU%p@FdVyI|;7_pMZ%Xy4OSg`TVG+_(8*D^WLyz(vHMm)LFkE@q=R(aS28_I;zLI*yz}~Q@^%o zEq6|HB8Yj(G_8XrN{RX~JJZ1jhw4|JTszID8w!N&ODmgKRIkouD11~^>fZ!X+k;>4 zNqU_p;tenV#_@=u6fgUKmg4S=IQjYJUzlLQN;pB6q=f@`YK>wF1|`AX)*+=53hz-M zT~8ja$v)QFwi3w-I*bsYb+z|)nybeAr}}F53TU!w(`0snXn7L;;)`K3fX+^_sq;yV z$H#SEwwpUeB?WSZ8=nGHE4vbI22!SlUovAt$_7v!IOwTAeB@q%8deLn$n|1W2iZ-v zUHZ2o1))1qu+g18+Zcop7m!P6Q6^paSCrdK1-o3tR77Fj&Hv*ONj=UqC~Fac+DST4 zK={lZRky;bj!oIrx(7;~kGcf%-B~qSCY#hopI?^Z&n+(lk$ajY)6~|Pwn1EnSf-DF9;;A!8kl% z(zV?T8aE>rs|(wnC#XVGD%Zk5Rb$wOWKdgb)NOXdUTVWP)7oTgP#-jSwqclq8R zfP6Q9e6JQ4<5@OXDJ%R!!W`1RxSC|d6@Rsenn>mVb%_L#{= z8pu^40MAO;+uLUsLm7iO9KDdJQYele*Xi2=Tv6r=WGf`Xn8+Li^Mujl7}o$0t=ze% ztD6XwHOh}EAZ}EU9nZJ_ac^k@b^&lLf%HA2fl?T}6Vqzv1uySlA}Rq135jRzF)5Sl&cc>o-5bJr>T@wJ;AGsR z6Irpnk;;F)H%r$G;!QQ>(8Q31fH3QZsxKp?23EN6IVwq?={9&fq14p_?cA6}n$uj- z4cIMbm=`5i8Pe4`%%pZf;a!kQg@ixFv8c|q<-(hr-NGxcrF}h6FXp4f{Y+|TXWnhc zWEn>+&Erek^O{XjnBE+8Z!UQmFb>TFZbnURJXJm!wclUQr`6ILUD zzTD>#2EDLK@xv5?H9i0f1vy{NROg7y}^ z)~LAbbZ&D=%n|4Z^MSC0@x<0{1simd#MfMO70wc zbnr;SIveXo!H;Mmr-pHtIMJtROZLE}LL?#U?VCg0^Q+677})0{2LQZ&mkfF43KaHm zxX^I=nzs3|n8ZV&j4jK=v#>$0tvqI<(RR5hsvjs}zYM+57%E`dcpkZ>PXcws5Vo ztI%pVL94fls!eJowEK0p{xAfNgH z6y7?pYc}Sw0{yoUKqb(PUhaKuw$z^K4fveuJiAKv$0tUK@(j|iX~FpRl0|QX5yfX8 zD(eZL31_S1T%07?>r8p^0im?{@m>6+XyVm~n^LdHdGtVE&rr!(2Jgw1b60E=AfqZ8 z3=9maS5QjS1agz!aI5a=eygfbEmtG++myo2g`l16=;-`0l_lDy`vwaK4^0zgqM&#n zSB$-?vHJNCdAtHRN_cJa)GUGLPri@W%ut8+EKsUmQ!w(pmHJSWn`_eDgd1&KWIG%F zp)ttu!x-$InDZ2JXEdU{n?;Z2ODrtez49P=jelcLWdm^;_|%T~W*DBo*cyK> zf3qI8w%m+4RIlQl@sVDlaXB8wh2 zsxo4LQ2z8mEu+wf>g^M=M_R4gRs!A1JGvJ?MkOZ5TqCGG9r;0rMWB3(k*$7VeRITE z{m};}LZ*?>9YMv~CuY55YWAVs7J_(SJ^;De{SX2m4sdGxc6!=OZrl0I1^C}wpDMqA zD7u8dviGL(KeZMWTVpfE!Wui!fj$6{>2yUxm2%3{`}AcheN+;l?w%y7$vz17xr#oN zK0_JBx~5I}H!DMO?t#d!;B@Ql9q{=9`E2%eDdLu zde70P-)wv=`+A_}-e9EWwR<#k_QBTa^;jem_gp^1y`d3?nfR>|FOONie33A$dtV}c zwqt$*sWYhr9%f?q-wpThe4hch9_-U#bolCVs4Wul=+N+!jplwh)m*GIK!qSQ&x(7-U1i!`l7B1;pk+4x)p>N<7qU?>j6y-vzU%7jmfl^UTx8 z5Z@E;nUU6p+wVCz9hB0+eUYYa5ylb|3+@X@7T)z5DFgZ6dJGC0(gY>)UYu%t33%R3 zmu0+aC=cm%)%K?oU5lVhlmcGZD0E$R3+=E=UHpXv{7_7>qk25=<#)(XuW_+s<43TI zp98HQHC*_%H`l0e+ZX}D*@%4mRK%88f;_Vms<$u>okxa-snkdMo926c7^8$L#@{Wy zFrI8EDw@%5{%7yxAqrJN6-*Nz#6V1o2k199+ z-=_M+HIe9gb1x6WhO7F419z1#1Bg`@%09nntI_%uG%Xm~w4nFda;ixls*WdbfJ+dw zr=KxJ(Sy+k_|R1~NP@`P2}2eKtk)77_1-uu=&3&_-g}9kpjEkjosTfRw~f}6=er3; zJVJ7A4xDfSn|CB>aXLRX%3kJ-uc_N;w)X1oq@7H>zjr+%$zYpF=7$wZ z80x>_@78rQzu{YT0|r;*-Ob|t0A9rLZt!D{F;ZUW{fxGPBcTPv76+J zQqm{H-Sydn20>TWfb_otEvjmdU2T`iN7jsL7Q%SH#TUQ-r2CD71E(_)qnh7#=5_tJ zu9N87dWw9L+mExke@fzHneNcLM9C?44Y}>iHtB@rgX4P`%X#KJcb3gq77gEki}12@ zj(t+aw@(!@=6yeD?@hvN@Kgdcg1n2c5hcuMqS>*koV1_Ne#2T_J+g>@{*AA207Tzp zKK-%@P4;qGR=AAhE#5!P^Z_B21B;j?xyM}GBBC*AQF5(}MzZ1DH#Hq6>F>%)L2$mt zek(sX2(*nKML)2#-Hm_qMuuj?B93(PcJ^|b>~BeGgu6>&pr=gVA)UUl_;0}=DhWow zI2?21cW|VY0GpGv$vztE9v>QWTgvY<<2j!4MwX$N2^sedI}`Bt!RR9z?81WObRO_# zi{(IMX?WfvWM87*ezZp}2Yx936RRo0mw;AigqOSrU#Sb=@EgLv+QPxkN}-t7s>(Z45YLFcgDJ1)(Jbsrus&voXL@nA&^WT! zNR3kwSA8$`r4Pgy{+FR*3>jb-RUgdhYP2!^HIIQ6Pc*qgzjh#%I{oKeNg{SKHchz#Zv_PO-bN=84ZzUb|%ZxShstG zAn{;(X`ZIzi@}=MGa7*J-tZRKJ;8EX1VvT2xk18^e%GFTY`o+4`7P7@V312n0CNP> zP3~@!GfaH{O3cdjkI<{!@TIoFk6r}$2)ZEYf4TrS#1t`p<@sOmUP0!Wsnz}?}!xaDkKv? zq8DasIvyq$@dAjT>6G^w98{=0Oo7P@bf8|_as98q-}2vsC%xZ}SG_U_^~|sShy7nP zZU4`xFn&#U$G}ui2{sql6v&tYAeCfB*j%jQ-US)q%OyOs9#Yp}#oLN)Yb%ozK{IGh zv)Zfp@)XR~6V|R5!cb2Vp1*V;qBT3v<&$plsz3e*_QuhJuq(Ux7q~^=7v(|YLZFl7 ziso~k$W#F1x7Hm~UZRIUITMEK36gp6PC9+XwREzZG;;;mQuV044Ka!W1Xof-#N7e& zO&JUXIj>AN;**!gc~C->&io_{^eV2gpgL#@=wjTq`LWa{dyC#aM&In~N$e&NV0R-Y zb8!pmToS^H4q|E6m8njymIQ)~mz;pz91CcW zLgU1|!WFoQO@S)ZweedGr8?dYJPz0kWH$n9yfS^fk-CXqGKfQW9eVOK8>Mvh3s6s1 zIBY>Mtc4<+)th(7nXB3*5rD1})YQ?YRGW4ARzE*BeL6>6gRE4;Ds>fb1E5u?X7g~L zN8R}7%({?Ll+wf1o+|(ul;`-rCfM!QwD4gYf+)z;j2aax)N+IbuQ!YLHnAE#g~?W3 zyX+OS&MolD83kO9%;i7>Ah^b?vVaP1Ii!PuUEQ7vy$qC?ORCNK7>$xHT&La_p2Pih zu=_&~cvEBgXzU7ZENoQCS~IxPCp=C;gGi_Ct#$P8m!8-XSy?b3)`JR$oYn;0uyTo>{QUP>dWe((>{`MXZpXT>vl|gBO4gg& zKn+mp;`p*z96E_2`~U;ug%emxU+|OTPLVt7Gh10%K`F1t*3_MBHT`r$6>d`E_!q z8bynZu&J5xC8#7Lz8h`;ur9LmcSL+Er4H6A z;3&QB0FjR&cTf>*D)iliZ;F*IJ1am2Y%CHGQqtd?yCwD=2L~tWaLa_i}}dev~&pu8JG=+?$m_J~8l-xu9pt{Z30Z?g) zG%8wC5MCFmTbd){DR^Huv3lgT1i zwIW!RkqAZLm{A{jl0Nf36G2j1`XwT17GD&xAh(AFN!M`BFJHf^mUYl{avDKcuoYUb zEZGH=!+>G%(>Yq@Z1EBFd?(xWM5&CyjU?#lE3wN!5bXEXh;3?Eg{*=cD%A>_RMW4j z0pu*hmD?7!Tik-FmhLp4*r67fjO_X~8uV1SRq!yt_Um3BDm)@e!<9HUsY(0gpYdS1 zNL1oaZ3htQ^<~dbh(tF`!^13>@GK>i`(fp!P?)Bzx6jq6AD4hp7OO20t~&HxA=4~I z1RiOmmY1~adzYNj1u`aPii*-~6W#V61t(ps&Jz4AJ7pf=M#6U*%S6`D$O6%%7mzmd z^0V#pnWi4f8%eXj!nYn9a_4?={gQC3s&$3C!v#udMdEU{_CK2K`_~0k4q<5lKn<{h zR*!EO&fkfJ8c{KHJ01r8YmD@_YWAxRGLM~nm5mV2euFn;cc*0P$paE)DHCbG-hyj4 zkRsKOZ?8#P#`}KDZFLSu^2NQaot?+m+^&%iW|{=F44RC0~rlea~0jfbO>l>*eTA<9**Lt0CR%au=f989_|xjks9L zpDLczL1Ux`k6>&vL6ljD{|e)&%*3h)^0Co>l1ATIl;iglJFTTVxW(}bs*2-WH|cnL z1`!MmX^lk1Kh$!+y$q?wHW=nK^$^)-J|jjA3^;z8kGFGTpF#3H{8^S_@@05I4{ou= zKgKU){c2Phk~}o;FzqBx<;OU`^#w0>UlVxs+yYp9)`%Sw8tHfgWLR%qz7k;v*cN#~ zv(!F7sGyuo5&|4W>$Lo`WAGDX?9n)4jWpReKmam#KL7_b**d^RhcZORUYYCMf4u#> zk-`~W(0NlMerL!ihhto7CH}P6d%s%9bI*X0Etudq>cnT%!LT6#aO!g7XxrnKKo$1J z!S+}j=?m+ODOH;nsYWWft{nnv3{z@{J6R*Xf`k=!& zl~XTN72rCochPseImTi`w{ap0TwxIr0l_zdT42A2+NiQQFliY9VKNzE+k?o1zhJv< zAlN4N@{*n-!8@UO1AkAq_3WVs8kE31$?TtHXUzC)!i#dUmIxdwG0bx5CTMW@zO;-t z@ppb%S^|B20*vROqg z?@P)!VtI|=ruD+p)9AvT_$MH^jSd3x_2>RBXb&3JZutyh z2mvhn)n&js4R{1B1bHw0?9I?=qgDZ+H~Dl&Z?km+gJ;u;6%#w8WF%4UdMt4i?q(2t z-`?rL61(PYwTjKQ`FFV!cE;p$iPimE%LD?i=!ieUML(WN^3jV?V-Ilu!>FhIiEdrF zi>U7-L<#-c&$zsA5cTgp_0T3+P^b(UwBLlG3uhNaQ_;yckoasD(1d1j$GzsmqSR1(@v3j9!R!3 z&~#S&ySuL;oIjeL^$c+xcKI*YW~$JU@-@3;v;E-+F%vejj{<|CZ~|Sh>W%nrz5@W+ zSjok$vWAF?A%~Kf3)CRHCZ^jvVmdFTFMB?)|8c~8=!ku`MA#o8?%UVFE6YS!Zuk?g zEGvF3diR2xc}fC2!R;QF{{0%lgM+jsVi59>f$#168;wwd{^R>P?hG+McKP8R7^=Q* zHt*;>{V9}Hjg{b;Ql$>cn>;<`n5X0Q3CITe0C`Sb zW-r%^&_2b6MuBTce9F$zhaTEEPLnQ{6fn*`T=2EO6Vy4J_nS=151j{dbPHa|Y5XV1 zyzET=hY9oB@Ya%alz27$VI}RJ)e~6t1uW~QwQ&vmZOWf&e3wtZZDqh5huFo2^H&^w z-I+a*w+gt%Hr$;XkiG@+Yu}HuiXKIp3qWx+d?s@*5ppidjVC9*H-4XVB2emQ0M9(9 zC;7zqY*<=&rjrspA`7r%ceyVXCJ;;z5r3#UrIQPQu?|66+ag%LOTf$RTwE5!ty%Oy zAMqzZ>kK9KVqpqgtrvK4vqKeHkUJP1Pv&^OK(O>MT#tjl@5p=EufWD$g!Z)->h^|l zKeI zWYjQq7^rXD2#<^h~L10YDV$Ek6vzU z;ZrRXQ7fMJ%bp3MjdSMDVTv|~AHK)!qdmP#{)7p6uL<;Z?N4F>p$cy~U8d8(w2KT6 zIhEKiYQFEx2dCIo>Np|XM>i_FVJgn~6X63;=Ew<7K&4O%48y?hwgJYpazJ#rJ`LQL z+PNBS}R4B?KK0`ef1cX#& z?n=q}{i@eC2gu_P-Sca}q;nK1O7tWVj|lO8Y@?7??vpQ#1@-;LaE0Yh;Q)Qr0zoDwlMN1rxT~5>^7Yyco zue%vsG$hR(U`gT#tQ?k}o1(^$ffmVe8!aS3W;1fInd+o^(zqXtU~9o6*{qaXehMg> zt2J!u%qKHR(w#!A{(39XKLBSw=K7|)3U#fPb7YRN7Un%{#(8;Qu!Hv zZ~dRwV;=xe!DRNlI*N_O=z+EXfFRM3JqJ4nC|~#9vv*}^#k~CoOO5jO-cQC=)CR<6 z5th&PMYV{eLU`==T?1SJUQte8*j>O15GS{rNvt9aT*W_Fd=CMt11JPv)51BDXjH%} zWE(8#uvmjCX9^P>A}}$(#5dDPG=^eaQshhPsvNwF?R zXr3jcMu=M^z~D{x!hdy^_%@c?K~t5-k+7+SV70wFG(i$Z5Z5fsnbZp{1BJ1i!kjUteq$Ct^|m|2xGMNsJi zVb~v-V|6g(Z4e-T_7f-xQt%n*K{0S^b_X^J#M^=eBxjX3bVnpYp}EGLb;~mBqU{x5 zSi59&VGhn~CO~S|eE3nR^Ln-*tdg6%M3KJ&C)Te~zvpKCCqNM6--IM8`k6=5}zeeb#-PFt6>%8t@O#K|aflq2bVWe@V&T zCfEcVHRR(3UV~?mED<@-u{6UaGhBG?nTx<3O{GvfSlxuSiotn@47qML>hHq)h{9)3 z1_8JZZte)gAUQAuIyo4x4;^{D%NcFvh0nx7B9%(m+eAItpR%mSZy`LMfI!fC>?J`4 z6krh!gl+CuUAF4{&w6EeMj4j)U?HhSHgHywH$Y?^0N^=DhpJ~r(K<6EaRCL(a2_V3 z7PawZzra?QCr*RBJQ2^zdnMeIwZdU3Nf1^IL;B5{m@X?9Hu(YcZD1cT;IaVudNjEG*9m$K=L?Ct{l~-FO=QnZ;mqlrcDNkpQEJeTRsFjsszj0> zu!3;=U-Vwcx%rT?{3~VjLu|Xb7C1;PNst!b%v_-0#h_n07L}@En(`SKl4QpKFa6~) zB2P~g^N8iJ0r(7pI9&&?%907l^5b+i2gTUQ6uNv3ck?>R{Wk zc18x)1@af8|6~FIk}dgj`k<5{4*5H{#5=F^K~KsAvhxS9WeW(!p)4B39V|$m4p_`n zNLvDJ6D-)#1h`Dv^*Ndw1^K#_A?6N1^+uaz_(2qa*4B@$*3EAT>h)IGDMLqT;uAEN2qn69pryV=%IJG@NhM3n2r_%hw7wi2*GvW z!8Y5agi)mjzCi&E*m6nREACUz4q=lU=7BIb?4qux^VUlDEwvWw4cIk64}~jg=?n79 z9aoG|5{Sgk1L5U`)0}#l-LmJkrI;tIXs4NH+uWxd*Af z@}dVkzM^UK_O=y}I2S%ZH`5mk#e;6e*j!JR7!va!r$<5H>n1fptw#^H*>($3mUmZs zMDBGTN}~{fZ#D#qjoKNIUK`~VfFi;wLRnPkVBQ%F%-nkCvG#5q!i{WDYy04_!tb;R zj$_5#y_2WjOR)6e<=IzxSrcKY$X&TG7=i36iJ#&Ly2f1Z4DYqdoL{`35m3% zQi@V3sk8?XDh*WHR4Of_q5U3Lbw9U<`+nZ{{pa~`^W^vYey{5s$9Wvb$$6A5bGe?x zkRM!`_CEeYK>eBYMebH`DjW~h=lMLhp-L-vvdn7OTbSh!8W z4;SYZaYH_*>ntS&E<$D*y{rmHxla0r-;9PCsMl}V1Z_R-}AmDtAj{y(2FdLf9%E12an7wwQ z58ZaffnEArD1M6g-PjXUlz<0^IE&T&1G%0Hw1?L=h zW0qjuiZtfz27K}ndo&_Erkuwgd0Te83D@Fp-7g2Ct054;=)K2Ps}8Tp(|Dm04|`(l zTc~RDM8#-v5Cab)Y8ZAPvvK;yQ51irSV&cFgNQdtZ8UM6w~ul|lC$L_D=34s8}V>s4mk ziE%=5j9l=WnUhVRDM+Hq+UAmJ45sDSgkm1Fnp^W28IK~G?7UVHDPj3$1k`7Gx9I|q zkFL$nP`oY7ibR=80ThUxu{!#BHt&IHq(6xgYOTF2{f#^y)0m__mk~ZEo+ZKy&78ALZvCD>=TWT12P_ZO zzkjt)!uIO9S)-%>dhR|PU*C7q8@$W0yw!n22ZOZ-cUFC*L^}IyM8=1i*$Wqf>fl^t=ZAj%l6Pl_< zs$^rIOWqCYhTw}=XP9ddpBv&zqJ#AvMr2dn1VHIjNGBfqW$4ztlktJa9S~l~4 zJg!Q1h@dn)BxuRuX*EQzsriCF0Kj;fa6i_Y$_*JoN}W^$L@dx%#!>hF`53P>V=u!L1RTj{E2CsikTb2gFywC~$nfjXFOJi>eyZuan7VWoKrIP7;&8%mMj z`Z`c(e+Sw-!&6L!CpsHrN?@hyrMf!xb+8rgI%S?<+`u?E1mR{~De?O}7|0b!>K2_0 zl;@n<+5E0pdixX(j!8d7t9bQ#$iRk#Zp2&S7bCb=6MzAKWI1qr%OFvmX#<7DRwMn| zqfj0Zbl7M~o0Z-(Y-hPnmw2hDCRA%Q?dw81&#E3Vp$a`iXi`y7b%jmd|8jgZ(Vu+Xh z9pmF$!vb6KaZJC}kEHS3SBeb6$jZ05H+-Ag4l*r6EwUfbto{??H2wjTZ&D4~VC$1D z{YUgkr0F66DLhf)vRIE?wx9nKFcBMEEh3opPNV|NIgz?~-9k#H#ps{+q72^x+lCP< zg$HvJWl0t){+lw=d_US>=|9!D!Nur3g!>g*q1gXU>48{D=aATWchz?l7r?pvYM}CE zlcyvD!Q(wRL>8K9RGh91^1g+|-!LRx_;^7rBwIBcdaJkBIv+5oq5mD#Xs3SPV>upNIQZ z2%6TEHL8DPP$hj%H?|Pu-rWj}npmC_{A02a_?Vf+;&%>UYHY9!&lKu&u3t<0yn4xz zwjhjgUV-sqy2xEaXu}r;AGq#5N$6RSgo9u^>QRgXv`_lprgS{sc-IK)KSus!k?*(i zLD9GaOwPsUSx!^C0fOlx3jgYXt!?fl4y}b0%9aQM4(BTN{J6r81*h0baWcq6IL}?} ze$*D-Gv6-I$aZ6PsJv(}Cm;C6j1iGo0+`%Nm>X>!(@nsALw8d?;xgu@iJwHcod=ny zmWvJgPYk|Qik_5vfPvIOgo21;=_wu&NU$x>m(qFpuK=*6ibvlLu!o=BuaiQo)}$*r z2#jkv;4#uKuKLZ%uP=GJv%iHs$H7-ai-5DNq7z+4fA!=yBqUB86 zxi!=LQRTz$K-vJbrf8u>5qq4gmt~|v{69rPEN&XF&`Nln*EhbrFT{ZA;YC`fi7rkC z2NE7{52j+OWZ;IvI)Gq)zzt)u6T(PTGtrZKQWOI<;^+Oi50%_&e1vxNC4xe0Pgkv< zpMGvHpd79W-&>fp6ln5$~8s z{6EhH;lhm+c`NkcX~CX}MIW09;7sBxq57biX;-9w^?)9wvyou|RB+fkb_8Y97i-a} z%#^w|#YdEs)J`(US<}3yGmk;dTYeTp5fQiHF@$D9N8A}%%Aa}Yvi-+<`dvXg3V`MU z0z;(kSnG_gU#9l`NnjkN-|?^Y(k8>)j_RaI)%vl)dtyrC(UJ5J;`gSBnGcZ4ghEqu z&}bj+i3~8WuV_@PV5Iu2jylT3=x}X^OaC1o43>}loofF2RsUkDU3#WCBdIXVAhCQ@ zhXXYYySlc=v{oM@x@~b`M0NC=?z;I+ARbp65`XLl) z#Hjv~3l>3toO4$CO4D`4nN)2dAt6rTZ|BXVe#9@g>TjfRCaT8zn!=lin69}SJ+#$( ztxMF=zaGWjjhbod&a%+WSX&qf$wj-AJu--bIc#g1B%Sj}dJUW}OK`msr z?LE7$%DvIi(ejzmYkD3xflx&DkU0zi(va&fn0Ej16PGnwsNK1Gt=JP|tDvCFB0ZPK z`%#9J+cMrJq;65Pk6K?&m1P3%`Z=wIEnc6385#c=T%Dw%T#+IGh@E(!(C8_}p*>S1 zOjZble94mXOCgg*l4A6g1B~D$j5tl(eGkHYbFER|l#bC9;}%+0Ix8>q-M47&rFETW z9uqAg%hM|FPO(>Ed+#fx^IUfjQdEdj08CPX$v}nX@ZggG$aaL^52a0#2@Psr`SnWT zqcKE6J>k~*W+PeJM!=9g{Se@^mxU^?9N&1n2SbgcxM-Uh{lUdv!?(MXu%K(wef61# z?oEIUrou1V9(9rF28v;_*3bYG@_X42b)gj?Gk!xH{Z!Wxi^s<~EKJ~}K}c7m8Oe$C z)30;^5o)K7Cg>HCh5Cil%kRvexxW3>q}U&rFKAY$cdK%Q0%ln^QA33NxT`2X4D>u~ zWTmFv_nqOny-0m&$pvo@#^E2`20xX@63JboS{1SN8QzC`MzLGd_p{p}*gSr+SF5OU z$*&#Z6VbRAQ99&8XD)qNyrxGyk${!9I<+IBNR2|Zj>ZzGu*_?#OlAKXXuVI18aOgI zxexeC_`|ikyjRac`hN4-!AlWZ?EA?_)M{c zsG~}6&AGB{7I|-F+iB4#T(y3|4KB2CEAsXhHFRLQx;lJPRgSFNlgW)(gH`ma?}}4K zfy!MoJsha>GEQjoaQEAi386>PMLxiFZ(s@cD+qc%sl30;lXQ9e5TzpR=R*Jl)f7=l zo9N?K5d$sTBe6Mdq*6pTTaq~7Bz^HcXI{MWmNE*zT+^BF zxJkP(60PFO?8#nScF6bx5S!!6zyrvtexR}~ znH4`%;1Ht}ODkoKLRXFDsv66cswIPtm&w!&sN$&+m1kQp>iKA6!$lifJ#Vq;$rVZs zwGQ^t^xTnWzF#6HLK8{!@y3TQswB4PuN>PrfnLXG(`woZD;0hp>1WAl!ox>AgsM|$ zxd~18gs%0IsAMn3;*SrEwTDvAJ{e3`(><5o)#E}qX78J!264vX=lSkSE9biZ8!Kh; zq$I*1(5y>*xL3uc`HHA>2l(t9W48;pYKMb8bzGo$4~EwcEM zeVzPg9s_{+_T`H~sH#N%)R-x0wtIBysf z{MU3$sTSCIAIn>D4sLYUcHj?% z^#AZ@+@w;di-e-sf4$PZ@>2fb=aYwwsR;%`dW~yOQ!N@ZKc*LTU;R{Lu%p@drYE;0 zS;tt!+RCsa@bEP|sT+~*W3S`|Z>aCQkead8Ikd5g#khS*Q_5VwPHlL{^oTMMiixcz2`upmO$sqx(A*3M4DZt`O= zy|<{X(ijNZp?#Xkbn8{`1V9CEsT{j$m%BVl)marI!;@m4D|=wRx!oEGb7|?!!O8PA zWclK4DNU5lSK0!P?7!z(06B5G8CKJLwm+ondVF>G-s9))@Ebfpcy@2#C7RsO?i|G3 z=*VX)w=teg6f)^D?Z=3uP33*91+5}Pg>f51>|X8}fF_;8?NLVHVck!_fVeA`bN=he zVy4g{vVp;_s?E=f$2Q(>j*#OV(DB9<`S+57F`LlgBm&?ONkx}B45 zacJvj{BixSo_9B`ZG8%c#JjZno%THgnY9dwI(eEAPM>JHv8x-p+t=@uj44A3nO6+X zA_QJPDKU&NIUE!rJtQJb+nDCtcg=6@UtEBEU1O}9c8(iZpXi@}E+FhMH}@D5p;Qfv zxM&|j&NKw&VEDY@SUUk+0TrO_r>}B1sTyGbS4)uI=rmn=zV0L6)yT%CWcX7X36@t^ zOc>_3Yx4T{U-ov>&Of7pA-Cn5_o`JLA||(Vr0ZZNXgkRaKHtAGI7#ef`H4ni@s!)X zUq`HB zc>VV3%eL_2>g%)@6bpA#Ed>+c)YG)%f#{qMqERhZ_Iv!UN9LtVQId*c#2!wUyX9nk z9e9Ubc??Xqm(qAmFaC+0MbZn0qMsFuKPztH z9n-wOmd&Uvq^fgVp6BMf%REEr$9P`KhY2R-DbT(G6ceB^j8Y_Iee2XvvPWpAfLQ`6 zN5#pv>)8j-DwL$`m{&dbaDIcvH2Pf4OSyZ@ETv3_Jf*dz$i9#5+9446?AHBg;Y zce@QxZ8Wc`N;3KV+m^NgrCG5qLp~>iysdUiEMwleTBlSLf{$I8ZBNfZco@q^;0` zWSu-N(!1xOx;;=n6Yjkf!APQ80AQP{x;YU+>_r@5EhKs z!;8UyV11I)y>kZeXINYu{wM>jC23$yz!1rbFH!+edO5UD@X^WgxK4(Y)lc%h%@sm+ z7J=9R`jY`>TfCS_M*2?!JV=-6d}QWvuKlvV)nwNX}ES)9n@2li2M zP&7mgcc9G{r0_c(`}hgw{Vjv=!+C3UAn&^Lc=B2Y_SGr&u#0k?#Pz@RL348+kfH7a z^V)u=u@!=`y2xzzLW|6ngknbropw@FcZB;6$ z8V6$FA~CFAl7Z51j|uVLTVqLOW^xgQ*Jk3rg_`#@M1Y@cz7m#(U2h$45#3JH*mf#2 z1$Y83TUk{*M#42ZkUn=f4jw|JE;%tLcbBDh0y=okCfA|adeiIklo@b+Vi5^7I4X1Q z(V*L_3HlP+y6TZFFjJ6wVDck3f27Jv5o6u(JE-K|Chd^7EOvpAFE>-f71NT1kLm)4U_<=(IAZBcR8us{W8F>e1nIW5wNLt>j z`Y{Z=`4g=!Phoe%lVp^0uex`a{3uvxZXz3`6Zhlww&c3CtfqQ|Q^bbRBPKyIE%S{f zgLLTlPZs+);omwZ3wXcJ9OZx-V$l}p0PMaH_fh%EwG>*{o!WPOmYspnFmmgFr@$KKnko%ibz5x~wd__drHhZOFbV`@>fh;72peH0WQ(R*+S- zjmMuj|1eVaAgHij;XK!X2xxo{MGSUe*baK-iFE6>7tw1?u)LtgtN;k05xY~KF<-nV zcT%!>H|=Jywu+1xxjqZ`oIdLJaTwFfxe}f~+cJukJ~AK2Auu^71ey6BX-6?(Gt= z(o8gM-eaOG9iNMI7?5Jj5~6=e^h6*JiklI3+GA(wojD{r@QKz6cYm_M=PN%{2Znc`;D+QgC9`4 z6Hvv2xk{Q!>QJFdVGS`9A%^DbF6!jk4t5Q~1^7c6@uW(QchU=lH(SUuD zQSt!Sh8|pUcB&U#F8UY6aWpf46xJ^@Cc_k~MeGD}j{L6FW`>D(}?r}Oi z_*5yusd{amvoa5reeY#=op*&P$nUkc_0!b7PB9ijk>?S7+vV}Sh_J*ZhlkGG?F^UY zrZSWH5^<{W_bS?HM{6mr?N)uH{Rqd`R|WWuUt6DCCZzi95Zu0`<|nTlyWdi30iJNi9_9D_kw9>43#ngz3M!l*N$I=+}~jpzG2d*>7BpoY@32ZOf+OKM+` z(yB!{u{zt{wCYk+T9>`G%VZcwA8d2#IQ2~DWK3@(BIE`I8o9o-d!HVQOdl~RRkF0PrqrQoQ; zQ>to-m2Vs9bPa!rA0^yc=(QscMqyEi}jTP(%L5-b#$e#J*gcYRtgCvvG3nOtJ-kkd2_g4;fiyH*qTTVP{+@0vhv>Ec(_ z1skOwPpr5RdqRx}%_vTeReZCYY?hm^ydk)Uja}70$osQ$E=aLKBMK%fyd-Ia7wUDN zXC`9%Zr>@z>yS|AXuZ9sz*n2u&9=Fwoe9O4;?I~43>L8Yrr&8u%$zJ{h-PKd+%W!l zZf4))eiGAsS9f5wec59<*@`ElJNd)w_EijN=B`6cq-Oezm_&FnSjz<^t%+9Q}&})}j-Oiiy^MEX0x&^BKH#fL zcXud{lqe9}9h=W%3^XGvNGTcQeZb!zprAE)>0h*`riWrETcRmnM~iMmGZa?p_4WbD zFX7bt9|Gy`caVg{kA$Z$+^_QDyyUzE#}}C8eSNy*JLZyMI%0%Nx#1ZX$}Fo>kLFx` zK2ZZdl!%^Aj9%NUX^822^+Rn@`sS0X^MoCd@S-1A<|q$-(#}f}gqHv;M*Xs!UOwtt zP*igzC;K+sNQWuy>8Q(328XxxYbLukYzg?=>PbXuvdVzaqx2! z;$C(G?X{ln@~6bk`@B?YVrwFdI$awb4yjfiegwN_mLiXbQjZhoaZe_@Cri_alK^cQ zS-F|&#*<6X*zIL(d#dzWmmId$>U_3a}s;ST_s} zZP0=zjcrrJ>V*%Ku+e}JNkN*e8-nVGG_dKFCQR_pFjqY# zeW^B7d~jj?3ST=TqbKQL6f3vm96e!X-s4o+nEOfCwI8E;nN#JqDx|E#*lk!?ujj9L zdob^(+Ntihu)aIh1xBn)PGsN;Y8oQJdLACy7{)-jF_4ryJw8}3#c~91Rao-xfHC(g z_^9CTyFq$b4OORO&WdA0$1fOkmsea`a0)Xk4x+RYSR>L#mXGGP$!?>jIw#pUZ@ZeC z&4l&=j{DUunK1bZ*BhCW>F`##)=z;;xgA`IgVk*CK44`SbIn_5&8C_S2mUeF0)1vN znmQ!u8HSw+J-p<0e#FZ7RzLIwAFd7UPLiI>4mYVtP;k1MYj~$qUyP^6-h%N;=wV_$ z+cmzFw&lvztAXG%?!I7>rC9+p_|9zm9(K6IDx^#TLb}yCx zZ63cew`}mgq9;ohx}q&=epq>47)vuVuj!m0MC!UU#SkxZ>KP2%DN^TN zHXVx>xL+8Q+xI5*Y`3qu@;KXytP8ekl^l0Qt$o>P!>Q=^^Bxk?BPjZYUog5+NNa*k zY*-2I{|{l{lv7ogT@qnXvXOowmanqM%*|!KKQMAiyVev9s)?X-n)}2)b|(bE$d1L9Y@hqzPhvb zTsxex1cnCpAR!$8F@k@z9o)~1Kh?CwBdmv5R}h(B`>b7@9P{IbIFTv#}G8SU{W+fZAk1H(np^mqd|P_ zdkvYE^}_CJYDd=9cjD$W*z-3kClB;|czj$-3)<(RZ9 z1h&0P?CsYR{U1k!)6?VXy-v+FKSt3#w&#!c2dL?~sbE`AkKM!CB@RPUIz7?N1gKkb z_6B`QYy+HCGMe(=G^@d>E>z4lqqpwSGo7hd9Fel9P1^zT-(0bIe@I9D^1SO8!Q}YJ zisO~ELpxizXG7EA6eWX4Sxl@2xkJ*3ew5rOO&9CC=8y0|nZ9oNsm9M8c? z6?Ly$1~()0T-<(`867C(oo9-XKiPsnrs&5D+?(Wk4HPy7R{CHD=$M5`I+?5w?jO$F zlGWG47KmCXkkmjJp7|R{to+h~dPQCoW3pgqO~wpLy4lo0?GY@}m>#PwxIOkoIO?kY zybvW3sk^h6@w|T_%$r#ul^3GdIGtkK80{6|;{N`QLY_l~%5KO0+e92k^AOxvPLQ4h zpe2jPR9IU(xnmo6J-P4g^N2pftVL~Lg_N;lr3%$kslu^dY@<}~Rb2SzHA88@fmk>} zRc=J*vx;=XA`amPPZfGnT-pVYf%5=O+stdFHlhfh%s^06oK5;L z`gDrla*O7sXS3uN3(~lBC8joGho!S}K*sO!o!XLwcLZ5kAmIIcxpPbnw;FNU&8@^c zp=3Vg$IY!=^Gd_+{;(f!?oAMCsPt5`1%47B5YT;fuEbpk?U8aDcst`Q;$^ks zCyJ^L5@Js9xv(bco76rG9~@}Wg=U~HZ2}~c?>MfP7n*d_?pAf~eN;l^9}Wz``OGPA zCZV_pMx(Yn{jR-zgLTV#0;-Vafb4zICe1zMye^>vra>C_2Dm}lvB>CraMkLA#FhD8 zvTLB(S4COV4%={F1I-sqEI1C7R24=ms97m!A7jFw+-;3ft$d34&KZ`wS7!Xt*c;#{ zFVGk2rvJ5GC|*JI{}!LYI(Hb$D-8R|Xr)u&?q(@?P>Y(4yq!gh$lK|ew_W!MbE4}e zOz&)ldo^-g8ONe(*J0ScLp?{nVD_C;b5i8#JVecXY1kQdO6cKw{2AAV%gpm?M;8=$ zMzm%&+RWnZe^G{gj^o2`a%|zVp`zSmg=S;7`$Q(BW-7$0(O*2}$Xuqh(v>{-c(5^? zTT9TEuMFiva67Cfb=Tw-<)h4Erz#BVAmf2gjQ~~T(2M7N;5TOlbC?=ADPfkHy^)gg zTTTe5=>ENqTkSQUnV%sOia_{P(}`0;_jw(tR>S;k2$E2lvvw`a1ikHkSi_KL;mtMS zhOU5^zhlMQy)@36SrCuq4kfYh#~o&UC5}bBYKzxw4>1taRjC{pBz|**rwTcxHf(OX z5)^i)=U{kW<)Jw^UIOq)L@c>z9L(U1U@_>>s-tC=6bxRy%0fp&nZLB|3z9Af{vUd- z5SSI}P-Y#=;Dsw>Wl4;qB{8m+LOtI?LVe`CexWLkeEE3_uL(2yM8&~bBCOX<>as8) zB~(18m2Aw@10hP~+a2rvM0W2yua^X9vl$lH{Fi$5*IMK&S#{m~IOY$P2;J8xQqL3P zV$y?ExIBzH$9PPZMC|E-=}C#Ic}(F&WW~%>9UCz_U|zgWvoI)v4;~BK86`6zwcgxw zHDw2L|E9k*#0R|yKOKgJ0f|qD2~F5ZN)_zk+yU??LueqmA{B_<<7c^K{Ui|DlyN$& zK4J9D1C!M?w0UP8I9HOqkF=?tDi#J)Xc3}0-m-HOlAAlZcdmKbjQdzvakzv9>8P2# zQ2)FF98>OziSMTA(F%`T5na9YzWChqB%4Tmyj`%YRj7Fxo8GouKa>4r3b_v! z#(-~q0-JB%pRsl5Z|Vtd3REvtox>ng#;YLBT8HI9x?|7xlQaQMd;AFq$_;Q@blM0_ zd4rhCF^Dy~(*;eaybczuNqrg*c*I}jbXP8}BW9%mDyY_o)x!;T1;&EG%eX0l08g~_oC<29dpGr`DQ=&;%-T>JpRM!k$k8Wf+zMm zaL)f7Ph{D%;0!QKbQAB%0*9&|;;g(&Xa)SRgYjslZWxJD|2zOmOAV?hg3uA2B-~;P zc6$h5jknGzqI0I)%D?YB%#+E+UV8Qq`EITUBgOOLJ;1yse7Zc4Lfi~7`~Y2+ov8Z= zuZcHm4WoO9I^m1V>C-Qv9f8y$L z4f8WxM*6w2eej@72azNqUm1cVN$PLdA7vg(P*p(3*P8Y)Uril*7c^9XRFmZhyOm7+ z%g@u$_ZB|Hj}58F#ke^TxWysd{c=}o^IEPkaeYE3JOTJO1}b{RhI$NcoISoFTgnl5 z-LjR`U|pR$++*q$9c27K9Hk9$Lj~MMadkULdkJLId93?>xq9L!T$MS!at?-%&ijJN zFfLwc%zgipWJOVj-kp7FiRx8m(&sVR|f`~@r8HWD61Ht4D91YI+<4+JDo|45n zR!xtu{2@WuNs#uSX8#GaaLy|7((;7VlALi+AAUN3Z;ndsxh82$Qw&qP_v2(0Cfb^< z=urfffca@_W|(oJFukg#{fv{JL2mvN{;j2p8>)Rt;Q$GoXi@;AF7(wZuwr_A{9F#U zeVvfnzH(ax_78^g9U!J`oG{sdv&nuGey4;6OthlXFLra!^e=2P6Pvv=ayr5V+{0+R z8>f+2sF&H!Xfq}&qdT2TeoRB(p}^7!w77GZzj7@xIItUNZc0pMRi|AqXy?3)2|TyK zo?0FqkHNgN@0BJe5+cvd>FxOK1K)Vglgag}#8&_6Pa8P(qnQ0{R zf}X|~$$3K=iFGU5M1?mG63 zX-E_|fw}{6sxN09>0h%}_%ANNhxKq7*d8q;(^FoyifRJ(2SF0)AJ$OjuLY|`&h;B2 z&#l%3w45`>Vyj-R5W>;q3*ymb;yJb{s%ntCw{G$v%i-TXh*j3p;0~nMwXT zsy#GyBB+3lVFxJxSN5ptDCtF{!gj9bg+$PeHV%uxZ_-rBCmR1eh7xe)(mGr&xf|xj z<3N{J)?46yb&$O(CYY{i>yTayYT3L8ekvus>ix}m+X&-%z-uKJV{7bQ9Ym~0gZ)hn z5APKnM^yFy3sLC6VEiUp=nM59Wjt2MKJqL8VfrrF6ODPrt!beqZg7Hv5YcYYomfoi zMHG#PznlHJ82sOW0)$=9PFyKxix-T)gy7jX2LT5LqD}d3TFB>Hkk=Kyb-|x7FEkiS zlE|Bc*^YGK6H1&W;+qDS%A2SdZr$-g%N(|Bj^Vz;`dK{xvCX>qXLcEtF(9{JYx;m6 z6AX#B@i?C$w-9Gw|i>jOY%wKMw4zA<8#2=t_Ed#&eXY-Fpe zz@(s|?d&XfT1u`E?;|Q=1m!;2+xn&mfQG<(el}%kUS_Q3v)$nVa$Mj0AzM=f6{BKBeIdV9N8>TH;$Vy<$-mjX6h}AHU0G$?sGX>rxDd4o^$Gs}-4? zl9xWKz1r=nE1U{Cz;%*uE>vGa!9&8NjQp+NbK(Cgc>Y(ZBz1&X!<@FkG8Pi?4FT%E zV!s6vF}x{ueCT<+iIS64tXo_XFH3$U;WFF0&AV&5QBFdwiD( z1LUay(mfk6em>S4s=Ki*h$b5~MYU%lBn7v)NWQ@+H>@C?)|^j7L=(H7PCv26j#T=I zs4W&RG1e(^++MYr6m(vNG@0}%Lr_26`Wlb>@PXo%s$Ssm1R3CU?SejLY5Te5;{K@d zn>h94DoV+=B#?H?-;rDl?4W+WE=Z!2?uEQJm8BmKKK_c2%7eg})}z(=W4lIbUBi2p zZ6zt|X{X7R*C;_Nw(y+E*0>;$f)2sgrFYl^Wg#&mjK*Al6u~e6Ad+ZbA!(wCe`{(9 zldC-8_DUoVs13SZ4#Et-nrcwEe|~BGqk+@OrJSTxWTDL1e#*`K?HxV0ja3yJ80W<$ z8ydd8dEH}ryybePD1)s(-stKWQE!XiBP3lEZmCR{->JGkt|@9JPIB>O&Lqx?lYgxa zr&fN2+)yA0%;{I~xDjK#<$dD}+L)tBngZ8q?(U*&MwMF{b}Q|IWb1-Ykmuz(D?{m^ zCvuNfgGggVogx3Sr5^*g)#9lj5gEHY&P|B+(sF6bhJ^@TUo39Gy=;BGpw%e?AO$~s_NP_IX*(9G6BReonY)mC+a2VEXWS_x6c_JZAOR)gorD2Y>Y{E{tN!bQZ7v2 zZ=gusLR{0S4|qu~G`nz@Lw~UniqxmJ1-rO@?<%B7t#P=?c5wY)?eko(F_S!qba=th zNb}#tN1mPylvB6>iU|G(u|JHyO7eQKZ}aiSy9bs#e|x^RiD}8|6<>)nM*fXhZ7Rym ztzZXwTM>|q5atN&Ncp0G>?ctBhEqWaK^Rjen;Pr!`gfMh*P7`gsTC+nJd*&}k~L(< zm$hSRp!iYi;&NsOm&RmW2||e10KMUJs$S9Aee*MjK?WkM%@T$A4xsI==F#f`%R#Y7 znaug&-$?W93kljIq%IO4@8uh}hNs9rw;No^`D36W&@f8g+I(PHjtk*0fROyzx=}AR zroP>o>Zt-L#~SWlDIr4TQK>+|KZ>?r;P(H&IA7oB(1=s;}$4qt?@i^O3bEpkHzP#vBRIj2$FAH_ncm-O< zWT((48v{6slf?gg$&B%8j6ztA&MDw)NK9+TA7waB{Ive@6H3P*{Cqk5>tuR}MLm~H z;zEK@`5z?I7<;5a9u!440nNTb@m~HNU$PXu-;S2)sVCi~XaW0h=%aA>KYt@e*uaeh zJb>QlkVTU=#~H;RMAcc>S|{!?sb<4s*POK@!V0W(ns|X1YNn!}c&$MEChPAg-_m6K zy@K{X4e5khD}eYwk|xn}w4;;B{G$q-2~VEu@guXm_O1T?V$o6OO5Op8CSl{^={ZjR z5@GI3?IoStU(u##{=%g@ix@TeuSiL6BeNdHGf}aaSbPTp7+TPlL8Ps7f!j>5r1Bcb zp$B1(+uMmibm8u0@tH~TPr1S-?5a`1X>Lc$ch3~as6P~1Aq5p!cDO2t)ZyKb6_8hz z9}1O-YY5}hQK$+I!fvh%0;7!3!(`cCSP3>&xaFeA>t~35kHMMkTfYoz6fdN?>~ctR z+N_tb@6t*e$O_*^EOgwx@N5SrlLTcTBT}07YP}r^F!|fJcdmH&!)KA(fkpDHXXxXqHg-O^dF;> z2A=Ac%U3~6`0dgaw%|`jwQl!ah(-vC%B=0;qoi3S_IFCN`* zZvhAqkSs$bzBe}^5e{4_RwD!dSfu8xFHsBiHVRuw5-%xYNC%tm@ zv34MoN~vye;b4oNnMX@f(KxM48NwV^mgGT*%xD&a34aWz^vMXATgaW;34@&@^umFM z$*QDWjM(wOo`Xh(rsg)7$LSqtivYacnlcrRQ{nG@3{mIz_iDOC86Vu53Fw#SG{jju zaung-A1-xhNbCqc+@>yLgE(mV0ioTc^om&!v|Zzd-Imk;5DX~joW8{0r%X{1oMkco z`UFwyNOobyx!Ps?3e)e&luC1-l1j8;$~ll&^LH5+sGwyH%WyUHGmkgb2(59VlK7I!_>1D zBdA^%A|fEGA_$Qw2rNKlqNJkCNgPAje;qvuTl_eZgj~_t1>s+k<~n^Ixr6FzH#Xjx zqXoFrIbr!$Ut!{0W?tF!!23{lXA5z=Uf7~eY%q6Of7%C6qtxInK(bb2uDAs^F5lQc zNZ<9E*l@^FMt)gR2E2`E+b!Hbs$0=FHg%GaZ8HQy4oRmc?jn0=(X$f(oGfiw)1z)* zrU{uB$qZxYB>4wUfbh;qqGzex&8;WSN^g{FEUZES7dpaE@Qxy_`h0H-jV&0E5xf4k z58cIWwj=xmO^+8$KVI9N2?ojMi+4R0Sm<)`@D~i^N1ghodh}oaTQmBHQc=IOKNaNS zxKE`eh5XEG@PjWrw=kfiX5+dOMOH=zcue-_DzdJ&R?8++L}&o{&u>BnrK6T-_3$2$=4%cgV#jFBT`{2z=9??vgY zqS~a5cq=PG$sqCTNar68`>zj%Zi;-p_Tzy4Q}O3zw*TuxR2!ZH^@gkyWh0JbQ{oeNSFg0x8GaXTC@i1;xMHz86qrz?}g$Lq5 z2aid{toG-(DDiKNO1Y)N;g`~rdY2b!u`GM~GY|;1gE_v;cmRrvSVRV!zIp*d z3WQAP9nU0LsM3!5&xjBY0v5045;j*GPxqL1chvv}KosKc-RsxyWmyTt-uHZ78G|xe z)OKir2J^20|I=d3e=5$bT}w~jHh2tqoLJ=?g&X6Y{kBW%UIU2e1=Sn_D~5M&_iU+u z5|=QKgOBGnOZ}0fe@of3zXd4|L2|2e2FS4|xKn^m<3tPSztg)4w$p_nCu|+Z8IyFP zHmgh7%LK6y+ahJSo$dxl<;Kt%jVSa(-@(VYr7#ED^p}=@Fh>`!1=~+cCY*FFz_FZ;6c$I-<_lWO#o9_3X z$0%qiSH50AqwT+Jv+RzjgOe4uQ$O+_hcT7i>vgYK+SZ@bDb`_IKw0?c{)0?|PoOj*ZfUSUqcQef!6xz$Bv!M6;I7#l>DVzKQo zD6Ry~2yMZPGZu_c3sE)pnF63yoH7*rSlrC-@P1*KsM||21gQ8>gRmk^20<`i-;}KW zNQ%f$)RtY$Sn*XDXLOIntI)5Tk?@D8jc@-R9X&6S1Q)fWBJG!B{9D=Z=Wmyj%Xn^0 zSUlWD>Tx~-9cl(=^ymIAl#k|TK>EW9sO|Dwui)TdaCyXrMBIymAba8==1fJZ;DS0A zs~&aGl%miAD<>FM@by<|H;NoXl@XNsA|4OoAbBQiD^}CdQD!);{Pl6o+($_p0t7!? z1v1q?p4wk;$=`ovR{~;3sC@^65al}cR(SXLV=Ef5jVdlK_9-YR;D_nf*|TRqT_Ko6 z5K7Hbh|3J7dfM7Hqt3hu11LG0_LnDWTQKS{?z$KRF}(%^0GvRIufwQE7JsG+nlA{!QWoBx#3UZb6+K}I-2T!6yZc0zga@u9Y*eP5V>#*~OBwy>}ib~hG3jKV-IwFmoRi+gDYb#Z>l7>y9J zNoS#b3MJ0xv4rOp6&1Z36QgTeQ7gYfmEJ)xF;BVVPQsCq46-@gwC{&F z{N{*}C_J%Lc0guXAvSV5t7a{><2EJTbcc-n*Vk4ehclM93&4MOen|pnfYEsLC+HZlQ@oaJgyLb%9uvJ&SS)+HzG_mI7%xjCuFC0RI*H8;(NQND<&9R; zQm0G326D;4IL~Nx(;_jLAYIF-ynR8EowPIbJWSF`e27NXS8n>y&=b7h#*9aWG#m&!#<-(5*>g<71Q(b!Pn4#n#F&-6-rKCVg$RA0mx z6cx1&@n@BUgv4&wVY9RE$A@}27*(x7M{t1VvRPSK+22WHM~qJ!4X*S}q!l7- z|8(BNB<|uu7dXB0z0Wn7U9sWyZjG!-VGY3|2_a@#8^I@VO*ous!*?0_5;szKr$$mW zy!B54r}ma-8GZ~T8odb@2(Sf{A48V z#C+8JcEMe9h@hj4X+vPzH;hx$GLrQo@nq@XBCia1J7d}(P+zL#N^P(hPk7h!UA!;t z$U_fb>)HQ%Sbs^JrwyZ@pN#wexEb>(ufd?HX7eL!ELbk)sDxydTy*EHEG&|wER^wG zb)8t}phqsV-WGQnCn`E>6|Mj$Z87w%ca23=AD!KT844@!i>e2V~Wgs>|%33A}y9b;wYcG>6wxy~ory80~J z`=poatGlRu0wCkxB*;ENj?8kkDZ zF~tG}`#gp2S4KKMG1YC{0Y&to=-R`ykE_3OCBSV^4my?cCRSR8eths{ z+uMpNQt`w}q|lYj%#l=AQQqu(mXu~J>+Nl*ZiuNN(caqDE>0u19zDeEf}c*h6ym5D zj6gnhc5=YBLZAeb{Zki@77N`$O>=qS*)1ykOQofGnOu8$a~^nKBth{_Jb=I0S8X$g zaorIX2MiYs#9(q|?VSz(WkM@Qbyc_L-U(GkDjen(F5E?wUBcRt^f}&s>-hF#i(DI< z27;;47!#~guE4MB9hit&7@_SG|Jgdf>gz^s;#2|V@`t`H9D2x<+R?77xWPoYj=J$y z!z(c2v;FqoahXP$;_y=KXh1{CiE}hxt6O(X9u5dXh6{0f4dqbzByRdE@WKO5`pK790{rq`Jj00A>+_Pb!QXS(zrGIg2e5mJpn}I4 z7bR?Sg5ic$XWqTuIxjbOrCAR2&kk0_7$nA^RdQ%T#UE<5IAPD3pNEe!Laj6f#4r}s z4~^e*-YeU9uaMz<mn`fW);hCpxWa6ZHX3K1CV9eyZF(tMHDUpTf=dM8z zcMxVSP5Y9gh7Cg>qCROXDBe?_sDLbV2 zXm~p#|2;2#Qg&kJ`&9>Z4XKY2uS_CEs58%@4(m_xEh~DDq>`l>GW>hCu_Detji@+8K`baHCPdw1OWqC~J(~NG# z7>YP|Jbm`8Zwh`UE0Yfrc$o#{g`{K7FDj{Ly4(G$WDgdq_GY`daYLROT7yoD4YUJ? ztv9_-31Uxcm1!5qQv055PfEqECi>&6b5LJtovuk-XIWT7TrjP<8OQ7~`_8F=uMt@` zRIX1L_~;B?g4e(Xe?o_~AeG&?bBD#&0|S_9q&!=#@mx5iW%XrVBxbh8ttuESeNgW5 zNE!p`LQgw!Y)Ogpd2erahKW(onQ!gfvGHQFpSGPM22BCzBMgi0FV~l1Q6Qv0dLxln z?IiJkUtsX_w}e3*6NT4k;-jGU`yc%C+nFoDxGD??gy}GO3K0Hmj0uF>Oor&W8X~U&2?pOdsT*jPn;R#6qt!Kbt=e zDr&1l?HpB*gLeRM3hIn-eWT+EMm@18H2|SDSK`I%;EndA{92F9kLl&ZRC!Z z?YYp%#41RIf2-Ux*3WOkO_>RCPM&3d@0|oxDkflM5-03scPaz;jT1nK;9NGd3e-4E zwTk5`W_cln%iRJ})?jqwnTU{e@Y2dHw`2I@o|TjwKncVFFnE=&t}fpW+F@D6!kTT# zxtoZ+uZi-wdr#7n7!0?JcKlVFJ&-&E<@BM8amsP(w6mKXl}G>v0NPtxvmuNe{nvC;xnzTFKBjYkV=E1XVR0T6m$? zk>2?U`vs=hE5C%Eir`&tT2}*6)5;AWiChtq@+{{@UV~^f7~kO|($J{_|G<7*Fto`+ zydUbAJsLn3fsHOjYmgBIu)e~mB7&RH>CWm`rR<O#!e$vSi)fBzGqcE!KV=0Z1;=;@SswmRR89@H%AnffeM^3|cd`nUrhL35H4Jc2EXS`cuZLJysC7P()EVLp{x)urO=XafWN_D?-v6~gX6j_jg{F2ghIz&cR*5?e^7SYT{Yi=A9 z_Nf`gaGZnjW>nO1RTQ1~5U%e%5V|@`!STOEi771KiBQ!nyY3stvZ^OLKuY<;j&O~wb?k4y9D%^Bo!18SL5%xUM1rS{tf(~#0)zV67x`+annrkR6jja+af!@#_!9_ zn}mv&{dgzlKWrXI8arS&<=O((R!2(Qlq4lR?fU9f(7LX^m<15n=4t;OQ|GGbQB>x-y&ywo*XlLa}Dh;;_q0E%sTZzDa6wnmu3lyV{eWUFLFfjwzN^YZl zB-=;C_KbB$Aj3h0=bnepgLB^V!OAgk=In*K6X$a{NRytlx1<#SI58P6cPaH3iuT1H}1YWrZJ z&grj)E`&yXE312|^%6&fk0V3&#iIu%tpe82s#NaWf#|;j=l{OM{16WtlDGX<+S9$< zLrLgJh|R8rfP-$}ksD^t>_{M_-)nLs-#EF)9v7n{F+c&AiXj5ofA@2A^pbZUW*>a9 z6YZ=tK04hGmio{J-L%My|9(UR-`=;rwQYx&PR+(ok4I!AMcb-$*wn8Sr* zdLBTfV$f%>-`R%s%~O*D9u3^(5n@^h8j(+f-rmbz$ZqWO6g`Onp8Y*#4-XGaG7p7O zR4$=hM^Zq?Pwl#khZ}1Bo`H}moTSiuIYIz0TIF^I z-7J`(n@v`JFZ!`!1{DWt^>RC!+mpnBm2cA1a~=?I-P6j&4LT<&v`~!(VF}P1@no`9 z43h=^b!)>}S~+nconIf~Q~HBb^Pd;|$D1#-B^YdD`l-*qMCHuzLXhlY2_qMXv#>34 z`PDOE(SC8U#B5{Np8;qU)p@4dsh?%VM32t~OYw4`X+RI*B;K}Dir3x%>rlx*5l z8dPLdr0kKMQHdhSmRTuV$lky6h05r@pZoqD-|v6l=Qy6@@t}|Q`!%j{UgvdQ5P!)G zuh!L?=@(66Q{KIf_x53@-^CeRUE+L}dNb6J#xk8%BO{8Q>|2G#X=jo&iiO$^U}j;P zPEE9fd~MD2t9@IgaL7rWfWI$% z2@9n@5oGq*m%r~9l)963xpN7H8uEZ!1e*s}$p2K^m;H~V==D)zxW57IxCWFQmU4Db zx*-7b!C>WmMBoHoOrW!+B9WcxsIJ=_@d0_vYv#pfxS`AA4H2r{#!JNb3x;qGzKRd` zUPp)Tb}}^y!`IfIAa$k0N>H$GC3&N6vxwfK@S!mo7QI}P)V#r!EgwcKsRMSzTa!Cu z17LI*32_2*OW%MbdW%`r(()OG?;JwPwvr4`M%h0TlH@$_5LTO%y6FH6)|AXz`+2n)`jrCSyxEoS04uVGg(=Pd`t>Cn(g&1ea8R-hAd-y=d_f^AMg-1af2#DmHo*G%Z{nn%V%?p%$u6JcGEVS>nfe~{@LfheHR+U> zBIL&xN}!0j1$f(Ja=NZMS#qA{*Z`P3=G)fHkNl2#Z?r8R9=>2=3cqo1H^7B$#)Ft* z%m%?m0ujMkttk||KFUjS{Y*uYSoCZ?g`%O}C)sWONK}c|LVGwmLlNuAL&%s*d^CE28uN$nuk52v>uL&76FsLpG}iW`bqkh64;o-BZZfDz@IV*t*>x5%JLQ3a0siv#3lu9p8Nm0bWAx#F zI^XqVhObT7w86=YD<_Fa67MWzaN7)*&O67Ea-DrF1In2r3JfS&k-iN!jxWeQj=o=; z-1D#r2OTr_IPm5X%LERR?2}X?(A*Yv9agle<|KlJW^K}90hxzO=l7Cigy2NWXU(2{ zo78sU;Qi56d~%S)%zy*4&FSMVUcLH^r9Xf|4a__kfz?d>4+(qf2fEr4EBg3HgNK~C zV5&;%rK39O3W}AdVJpzPJ%aRa3*`6E-z_lgMgG>54^bzR(;V6uq9fr(BSWTlkybqP z5JraOGCdQt{-N_0(-n!JMdaaEH2IaF-|}Iw!LD^$K?v)K<3A7!8g(Qp64nr#&?N`_ zIWbTu6e^+7h_AA;@~A_fMUp~?jgPRY1}N<%Vnt%ByV{6|9}&uXMP@f6P9g2L{ur3t zWlle*{l|}+q{$(}vJwIRNxHvepRnYdI?pd7N7Uu>eJI{dV3JwoHGe;71J6^lD zVti0X%$y04r%>Zrb<2z>*znOjwx-HchP@(_L3a#`7eCy(i0RZ}H=vVQ@FXK$-SXI- zJ#pzsc{zzMM9lRn)E3iP7%V?84)>=}A%M$?moH!XL5H^{Cgy=LnZP%pvE$^M*W$D# z2^b+ka=ZDWW0hCJ=#o-`O8M1@#!kp(eX4Mt(+(Yib~|u1?TOn>)dySPaBiI ze3ALBB69%`KvL%p8hnIFCFrRIm`2uLF0nW5CK>{jb41P`H_$7BLMfWFGHSAi&vfI-2V`p?kHc!M^s z?1+-B_Y*X8(b)kLz8eh3G;?|$!;FZaq~!supDvNm>r3EFcOT&qh{HT6Z3!J-3fs*v z7;a}Li5|tXuhx&C@5D<6iqafap->}4J{12vGD2AY3TlQgml;uRTrQ~92o`T%yu~rJ zNxFemp+PuVG^LiKF3!o2iusjS2UEjhe%;Oz&<}7`uhBxx5^li{7q)8unrJUl+QInzfM#-9lvB?*M#BAbbPr z2wm~$FLxz02+aC2IQjL>JW-K7OP^q{4Mvahy`^4?!9}xK#MB9Q1!_d=J5S8@`+8SzY#slJYyLzKMo%Sxjk*8ug5S-Xb&7?Oi)%fZ!jiwYzkkZ=>mZ=1%06WLh-B zqlE!C^jy=R+t#do9duit;enk57$`M^sv)6YkWT$A@6rAygY0m>id)U-rQM}AF7B#c zmWs(#UN2)Y2BpC=#c_Cu|74T0g$dvaI9Q%srw}mv$$;sbf-3YSs8(9xjsTq&p_cQ9 z1}?ZU=+%I(ZbLx^)xf~oE9*XUDLowp=5Z~76yHLt#Y5QO}P^i4NgkaI>zI4y>@=($|i4UC){ zKHF`qR*@mFXV3Q6&oE_7mUQv?*n;Cvrl~;GTAF6gaw8Y~z8YA@JizUQw8@x)Lyq`+ z`ypO$k%C)9jGxI28k$US{=azLIe(poiYjemM#PV?(%zD0{a2^v?)p1M#@}akZ~%F` z$qt+jP@8hxcad?Iob3-|juBSby~g*!6-vbDL(OzG2gjQ|J!sip|Es&gvVJA$u0RV# zBwz^((UYP$-zU(ELc|xYH81X4k26Z9Aq^@^V^!@Qk4OU-b@#+BjszebOC>rn+~`{eb^upA;c) zcsB@lm`;o6$Yq5p2^x^6c?g9<0~wLhqUN14254XC@0S4=R$uN+BJ=bYmlyJK5vn|3 z+B~Y``s?l}paF;t9zn=T@X4NTjFmaMEWes*M=3Bhk(?w1gu9W9Z)w>0R%hPdS%iN` z5S$VTu=bt%-N0m@Y{o#Gu+EBm=Vb`W&$;&6D%Tpyh%*rayMo{o+@;Jki4+BzPiK}S5mt@T+DjxNBTxtJd2Y!N+zkg*lf}DkN z^QK&bH;nMvx|em2xYnbXqqu49w2D=(nS>en#ZY5hL|*7dKqp@MWbm5i$-fBN2T)-@DrKp#oqzwx zsZ((y)pc(&)AcaXGGsI3dccnu(;9-_2vLaM4KRRQ&OO}8F(`}>!Osm2^fRI_pSMrO z9BF zpR%qtto!Q?)6vC&;J@@D(@Y&U;h+`j$&7TQ)jg^E_A803ObN>|40aUu)nXp z1!LBCoG}gFkIq*RJug#HQI%xegj(QbXpzO*VHWYjt*S1~3P6HUVqOVLBh+kM9=8uv zUWS%V&9LT#6y;>Q_=;GX1Nu+g6_AgtCQJuht+(K^y*r$g-grN{&S53On3IE~1Afap zD7UX56JIudZ)q_l%@$3m`%ra@3V7=w7(`o{*Cl9~#YIZYuC82bMLW)WH`12X-5zR%G?a98JAil}I?-?yk=Z_!K;q|2PW2A%@civf_V==0I|JM_sxx3uq9^W2$ z2ol}MO(u;NAPI2Jn>UZtlW1qobcNEv8m{!Bw}r=AKE@W7MAYD z;f!~wb9Zr*qa*F%Fy-h>%qu5alRPpZs^eyWCTb7vskSXi>^R6b5d&XI-smHoz<%YL z^yq7BzL^SU##gk^N9BY7)P2r!QZwSnIrU__j|Is31%}JX&<;rggreYa3$t{0A(m?5 zJD|OMW*ljZF^Rak=jCfGX#C>4KgZgUCu@~(CVi)fxz+&W)b9c|c^kOvZd6hDpIrt9 z1HvPzZ{c{poxSBK%>@Zn!irbS>-bN>ud4>x%R8F~S2{`Dq$Ei&kqr;+1~rm&*xlTG zebgY$WNp=>MC-&YGIp1}zpsz9=zPs|%X+$%z7p}UKnBY6t1(B~?%7`(KB*i&e0W3^ zLWk%_kjtNX1M?C#ljg!4b*rOZ{A9;#B#p-#^4?Y!JAluLGEo32JJy!wZ6(Gxdx{O+d;lI2c!Q=BKk zx`2vMk^8RY^*z?$#bPs;6`>bLF^H$yYDoS8SRz#ddY7ubn>C?#)K7|Zb{hTF+T(D9?>fg6`f&Jznb)rkqi`}6w>TPIrFLJ zzL6kL==)lWV+k3iJUdZf_T+v}=h-M=E1p-IPcu5+Z)yxO`TmYvX9OEKNcfHIO|OBO~)u7-y{K^JHiT!lVxXp-D8f_^L^D3g9=q~Mu=$z+Kn z-mPQa*6D0T4$p?L%$RhyVLwD?$TU>}{k$E71{mP~A=QJ$Gc?kB?y18 zPlNOPiZR1qT3Xf;MFBF{`(z;*Pe=wBf3HP17}1VKntK~%^(;)A=0FLU-*L#U{S?qr zom3}^y%lxppoV1+t-`(U$OZ42mX7X=3N}NZur`@k^Ccg{Z*b?YUBNK>{$Cd9>ap^URbeGPHt_(R#Gb&Ukou3F@>0TbWdu8*$-1*3ISx2JA)%0O;Om z$`^dI60IHORS4tBjP%f!lAS~|4J2tc*X>y-RQdrbpAScK;ZVk-69Tk*(nwPo<@D*z5&H6l z*b1-oKAb5}!(mv9>UUeGrwH_9aV1>9^c{oyNf)BS?xKK0Dw=y!NIZhJ4v(XRcGSfI zQG)O?`<~VZ_G)Kh1vKhn zv1S3$*{hNraNUnlCJ1o&fzJhwUmn#0L7pAkw~s|ArwGZMm;M!|)UM}Yx=*MFM4hbu zrc=o6QbSfHnCFeM;?uz9SCl_yXN>%EgpMgfq2_X85sxw}e91LoEkiPS@WHloJ1#t3 z@esP9vh@d_VBQ@kd0cAcgPqD)kIg6R?}1CTP`JJ&h>FgvbhCMH z>L^{DRe+@5KX@A-V; zF)#g4PmQOkAA$rnR~b{aZ!pV|CKa0qmCW7<=;sK34q(eO)2d3bWJrEKMfxLEt7|v9 zO>qYaMnDLIh<*ep6jfKG21ppzr@vl zCsvJhzJY?k%lnVEYQfX7&6eq&t!Q7dfp~gS-gS0%KG|=h-c5zOa0?CQZ_#ZA20vQ; zBZwZ;{Y9t>Vto4@z(l(s8$jl9)`&dt;L(o}@`HsOq_^ZCbf*YfbsOf1=`KpKBXfRN z)en++O;3kFHT{YTM{Fbb(`ie?3H_Q3*mx?8Iu{f`{Nh6%9D+tkbWZVyVA8@rf-!?D zl7(8ck+?>*fda-3UWgY5<*xq2gXvc;jvN-JowLyI1A61Va>#NCOv~qjxcYo1CaxPd zZtN}of+F#~aPASzmV(Kmr zX#%Ur7{hZ(yZ{?i1GfAtUC6K2oJ1z@6=Rh`9Rd5~$3pf8Yv}bo>`}1-31RpMhQTka5RT#w-=LMtxamGC;ygV9J+AVb!<)9EA**BYgQ2Rm$O9<6s(Ou zReepEq=GeB>wjh#|MKPx5U6G7((+maXw1&&<{295Hx(7wE{7>NPpuCaZR?EQi^c(< zqsLKr&yP)Rivg}q8+uk-V&{=8|Ejtbf0DdjG8{Q+n zfK`v^7lC$MPfyS4AUQ5b<2DdddxH0~8gCM2)h6(BPmX!iSGc-kUdFivWby@x4&HiS zu3luarhhfplzQhsT9r@mhzsRrjBxGJD5E;c2ohx2=Lwx~T>POMlriXYUfn09$qR$B zh<<3-6Or@gYlLz~Y_GDeD@;S1)^UkPk&zN6#tDgEF%vE>OTWCT9E;(~F$ zLG-;6(L~~o#qun}WhM$WX?ZvCkd3$z9~YF7H5Zf^mFsq*3P7- zk7l`oUcn3~y=lADSIKtFN`vaSR&j1h09tyaA!}ip{8a~FV5CLuUP42ET1Dplq@)Cs z(ZS7S>^L(hb_I%D+g^160g>abTWqpTDY?oENZ3uL55e+zqy$k>Q}2d!&}v{v^>3xC z{>kS}+Bl~yf)s9Czl`;h3MDPI5mDbKovqq+p1aTh(7OxzXfIwMTsT}x`}&jgIRpE=y0<*qe=m}W5` zW=*N3*CK~74pChVTL{R2!>K-ST;#!s;<-9k-)*t__e2aELVuUCb!Qu?( zs&`z8CVl6W!FfRiVVYG`2cZkPjpTp=SAnE?%;n9IR>Y4fyT5W=N_ zLcXh7*3mK#V7-}vT2+Rxqs3eBSvL-08o}MQEBoO9zb188S_G;Af+rn6dY&#@H3==` zg;vBB8XE|s6mYh3C)w3cl~DZqEX0fmc7+nbuFb?Lkmm0^2zsrG(0MywKlkCekTHa# zV->X#%8v(NNsgB0xrPE_@({jd@Ftybdb%G}(ex8p6-c0e=?I!Jl4Kp#u#XCKIX8Ce z0%l=|#);~5dH|Ba%h+^PP}76cHKR+&GmLf!N^qxTV7o}8tN+$ZG?vsPy1rLY}isVtYwsvP}UmvG3W< zsE&MOF~}W!MO;)HNXuk=hk=Qq(;bJP%}}vbNA~01Xr%wq9ojWpjbgA(78+$~iN+;P8Rs zFip8=eNvwVfO^xQ+23zPM)%qsl9Ko4^6)CQx>w&ZgT|>E#C7#5t9)B5!Y|X9F5ujs zR$kc64b7Tj&e3r;q3RnGlRK$bvDmdb_a0x*i;c`%%a!{n3{cRUznbz+qUoN?G`aV& zemmyA-2|m6;Ov=%$!BhSGBMuR;P)gRBO~F?=UFgalGS1+;;GA~1wJplcW_`` zy&bLuYU|z67H0~4@Od#Bwj(!g{t3QQ6}Vo5B0*bM*L&tSSLct$kBewmL$ap=&He@I z!XaN4hYQ;GS+-xRy|h3Hg%z2{38j!F`>Z&Uk%hK>)dtC2mJCY)H{-QH=+KOe!#~GIyA~0Nb{{+CN z+;HBYa!5xKc_IrdQi^~e@84gg$$@q1GUDno6m<8_^n2u_twqGsb>Vv-R(%ch*UNoK zXjD%z8e+>!6ttQlb3@#-PmnP^vxq#=iebqY_G<973^BbDz5Ii4ScB(*RV!BlMfV1< zPF&Rvai=|FA>9b|i_o;dmb@+NU&RSl-qgY9XUydVk8yc+rAlrhY& zBAHY64I(bz4{CK{a|sKL%JWkZdY|q%FeEB;lzl!Smlve8)#f8J+VpMrA-)y`qZy{UP^3fM37B_@ayi>vQLU^# zp@Va@wI&$BPn1{}btmatCd5CN@p3Z!hPd8*r3_2z@<9LLnzt$tv{Oe8>#nm!Nnz%! zS+`JsEzJg+qyP}1Ht>7X$(`4jbT6P)W2CS?c$cPL36j_liZNvY>D~L8Nk!?m>W9;* zHcO@|P^b~BO?&;gLcGIZLRPZYs;BGW3^R!KGns*4ot-x~2<_K0Fe&ez_I+dSqt;Oc z-GvJ(xVxXd-AP}r4$ehN03o}{0S;x7{|kJa~kF0H>_Blw|_hj zKXDQdAB|Q}k-3CKB2)KjACrHWLj68OW88o_2%Vg8Nw#BH)(t{SSnlHE<3n4VSIlGI zeEAR=8a?BzzkpVjjWHP5N$PayP-+dy zFShRX`7dtJgLGrCy|X;+R7-5oM#JnqovO*M|?>aoohMs^$Kbd`nq~$W0QiSA7)DkQPu(m&7nc+ZHMy6MY zI}Z6h&K~WN|M|ZEbQMWFL0EbR_?DiLVy&hl)9t@krI^yd^eD9b_>mcRMptM22f02a znBKvx*IN0%I6CU9!>w90Xy?ok4uR{buv0FjixbD34ZYU03aDhWk-}`e?H}m%1l!Ma z=QS)Yo?}e}erkpYPoF5j;=iPE*AML3n(3cV1dD7mR4bbC715naKnCeQ#KpYuH0Z?& z!qm0*c5t#owW?mmSkB1j=92HA%m({Dhubg5Bpk0K0E;T^RHV)s1^j zK@bp?wOsoFAQ{8^hsD;sHsS!v&%V1aW?#x|SdkFDgg`Kuuaem`-ExkvMT9gAY61&j z{>a7=d7OPmhAY-TwLdQw>b#*(v)-Sd^CuU8cPvjYX?_pFQIA1x1Yw)LjeY>4ARCS$ zFa7H>qzPGZc6N4tyKbz!36?H#T6K*(pbYyiX9r>`{;-{)v+0;cPX=SCRMk5<&83~4 zuOsRj_Yyb=n8NxCf@XUh-{<)Rp#k$__IUM6GOM< z#SP4Zb9;aUWMBBgIeLWRq9h1UZ1p!cC5=lKhhjosFnQb*cXZe;1o6xIip1FXkk9n~T>4p~E#z zr4q5rZqOw+E*C|Y`WC^(8RmQm>~yndTN<#(_s`C-mnQS+F`3~p@Y?UY2>DzHj1*2} zyT@w-R$6Rl)gUpKzLwYdlK4QqRx=odrnSS7eS}r~uJOT;!wyYu!Bg_vV+G!1Lf51` zRQ|!~!A*Tns`FX5+FvUyeu2UjcV(-Vy%+IzSCr)klmlqX~_bT`mT(3|` znFEPEuUi2QZ7N6t_gs9SP-$JejDaccXf_i==>jGu1M$b59kq$5JABLl+t(j2_b|-y zEQ@%<2IuyFAD*#pAa;I<;-&Rdph73w9*n%2b2SZsmZbO-8N|qA(bOjmN+zSv)z#WF z_UK)dLZ{NZ)t)HBk@8loZAaY-WDah)x|e8_)+h?hBpv8`Dby|%7JuR$bh5z)10iOI z&AAD4%U1H7#!xSnW^CY%)k4TL6oF7i6u0*}b4D05Z&!qp@sZ+ur$Bt{8e3CW`BcT@ zU5iSsaLZ{!8X@e(ja^z#^q1<*>61wWZXZaKQ+I!vjC9u$OdNX^(-<0h9qqpI3pC~4^X_A@iX)Z-=Z6w5Yc;QqV? zL!NZ9zQsW0^>sO?44$<6|3eNs#iMuwn003l=C_zk^GT&Au*w-p9XdWje)*PW=_KJX zYc}R^Q_1O(xe^4fBoH^l?89ytX|M}3Sx9I)f5{U4zYISV!G*5D>WxgmE6OFVE(_@F z7YwvtPd^PF$BHfj^`Q}gCx{6Aq!cc%#{5}796gc=>LU=bFQ*%jwoo2pA+cVg!j{c% zIF9lwzfFf8k!Qk8@K*=(=Ppe8>kgNKH0krewThIB-4-y`U!jbO;?MpH>8?tYGecl0 zvyfcLRxzbNr0^kac03@EUoGZpqc-a8>&rw43TIySPR=njRvO-hHWn82ax=r?#T<|% zc6sG~)BZeUAfOk;^FI0MPsa4-2M&34oZEzF`UR1o9f$ka%Ru#gl~Ach2bt-fxB_6a zUkP{sp3^-?z+dYqUiWD(dj&-9{(0ZeYKpnxoI!eZZZwAaJ?;xq{ll#L`uxK(h!>`< z&fBOjvS{ouPc*u}GmnX1v9^;NXkB#q#^f9LDHj2$b;pkly*i`@u!kdXh3h3Mgo=jS zSb|Xtk^KlbmTJZf&+SaUL{mH-n89XD3OSjC$wM-5Kko+dD$=MTzcU_mjdN(VgFXVk zWu3f5@R%5ZXBn=60L${?Ii-fZ-6_MyFE;>0WdpJ&oDuBG7ZioY^ITsZ-m$(f(P%{e zCwfssY>rSsm`^xb-*W&ZV9bE)CB?H3M&H1d-x=cvI-ARi&&u`o_P!+?zZsawQJIXs z4n_`n3E~K~M^Jh2cW_h$rq!T!7196`q2vY$dcwaYQJ5@C*~UaGrdk=Z`_B`0`X`tN>i+Q3|#8hMgAv9!^g z$ZnA<9$hdj4vxJ?LvWuB$b3muWl0vbF5o0cAvocK($(-+oH#P$gyFgW19h8zoVweD z(S-}#d@{I|cko^vF$B$x+K>@_7W>D@X^wwwd1n0iFO6fu|mq<ZCW3db|a(M4(ZvM}bH zz=lPF7U}UgI5qe-jZtBkrY0ilJ73KJz#W-kwMOOM7#B%r8%2g0JH3|q?0b=q^Z%)b zV-S&cCOozmWejAVY8Z8H)XI`ImM0?b{;{J1v2(g%ZSXEQ7glsxlB!0zCyq~PPE3GU|ht*H#u>Py0j z2BJQQqk)l7=*t=qACS8|wApXC%5d791OU&DK!$~`?5NX?y(?MWHYL}LfB7#{4BAl%vT_xHCCp#V6;X|{?pbi$23_r_l<)xoi zknmq{ziG||ATV;D*rHEA(PCn6)FEc*enm^aD1`XGpl>?-GfODZigpJD%Deq2DGIT4 z#h47EIRNxhpjM#c%Fz8A-N6?dJZ(bdyBKd$*hLI>P*Hs4p9+^v<)-xzD*jgvS&J?7bhj7EFN zu8indnA4jQxTPZZSBFZ-?eFwxm05}bbE1~BbMsuYNi}Ei+;?hBRW_O7_*iPB6N^+m ztPRZWXe$H09vKA3=2Ww_2NRv4!b6`10bo`+ZK}jjz zdy=|=!=>z+%YuBJxH#grZ#fo{7}tH@{(WUjyLZ8s!l1@loa0sJ*aLj(6=E}6L!Oy_ zn-<#9E@Ut4=VPyC6d?)5fLQ;%XbfkrU7^0krsE<8?j_&V4-Q#KNd^WitxwF9a+*d# zNi}mWtHkIRw;?lAtf?vX`pMaeKOny$acLH#WUAt&rK(~R{?T04@Eg=rM#eg3`EFNH z5xJ@Fc5eQYD2YOe;LKY-;8%!zb>05M5W2^+5=<50U#^<8b3w_)~}F1HhRQBW(Ywa~20H8h#jsCpd)|h9wEvo_tac+N95x>F5&+EZrbsKf_2tEOwyv zg7~}Iz0r?5gR?BsGo7{TA3k+Y8R`uCX4CleibM#NkQ8LgchPxnf)rRIGb^-15}S@y zSf%t=s1_c0cP1|<6xa4x7~@MpR8)D4xX+m?B=2|CiVRFppfL+!PIgQ8tBT+M;kb=) zbfMx$s~xDZM9HV?+z|`Fu2w3RQ`U=a9`rUIeaqilmbu-x!Ewi<&0PDbCv6Ir!#Qu^ z+;D}`!6=*Yn>?5G11t7R@J$SN!a_>Q7z1aWna=YUoPsiT)VeE~yn_!qP#$;$(r9lo zyyk0E@O{oHg_tZAvpdywp;=*iqKjBx-7X4MH#HwI`y&uM_?VHI8jRxQF$xy9&&z`n z9lrHg4!DSZzX)q(9c_rIJoZOA>T)A+6z}LM@u1Nk&l7yemybdX?b;VHUx^=sICBd% zMC`4WpD5V(u`Qa~sPNlItrK9_FVTYDV(7xh3i)4zV7Nfsk38myPR!Z^VjvBhf7m)B zB;O*Pmi_)*jNi0!2tnV&hCG&yA7={aIA2>3uAsjevnl*&cY}zyE(y=?ovSj?m}OICGW(wA33FsYvsei#~&Wh zoxGnjgM%dK7Dmitl|KXrD|%YOdX}G45M&>Wi-w!gL3$8kB3Eo(p4kicQ9 z9t_tLap=v`Oh0^KnY&lJXVn0k1MbnWmSRRUC5ObOg8~?3&cL?kVvBu?Z6W3J}m>Hd;v8Xlz8>6I2-edZF6w z_T~T~O2Y*upt`%wvO(`qH7G#WiDuTUzcj#bHj4k|9+AIS^tJPh<#(xR73mo?3|Hui zX+ig8J@l%YI`Qkrs^+-iP}u8ED~6p7+I?{JmeMqbzxwN*b>GKw#0ori+=HP-2zKgl zVf;RYbXaVpKJiiLxkTZpElKXLXff&8ojqn3ocn~!b*Oa{^${Z=C7pd4_2}ui40)Ozba8qpI)r>U zp!{YAi=8X=Nk8%On0cc{-f+>bt(^7!o6q?|3)-%kJyMXjUQGb0$8S?2t?|bPV8+K}aa|(eZTS3mbLuf0Rum&Z1l=b4L;JB|CiBYI6 zy;iZ~w>|vj%M8}AY6U6op&5Ja@xQ_0KtZAQx@r~o@0FLArsRCyV5r%zN;(mUn@R&G ze;z3B2I^2VYeSE+u0M7ddOzWMS<(-*z$v_fa61FJSkwui<@_Mr%0DxVSx3241~T92 zH~Gpp*B{fOtCIel#p%S|xtW@Mucr|2tWVOPufU#Z?D5(psHsy3w`~Ep&G_W*=~xI7 z-lFCpLTF@m?Vu-RTwY4z_XNYwm-$}YQ(6?>S$bi(uQGD-3%y<&uJg7wLay6!Fyk?m z(E`VmVAn|t#wy`^n097~f%2dt@5H4fZhZ-{U)42S*M9vg2@0y^Hgyj~8lF5V5@`n| zYiU*Qdn>x|gi?Zd6yN37l@=2@ch)~O1RGuaol5@oG`0jokd8##FhAuaZJSBY;MA?6B{^%FZG~^6y zjaG}8{1XbwZ#?+X{lp-l-~Z~|yf|>TkD%ZjVfOi@!7P~D?d4GXHLrVcVd2rk(f6Yb zQWfDqj=Vfed-{-p{xRiVvU0=A!{p9!Zf+!qIb3`ON(I{3%C_8{EU#fXqFO^Nhy8Lu zKgE>po zRK?vf*$-)d*^ZG&Oc1MWGV}MN|9ZdI+<2{1w%WSCzVrKwhSm7lUUeFVpV8i2R(Tm` zz}By!7rz8K&{gbahPUnaC3r4l-WG({@WRrDd*(#9x*kNuXW{_>%WxZgBNCUBS8uVV9(!Gy2PKvuV0%_LqI z{qt;u$aul@1ey!;xR=SiP#66kSKY^x$dQB1ylou8G4b8x+3~_BzVP4ubwq!Ckt(%d zZg}**c-hIZEEY=Z6;6GrSxo4ji550f7F_iB6VKurtWoGVf~y43XfJk(TN5X49&WE^NE z1&Juu&}XJY1HJ2kf!SR~5n!hcwgFc{1Au(CixtLj%_@^#q2~5k4g9Ao)Bh4-#Z?LG zR5==Lfw+CC|E!5V%K0?e@Wn}&CZ2|pYx3ayTP0o$Tt3kN7mArhEsk?cp>W*B{auP_=n-|ao~#Xy}v|N zq=r*z!;moHug#px5r-E%9h*uyeuKx}Hdqm>?>T9O$sd&XyVchrtG}e&k}43}b6j-N zM&}Wm+qdN)#iWPC2lkRPP*oW?)ou7X)2hRz(FY&-HrbqYt}#OsL2eN7tyXi@%|Wny6o7)-^cZGJ0M&*=?des(ZYv#T&}dx~veEqGlUOB{>XyAF zf+s&dR)`*CuUaQo!=tJy(5XCg^t0rJapF(3^JDbb#EU24l$y?nC~kdpw&X-hJhG}& zf1P}1$Ok0LSkNf(1{^HTi5HrdhNuc;&rvDVRA};OgEs3(vOp|s)lq!RLX8^Cz zrB_{gEQu!hTpq&4%q;JQ1f){dZPDY-GwiUN+IMR0W+qZl@w~>4A z%!I{_=l#R$GKRtCd}+SsHqoa}S;1e6G%=rJx}WMgRPj(VtgEnLxm~`@dW)gj-sCMY z#>-NAzw;HQ4^%nsy_V?heY2(PxnzD^TUGZ(Ay2`e+}3JInJ=epD=+F>h-Y2DzPMCW z$hdO#@IX&cRJCDZ?3J}go+rl@*lUYl79AQooLrs!AnB2$n}w-O(_l|l&ywCY9Lg0g z&S zF7@+fbe3}stIwO4q1kU|_oMfRk!BRzwQBbb-B)8gg*6{FMlM)j{_@1yv@dIhH<<#I z5cIlVv-#Wkw)Mwy^NdRR;ML5Gu8WujtTj-rC|^|3ee`*2<>u~LqvuivCtmlXqU6-g zyL}1!%CGOo|L5nQdJTM^kvCFI35Eg+&U~=!Cl`yZODL{oU6A$Zw!ZJsF4J54^A?E9 zd9|z9!C3az9uDZb++1d=Xzamw(Je-bTXv|a%%XbO_EU&aO9N$$m|DV0RRP22nUNXK z1vZ=?&awSC)Ja>ishPVh-AT+#DSqh3f(5yL=A8?2`gQd03)l^m&RjMc7_H%ig<4&9 z6p}R~hj`qXq~C%4ny&wh|6lJ{9Nun$l6dg>(YFgrU2?%>(ax4K&XhfL8JoTbnn&n- zyi9KGKyi0>sL*T2EuC98x0XLM(#(19nb9Iw8dBO8VVZL-h{Zic{;?vuB0q+y6*s^TN5 zd8=|-7-WN}^t0~{TMrybOS>!79VPOlRNqRWy&*jwBk+4@ppD z$s55s*{Pm)~p5|bXE?k}J3 z98K5P{3t}zQGcH$h$-3atXAuDU|P|V*gbE=!tmm<6snxhxX)QBn8eIQZPl!*5-p82@xJb;-SDzUe0b0%m{~|rzvfHN3*)e_)uE#PN=+XQM-5m=N~V>| zxgYej3d&B2Gub4+q*CyRUQ5X{>sxkBnG|ZrI?Ke8ldUoadfrj{WPDnxnMq!)=y<38j5CJb}rTMV^9J9$!n@MP=JI*xQyhkHOGbpLO)cF2IfT&*NPsq69cJJckGYh9>3vXyh6SUEGy`98mDqn77DLYBKynpLR1*!SVj4sDVJ z2h%#QKVNix&)s|16Pufo9yLo-s2Zx3pUIKC&iGj2qEiS9OU0Jt=Q3r{yb+~sZ==Mo z4fllIX|!~CF|y>ojKnrypU$^KnqL4*w zT84GSiLW0dI_^7Qym07qhEMuOM=ZutZf&wSlWf^|>+ugr<&<|DHIj_PlCIQ0t2^>B z&Nd5)7rSa4V-Qgql)V1L;Q^mvhosUrO-I|>65D>E-sGG+p%-O#CPk;a$$EF{+dbFY zeRa&gvDh{;+^T+28&K<$(-3QKXW!z|TYGPo*!35}$+zv3P4!yzBFj~cX1^)pspiL%(Hjm8eJX29F@68^o%yGmI`#0nP-WF*DBt7 zmHFb3WFA-6=+){Z$-c(oLx0G&Q)%WQl-;U)==R^7*YT%)beDrwNUd-5vu=l_X1-8x za|`9vYe3^%)jkU zEbJ@Kl1f^Z^IR~XsG5G%fccXLJ-A8NYwFpf2VS7Ky(z)t4_i0&8Sx$8PcVRVm6Wqa z*I?-$-(ihqUz%Q7DSzur=}BRz_J7;1B-#>|l4Q}8WbY{$8uW1Gm&>^=Y55G|@6)f1 z-7)dnB6hY6ShgH-pF-3Ap1#Uu?KukDBRzyn7`umjt&H-5%|mzsy*>B#bWxxRIK%3a zl;Otyu(j_m4%>AMs%9UaJ8G#32<~6BzH!Y@1O3bD>FpwxdVg%;sC)Toj;AicukT2% z!ZPWp9$qo>9p6f7CSe1{M-Bz-ObWZb`|Rp3i+pb|H}ndU>*lSVF_9@kNH4H0gM(^2v zz}yOfUQs4fLJa5gBvV{rsdDYt>->)|d(VL0 z;diRxI%h(|3LndA$MS_lZq%@Ibnuf1PcZR{HJVo;8guojh4_`-8^oSA)~AmKKsS-T zDcm%U(wq8pG~@(23#i`v9}o3w7va+)|Kaui{2K6oTPE#|v?i9WzFYne4)Yr+aM{@g@&DV;=Qx6w zmMmwMnZ&e+CMQ;gF~9~ z32-wtRrtg5P5o{19ZBLAFM4W^KxDty%GVuhZ9uELc;fDm{dT4#Zh2Y)=fAb>$U1nt zCtWNKdtza$wlDbkn!jK1HbEtRG>2sV{?^oA8vh;Ni+Je;<`z7^tIQ*RtzbXan)ku% zNxMUiiu^SEd1?vaHB(>jZ@z;z$>qOJ`{)<`Yq4}4wofRhoEPUG#G)oADJ=P$Pn+~f zR5KYPNT6xr;y*E)cUDD&9k}N9;Xj=3iQ(EGeqqY`p|U!C;_vx(kn`d&N8$I*|2!u0 z$v6R-v9Vl#SiY&hP0qzi!U#*tju4o*9`XdJh(60d6;gHr82~#st>X5?r~3H_fA}*9 zW^5<@_;xauF0Tc-$oNAs@=KXYI03y~ihn9uPyKCjF4FMQm*Y0NPr5$_1dec*KRVl6q82I>P4G^@rVSF)Zf>{NO5SQ6DT))4bt8gdROgf;7@ zUJXeSPSmOCI8Xi2pT8sTf<4KM2r^hXx6Nm`*T>p79kjA)OySm8bp^v+=tun%uQQ`} zJmy%1oyZj*te2~f%;HmPj7#|INn@0hY%xd*N++sm$HN2@_EDmZlpQN8mH%}pbDM-R z{3fA!OwUuv4r2I_3wru?fN4`)w?ZrEMHSyPvS_|(RPQlQAr5@s9|{5Mvd{dri35$A z%{mY;147z#*6qG8xoyuxH}*E^(8D z`o~N@f>6|3h{Zdyuu>u4{d8+*@1+^B*;Zua-3l#EOh>|C=OgDZE!n#x2Z)NEI8(MLU91y zFzA}keUwh=ZC!J(t}M2N>55DaEl!eEv{Q(i+^)lF=*SI&qO4j~pW^1$)akvPg!zOf zUIn#dn^GWRfipcU6!mgGDFhv%k5Zp%+5Uly`ZuP7hujo@hN7*$4%7y8B{mplN;a-{ zhmyLvr|{Fc(;zxMTY$AM}Q zkmMgh@w&GSw>=FrHzIR5^|;UJ2%1)Ic4$4rlJ@Ajm`AgC&0Xyy7#<@>yrm^I7DXdCwUJvD@6(5r1Pd^mzGtd6l+dEb%a@aX zh{#ZeP=*GSLYZk$R5B%rZT4RG*6G`}x0By{e&4m;^{#XNILorz+x|Sy{oKzzUDwq_ zcVB07y84G{EydXmfj9NrWn-@ay zSIO7DsaWC7p#kP{)qu~Aa%_Gez7gcY#m(TbTiBX-rAjw9YN1jRSi9qDV_%ecS%jYi zvq)lE3&wWc6)FSHp{5lgGH_&YBAiu;fe?q%iOD9cLP=5ChrZ^j{kcXKi)Vnb>fWVP zOsHP&8mTJE#l6&k9$Pj`VWJgK95*5XF;e794t|N?&?Y&|mpr@h1&sv@7JxWQMLK~{ zOl>qOs~!9SIHR+5wx8lFHeya$X6rM5{W+Q5in57tR%WA$WQI{@^bYHtwl{CvC! z?4(!W=US4cMcn<}2IHl|PVOpS9>sx5p;{D}0gH%Kh}iqCZ-9ln0P`Y6=AEe{bZfFn zs7l>39fAKwa|%O`a99asDtW6{W}0;Y`FRyr190Ka%^zkb3k_HOs%2xDa;U!yz*%mQ zOS_v%{kf`tjRYcdE66$>sz`Q?42=$RTW2RDIeo%$(X-+!`x}6{!Wl5liZ)I4j<0c* z*DinAO$NN5&TNo+$2NIWz(@BEE5ak+DGNBKnNhJ0sGnJAcB!?zJNFT9eTsco#w}!& z@JW=*0&UtDn~4*d$KEy<9qMZyBnOH~e9hT=rz0|FuJ5xG&Ra z$a<~M;Ti5qRl)lSjC4K7H-o_k%!SB2W<^g+o=ZY>H!h$9svtvtlvJ3@=m1bkD=Z5a zOzJS@ni0qhjRha-#0Q!=ISm+;h8f*WV3H{Q1f~I{W5dJ}!0u9cpe%HKhq*%s9f+ov zyR+jed~FGw*V2hqn3088c&x1Ha!@3xm#zmbxqQc}!A^6jN_Zqekv-}doJZGbt+#8> z^pWj}zq_tRip`7_ja)Gsmd77?UXbykv{QZtys%vO>J;7Q(a+~H-<)F317cZmiIl2) zgTxK5=A@2UFPhx`UOGw!IjJ+?N9vUHk zuO%^nxWeBT5k$Uh@DmP2pWsEv3>r^VYJdNRKl{Ny7S7mw+7?l#oKcJC2wybZwIuc< zMBY+y!+qfbF^3@KLmTP7O#%s;w5C#J0EInn<;y~Mg?9I$hAuJzOTONSf_cML`Jo+` znf&6M25_cPb!|w3)(%+GU*#Ke51hy!(uVV&?96M|3vAwB(A?%uUqk5T?7^>dz%% zSq&fR0wDiGBm<*C6mJmv5Tx6pn9O$r>R8Q7etSYe{a&Hd_ixdd@r&)ZjUd)`Yetap zvO_6KNM7_8DFkuY!lJmhb2q4yk?yfIOeGZlfyV8-1L`R}{EOFu}P&bj6qn&7y(cf*YJeyO}SVDp9 zNqN0VY%j6e*mwWVKFyk;XM(Ij2D~75AA(XO4V$(;>>U)%6p}#mQPDDr3-vzl! zq;Mg&tnAF9aVH_LBxCH*@}4TNRGHO8&P7maH5()&^+*KG{6NA|TD1N4tBpuv>hJSO z&j-UMGkX!Wt+{y@Ku^i}&pXsZkk~_iTJJqc1S$3^NHeiNW8>tSCQ z)IzeyHTqyW&P;kBB4_SlWCdhEX;&x18cm1I6_y}0uzoml(ZktE^=>)`RS-AV#E)0K zxEvG!KtcPeZbBZGoK?MNz{ssraWaIL2IaGtOthTw($bq21$}NWO{Vt5JA>eUTyng2 zAOvy88Yw9^1xnjX@WtyNShhh#V$a0w5^-f%zo&<%o0n*RE&#ivub8mB<+@*lexGDp z3RVqIISXA7Jw}#nlUP2H$6EF4vw8VVw~=M=wN$}_C3Q(Vwk#8TxGL8MDo|;VMY#T? zJKok1a00%2a+X55t6erKP2v>jH!1RH*P<`fLVGFASFrq^@)*ONa}4bp^X2qAQ4BcA zCw(_E%>vs#85Vz5><(q^M@DO&0Le=__(6W<*${RYx>(XE1PB0ft%sWISJBh4=lw(+ zpg@~0?NIO1{=3A~uP8NyS^G%1PBXfLvQKE9ey-9h1#aQ9IE%;fw#TraMGr5$q+RXV z$o|rIN5S%gh93DD7kSCUDUw_|17da4qliWX`1wm>odvg(1cs}8)=-0+m` z66>y_G>ocY!O>0t`dvc96C{-zv0x`rRtvBHT_R-@aY0&lwaO4qVyHJXE<(@^g;ECkLYuR>AT|5ccev#3Xmz9VwaKkIHku_lQ4fKHzr`r+Y`Wbr^O-|O z>lTZ3ii4+z1y(89Hdeh)oAAsZwM4mKn`zycpdW1&ojdHpx(CxCQ_PzEv0V_~Zh?$> z*ysaaR|#B78&pH-@Y_)n>h3mZ3I?o-PpnT9c@8c@PjC(-MS!-VxB@ih^LekW zi`%)urY1|d&UN%km7Dr0y1c1N)feUHHV$vit)NqnK3iN(5sEg#a7YK6lI9&B2IfiE zb)a}B(RM$^<0*cw5q9sV6<4H>vz@NJe_?)o!Z)vHH(@6>e9H#y^9HR+zTl!JM^VZs zOzxB;o-TXM3Hl(Bdh|NTB=f84T~rSh7%z8kLN%?p6O|&f(&A}@)+VA0b~9V8?OmZN z8X!?4GNU`%E7VUdn-M?B7DVnZiQ1{bTH<5%1p}+c17bstnF7~9x6+rL5$f%4hf^zM z%UeGpXSZ?7lz%Z@c(#=&fu+~fWji!QSvrQ)LtxmjTIrQEw=g-CV+Gb6oQemzBJHXR z99+?lTlb$xS)-Wz&1GH5BN}o}`8<|V;H2Zr4}NJ3=gGW4!R%`dCz~o)XgC^-xd+|6 z0B>I;_)g|qLr(r0I7?bzBA!v1I+16G9k`*MrUf!2TqkY^|Fd(nJ4HJE2Z?HhX4#{9 zs`u#FxB0y0iyFMSB^zC&cDbvAC7z61dgw8`_BLJCPmT-NldCv}C(KywmyMux{Q z&_DG2fLQ8}5~ZGlCWPzCvEdnNr9`2EI(9^?)EJ{ZtK3kkJn~oMw|mFk`ZR@xr?OFu zO1*c^)51^CaX;|PEPsgP1Es=PO$M*ivEOi~UD<1x$nKj0si5VNumv2HoY$|X>7&w$ zF!b7a-U(VO{KKJ)wBF7-pr309n54QZ{^6~t8ly?N3H_{?s+Z{EUd1IJBW%$hulCVn}&nwdE7Ez&cp)kKI^A4 z>6;5zY!Js8v}{s4x_LUx%@~;dkB}u+s{_H!rOdUtHelYPTBNpbrgk^XdEFwXGZPo3 zr8o_+P-#PP+j-tF#YxH+=I$fHm(|ynKUVXDAFT(subQ@2zG8*vPbcpZAzn(hAm#b)>Cwn8r-$VwZ-Y#IA~hf*L90yT;zBxwT9! z8#iAW#!RU6j>cxg_(*;(8!!Nv6^i^7>q(nD=@K2BhvwE39}WnlzoVz=d9b?gldhpXuO%BZmlW<5f!^G5DJY!gTgB2kPEZLsHh!JWI)5;3Co$U z5#9Y%rufRY3mrVi>%gbd#EhUU!sCBaS3m0bi>SiP-&XpHLIdkjQ;>-}bg<_=IQA=vt0;4`si5{XOVf*1mnC2d4~YTcc&pAQV+`wDtt<5Hh!ENA(*& zy;7zYiJauHaViQ{$Tjo_S*l_x;XrT!5WUDz_z`?}30FX6WxqEXV@lty(!BAXk$TEP$1fr+-BG9{q)GEUPmlGYz)eXGSMABL)9C?1mmj~jVGx_e_*tTqg7asE@WqDa> zSiWhnEp2x%=#uExdp!`@W^z1Y}ae!4Q`(yd%(G%;Z-(X*i|6Ft|_LMOfPplWc zY9zPrNChMxZU{j3{6sg>)4NjD0?M3iHo6fC} zKM8_clc6AqI7Jg{Zw81aE zX?`#5D0E!R#&g&=ClNYw9XkDJgUk?|Vd&|$_}(Rok!G3a1(yiXDAoG*vN6{AI&0sz zF)*bxcBYAjA;yGSl87G+U(6u%O$gCc&ba&ee)}DDqX0~GFgfSK?|kwyYB;-WMoZU= zc!s}J)fdM%7f%dSaK5y^AJ4o=b90Z(A6ahk+mRXQ0CKRzsrtoYFXdq?{e10LPVkj@D!U!Dt!9Qc#aj-ntbDcMO^Q?T zou!#j19?~kw$v(}pv%WKVi2T+Tv$@gKDg@dO82dToLt|XE9DLjM})z2#U;@!$+MZV za2teQ1C8#exk#a>uN!VC1bxxN#_)}z|-$r2(vL(UGJ3ky7ZUm7Z zP(e>c)>>$iT}RVuL`G>eEAiqDv9SfnwvK2VMdc+V(#iaL<>k9o`u_>b)yY$k>j6aS zJ=oX+7?RF~CD(hbge9Or7w&N4hOx%u3ha~b`}{sH&F|lPkk5+9S>uLFEJP?~A$uct zDG}iunYa4)-Xesa>3|1;Z(7WP2e4)h9AG-M4*qf2VP|ELzGI4b5Rrtud27xJ1PLK8 zm+z{7l|Aa2Oh}#kS0p%5VRC$^461^(%jWf<&f$O#p+BZ&h}a<2$F1-A?O|-1&gY#m ztb@fm1R*qE7`BRa-LSk)qbOo{`^=-yb)5GE9PbpW`fiHdKd^%mn9n~o9J^nZwXp99 zihI~J$^CKq->b2d0y>3|^~zl>n98=Q8bjzQF!!VYnu|__5BCk~z?9f~PBY5Pc3bQ8n*%P(UBRd{b>hXZ6 zlJvmmF!#opaA|Q-tNJ{FyN2RU-f`gtEZ)T5oLo+3upEbB^SM;U74_mUBFgd+-)+Xf zot&@wJh2|JMK8UDRDwgjEDq)X0b;qGzyC2w0yy#vUu(f~WVPh*S3^wmgxO)aQ?F2{ zJ#58emN)dUknbOlAGC<5^}XMfVL$#-gA-JZX}@K3%syGJVLg|B^Tz~euIupeP&t+( z8!U{!sv~Re#vxf?j_Dr1wwdJ(JuKwMKR7iRkNhQDYtWCs9FG|dHiQ&ylE(kj*FpZ& zNCdebk$$P5!eX5&u=XKE_vV|QF$pi%+$1)>GEC|{;cf!7;Ug-M0v20+dMCd62X}`C z4F-5jQIYGrcKfDtA2Liebv1W}oG zQ|tXUR)Uw$4dy2Z;)L`+dvK02?eY?lw{&w%{mm|af2n97j6b%2!m?50BU8C(PE>T) zqY1@Is8;Tzk+U`xluZ;+BZmlm*t$e)>2|kT^PP_I7JaFiUbZKHzDWc8z;@ru zQLL8FJcyPtJ~3nzN#>#(TP?+l8vx@kagxtI1GSKHq0YdXgOJ{{2g*d{*QFki`X%mszG+ z-a?2t-BFWH1KGAXwRb9>g1h=5vS%Ub#IM?DK+L3eDKQV14N`BO<4^-(tc|GY7Bi6H z%%}PiHr*N80do4_-ons**BE9?(*n9XzoV7N4`t|Gs4s~;M+!-K#8v3J)T0-&&8NJ2 zHG@cR(TG}a^NCOZ$<3+KWvKq01czrDRb4v}pzk(BAB%dCHA-ktW;TK9T#ZH!+Y>6i zQFxYYOM((XDkOh5A|T!8*gauBm(LQqnkc6bmH3S&npA~O-6%(B8*Ks|=uLDCbq<}Z;yZH>nk}y5uZ;J5El}_}M>!pZqH=;IK&Nv)lG0li zuThjPz5+oLDq#j>ZKJ_C%4UmO#qGbYvMU!Uee_y_G@-;VzFw7MyP_niC38m_Ff+y^}MGOun!HT+9T3&VxwMfT79AbE$LBEOUk-`o9M2^FLK?`KC!U27VSV+s25U$ z;MOY%!*a^xY3j|Vk%5|tr~Gz(!8o0${M(=S`L&hXr}6Bs!R%ah?OXicWs|$&Zv%%4 zdvy3#X>QnXJt#Wx*-bC4aB2_^e<4}3Q=t21_;usF{st&%Wpho*wLzC|b9PD0ZEE!L zMV1sr$*Mlcy}%VivPnF8VTmMLp%HnAKHsS}*ae9PJ3#%T@^zN)R}` z*bexN*_DZ8nzZb0V`YA&Nz$ss1ACbCK!GEJSEdvk^MVGQJtWPi$v4$3&o1E)Ie$M! z`%%HMSg1m#FBoIw(P4+lSdZvE)~YzNHDe|Re*F?C|c_Df($x3S7>LQ90nZ+>t_;@2EIM_#TX z9NbN-tBkP*IQmZtL{PT#c4)0Eu7I3C8Y)&k>&?147}*wgfl?_)p=K{Z)-)B~9%=VP zeK8HA+{V5eLYeVxh^{{d)GPHoeJi94VqM|S6sZ>U(gM@d)c0~zQ_hWYpp0D+ZS_TT zU-H$9Vq7&zmp24DR0o=TnX(Sm4D?oQ(`jJD+d551#=iAy&Fzx@2ACUp%m`u}2OGGQ z?Gz?4E%AHTOGL{a-8W1`etHh(Jb8yKtCOv4-R4-E@-!q-VwQ0ug@2ub{n*1gEsEKB z#<}+(#OQtF{C82J^~rLtt`C>Q)(|xA4m6-5>P#XQ#SDUD9b|m+vA06e6@?VOGxz0) zj2ZlWrkS^Puf4HuB%J9=>N6L2dX0DG_kIrFZdXf`yATmSYrk@i2o-udU(g^yC4&G< zFA4#Pncn;>&$_^8q2xByZ9ct>*HgW$&G^7=*+I=}{u?O4{w9xpNOR>%M`60cW2Jev zV<^EfG)rVrAq9F`N={(!4b*yqiC?os?1_POO8-Ic>^rO1cM(L)a+XOb1YBjHKwIO@ zzg(|DbVc28LRB@yCNS6%aZQ+1!*2FTn%;J! zXgWUaa_WakTt@}=qayvMPN?nyd#ZXtVf^6L#w=e|W%4*Q*bj~6gFc%q4ok@kMm7Rp znYL4Dq!d6>^vj_e+4KA`G_8UjA5lD; z9`}X%kTuRzMNpWui3M2si+9JVy}+18@>s=Po4>;DmG7eWbEZ>l(?kt_q|s9QxY?55 zeop__iA78ZKS52)4h!M0s*o z$Br6c{d5kLZe^mRGp2lAFvwC0krY55deWZM?1K-!rI8t(KgFaPQ$0`JL$fcuHq%}V z(WKU`ir2j8#5H_v?B;&^F~HQ?0|Qv#ce?e*U$P(a|NO@PSNg1X>*RD&xx^jYtp(k6 zqA(dgjabj^jiO9fjI+$gmnp*B0tIF=2fu!-VL%5Vosfsvvr~h{?H{kHTpMlaXwMZ( z7($wbifBV4J&hzO2osT1hyKrF_azgA(9O(X1kDWRe{)k}?+&Y!0z%oMD{oXWY)PWK zknA<*I?*r+Sv88elRh2_<9CS11b@YJ3C}N!)LWk)yabp_>HUgB=a_0d`h#PSy zK1mANd4oc@Ht)moRCy-x89TrLGYUlr?>OvMQZ0=@p}##3ReCG!MytJ8Cj+0CJRFW) z9%!X?6R(Rd{+^@%><3{&VLZ^&KdG^bTU#NONX~bKL&ccc!po-a$AAd!5ows989_4( z;}ROF)iDDdASi=AIQD*mpa_72=;W9C|1m7Xy^D}xfHG#Q6^fQ3nx#_^8v#9Mf=Sji zER#;;hgfMwympE%80n+5+*%8aKoy4=NKrRTUx36bBIFR(0tAq$boC(W#eX8KSRiyo ziD?xn6})yM`x{uOE3cku%KLn|0dE<;v1tID6d57;2b$ix3jXuG_Z4&NkV<9`h$fV1 zTx(TmRD(HJtZsy{81=7SoAJJH&-Lr{IKHa`V=k@~ZfpBlBwyZBT_BZ{yvdS#a2_H5 z-PyyVe^AlWjp35`D9K}-Z@Ea+dOp?nF>vQ=SL!zV^;l@xogi*>8szMd_)IY#DdrIV zLoAsp%V^Atxt>294LeLHTg@qsN#5jNS|%y>sHQJsyRhk2cl8`Fkc zxP>K6EH-6l(TPTL&AU9n%=BPW5aTegom;XS&S@%1@D(n-r?$SMYHfp=ol+CxxNP5> zF{N{`#yNjdOFxQn*h`AX%6C_WL_Ok^hXCIA)k4X2D~+aUHV2_@@hU>{wGw5t?opvE zAz1e{G28}Y1?n|ErYX{OiWd!u0k>v!yhKDaAg=%gb+Dl7!u z^?O0~z7f&PUQAH!)8>5zC$C$B-g|wWW0x}0o6qqU%;|y;m(0?xWe23eG8{^w zqDAOeYg893J+9lN6cBGe@`L@PhWkeaO+;ElX_~i4GAA?W!hNbb)okVGP<0xQa_5jf zE)6)anfSID-%npmBGaN1^cUt~0;F-r_A6 z^A)WS<5@pnFy?(aiB9mntAdFp)j^)im8S8!eCH-EJT*=XgMC?yz+@TVu3*lO7=Z~n z)^csy8jqmpQ^*89Y6}tbZ-ge-mbc^N?1P>~K-jIKa6JHY0rqk)06i;5 zOY0TyYU?4)Wqh`iD`r4!j;w;}?%bwVbq=$A?Y=y&X@L8^O1;yHuQzUVw4+&BSWfCn zaJj0=;``q?#o;tmsbo`kO#x9vS(%G;cy1nq%sdn4P1n0M>4l$fJcM?}+s+w#57IkY$3P8LI5V|%CTIP|xfS9^p_^3W;f zlev1Sro*DYN0=NokF#s$q5jL8w^Q9T1`N52Wco~JR0nZQUeX3OA;afT7=@hZYxC2LGo~&~`x}dtXs5uiLvLxhiP;ogG4{z1`ZQE38b`tt6HdCN zFme$}j)(hT)=CuXRm@n^fci&OyKzLBXg6)2oPQNEOIlw=lJXM6pV?ai#}&(z%zFm4h~1s1ohR0 zy88&cj?v)2$~jk<9>o6<7wmn%9ZDsqaMaRx#u$#EnJX2L5)eDihb>*`#Gb_5L+GRJ z!EPGcsIp|UhnHQaBbW=WNoVF}DfSvJ2S-L>K|Km?0y}<_{8ej>)2%@yo;{oBPL4Bh zJX{fs3Y<4I>i&{zxL}xB^QITB-A`jfP=li5sNER2j;ojcA6c3(MMLFzjeYI`y#R+s zfxI}F+&mWg=WxY-%kMXCYyZYX<2)l07&?Wn_J&P0MK`NGN}#zuR%YXT;9R3WZVx{7 zUt1o8{!HSHi-k~*p($_|{ZC4J1C#e55pzLxK-OLHgC}8lx0$c)?j=n#8RmZeI#0}~sewpGwB1*5AQIOh_Q<6J!&db5j+uG^pS3Y_o*e6)WU`wI zs_tI9Ik;m>F$&qzi*-1>){C{!J9-Lm_0p}NyHb++Tko@U%~-z_%aLJush3maUkBjX;7RiaT%pjdU{`=m&ch561>ooRn;Nj?B z{)fBs)5X@1p{Qlku#ldM^p974>v-(nQ0{F)|Ldn@BaO7DiLT)=aoYD4<+Mf5G_Zfd zgL8iW>9Vmpb55Vdx`V02%j~KceE5I*#-Du?e<$pp+=zfJ|9HQ88i4Zz9PF{XwK%AP z|1%>*-a}N6i=X^_*?#u)J+}K_$Na<244rhEE4Eesng0L&Z8APGlc!VsW9rqYIfof; Q{qWzC1)B52=b9Y;Ki&`M>;M1& literal 0 HcmV?d00001 diff --git a/docs/src/piccolo/projects_and_apps/included_apps.rst b/docs/src/piccolo/projects_and_apps/included_apps.rst index 2e9fb54ba..c2f7cc54c 100644 --- a/docs/src/piccolo/projects_and_apps/included_apps.rst +++ b/docs/src/piccolo/projects_and_apps/included_apps.rst @@ -9,6 +9,8 @@ Auto includes The following are registered with your :ref:`AppRegistry` automatically: +------------------------------------------------------------------------------- + app ~~~ @@ -18,6 +20,8 @@ Lets you create new Piccolo apps. See :ref:`PiccoloApps`. piccolo app new +------------------------------------------------------------------------------- + asgi ~~~~ @@ -27,6 +31,8 @@ Lets you scaffold an ASGI web app. See :ref:`ASGICommand`. piccolo asgi new +------------------------------------------------------------------------------- + meta ~~~~ @@ -36,11 +42,15 @@ Tells you which version of Piccolo is installed. piccolo meta version +------------------------------------------------------------------------------- + migrations ~~~~~~~~~~ Lets you create and run migrations. See :ref:`Migrations`. +------------------------------------------------------------------------------- + playground ~~~~~~~~~~ @@ -51,6 +61,8 @@ Lets you learn the Piccolo query syntax, using an example schema. See piccolo playground run +------------------------------------------------------------------------------- + project ~~~~~~~ @@ -62,9 +74,14 @@ Lets you create a new ``piccolo_conf.py`` file. See :ref:`PiccoloProjects`. .. _SchemaApp: +------------------------------------------------------------------------------- + schema ~~~~~~ +generate +^^^^^^^^ + Lets you auto generate Piccolo ``Table`` classes from an existing database. Make sure the credentials in ``piccolo_conf.py`` are for the database you're interested in, then run the following: @@ -77,6 +94,34 @@ interested in, then run the following: current form it will save you a lot of time. Make sure you check the generated code to make sure it's correct. +graph +^^^^^ + +A basic schema visualisation tool. It prints out the contents of a GraphViz dot +file representing your schema. + +.. code-block:: bash + + piccolo schema graph + +You can pipe the output to your clipboard (``piccolo schema graph | pbcopy`` +on a Mac), then paste it into a `website like this `_ +to turn it into an image file. + +Or if you have `Graphviz `_ installed on your +machine, you can do this to create an image file: + +.. code-block:: bash + + piccolo schema graph | dot -Tpdf -o graph.pdf + +Here's an example of a generated image: + +.. image:: ./images/schema_graph_output.png + :target: /_images/schema_graph_output.png + +------------------------------------------------------------------------------- + shell ~~~~~ @@ -87,6 +132,8 @@ Launches an iPython shell, and automatically imports all of your registered piccolo shell run +------------------------------------------------------------------------------- + sql_shell ~~~~~~~~~ @@ -101,6 +148,8 @@ need to run raw SQL queries on your database. For it to work, the underlying command needs to be on the path (i.e. ``psql`` or ``sqlite3`` depending on which you're using). +------------------------------------------------------------------------------- + .. _TesterApp: tester diff --git a/piccolo/apps/schema/commands/graph.py b/piccolo/apps/schema/commands/graph.py new file mode 100644 index 000000000..4a19e4b23 --- /dev/null +++ b/piccolo/apps/schema/commands/graph.py @@ -0,0 +1,113 @@ +""" +Credit to the Django Extensions team for inspiring this tool. +""" + +import dataclasses +import os +import sys +import typing as t + +import jinja2 + +from piccolo.conf.apps import Finder + +TEMPLATE_DIRECTORY = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "templates" +) + +JINJA_ENV = jinja2.Environment( + loader=jinja2.FileSystemLoader(searchpath=TEMPLATE_DIRECTORY), +) + + +@dataclasses.dataclass +class GraphColumn: + name: str + type: str + + +@dataclasses.dataclass +class GraphTable: + name: str + columns: t.List[GraphColumn] + + +@dataclasses.dataclass +class GraphRelation: + table_a: str + table_b: str + label: str + + +def render_template(**kwargs): + template = JINJA_ENV.get_template("graphviz.dot.jinja") + return template.render(**kwargs) + + +def graph( + apps: str = "all", direction: str = "LR", output: t.Optional[str] = None +): + """ + Prints out a graphviz .dot file for your schema. + + :param apps: + The name of the apps to include. If 'all' is given then every app is + included. To specify multiple app names, separate them with commas. + For example --apps="app1,app2". + :param direction: + How the tables should be orientated - by default it's "LR" which is + left to right, so the graph will be landscape. The alternative is + "TB", which is top to bottom, so the graph will be portrait. + :param output: + If specified, rather than printing out the file contents, they'll be + written to this file. For example --output=graph.dot + + """ + finder = Finder() + app_names = finder.get_sorted_app_names() + + if apps != "all": + given_app_names = [i.strip() for i in apps.split(",")] + delta = set(given_app_names) - set(app_names) + if delta: + sys.exit(f"These apps aren't recognised: {', '.join(delta)}.") + app_names = given_app_names + + tables: t.List[GraphTable] = [] + relations: t.List[GraphRelation] = [] + + for app_name in app_names: + app_config = finder.get_app_config(app_name=app_name) + for table_class in app_config.table_classes: + tables.append( + GraphTable( + name=table_class.__name__, + columns=[ + GraphColumn( + name=i._meta.name, type=i.__class__.__name__ + ) + for i in table_class._meta.columns + ], + ) + ) + for fk_column in table_class._meta.foreign_key_columns: + reference_table_class = ( + fk_column._foreign_key_meta.resolved_references + ) + relations.append( + GraphRelation( + table_a=table_class.__name__, + table_b=reference_table_class.__name__, + label=fk_column._meta.name, + ) + ) + + contents = render_template( + tables=tables, relations=relations, direction=direction + ) + + if output is None: + print(contents) + else: + with open(output, "w") as f: + f.write(contents) diff --git a/piccolo/apps/schema/commands/templates/graphviz.dot.jinja b/piccolo/apps/schema/commands/templates/graphviz.dot.jinja new file mode 100644 index 000000000..e014fa89b --- /dev/null +++ b/piccolo/apps/schema/commands/templates/graphviz.dot.jinja @@ -0,0 +1,53 @@ +digraph model_graph { + fontname = "Roboto" + fontsize = 8 + splines = true + rankdir = "{{ direction }}"; + + node [ + fontname = "Roboto" + fontsize = 8 + shape = "plaintext" + ] + + edge [ + fontname = "Roboto" + fontsize = 8 + ] + + // Tables + {% for table in tables %} + TABLE_{{ table.name }} [label=< + + + + + + {% for column in table.columns %} + + + + + {% endfor %} +
+ + {{ table.name }} + +
+ + {{ column.name }} + + + + {{ column.type }} + +
+ >] + {% endfor %} + + // Relations + {% for relation in relations %} + TABLE_{{ relation.table_a }} -> TABLE_{{ relation.table_b }} + [label="{{ relation.label }}"] [arrowhead=none, arrowtail=dot, dir=both]; + {% endfor %} +} diff --git a/piccolo/apps/schema/piccolo_app.py b/piccolo/apps/schema/piccolo_app.py index ab8f2d13f..6f8ab0b1d 100644 --- a/piccolo/apps/schema/piccolo_app.py +++ b/piccolo/apps/schema/piccolo_app.py @@ -1,9 +1,16 @@ from piccolo.conf.apps import AppConfig, Command from .commands.generate import generate +from .commands.graph import graph APP_CONFIG = AppConfig( app_name="schema", migrations_folder_path="", - commands=[Command(callable=generate, aliases=["g", "create", "new"])], + commands=[ + Command(callable=generate, aliases=["gen", "create", "new"]), + Command( + callable=graph, + aliases=["map", "visualise", "vizualise", "viz", "vis"], + ), + ], ) diff --git a/piccolo/engine/base.py b/piccolo/engine/base.py index 2db51692c..657e6145e 100644 --- a/piccolo/engine/base.py +++ b/piccolo/engine/base.py @@ -69,7 +69,7 @@ async def check_version(self): engine_type = self.engine_type.capitalize() logger.info(f"Running {engine_type} version {version_number}") - if version_number < self.min_version_number: + if version_number and (version_number < self.min_version_number): message = ( f"This version of {self.engine_type} isn't supported " f"(< {self.min_version_number}) - some features might not be " diff --git a/piccolo/engine/postgres.py b/piccolo/engine/postgres.py index dce9b1b81..a823c4b84 100644 --- a/piccolo/engine/postgres.py +++ b/piccolo/engine/postgres.py @@ -10,7 +10,7 @@ from piccolo.querystring import QueryString from piccolo.utils.lazy_loader import LazyLoader from piccolo.utils.sync import run_sync -from piccolo.utils.warnings import Level, colored_string, colored_warning +from piccolo.utils.warnings import Level, colored_warning asyncpg = LazyLoader("asyncpg", globals(), "asyncpg") @@ -274,8 +274,7 @@ async def get_version(self) -> float: except ConnectionRefusedError as exception: # Suppressing the exception, otherwise importing piccolo_conf.py # containing an engine will raise an ImportError. - colored_warning("Unable to connect to database") - print(exception) + colored_warning(f"Unable to connect to database - {exception}") return 0.0 else: version_string = response[0]["server_version"] @@ -290,15 +289,13 @@ async def prep_database(self): f'CREATE EXTENSION IF NOT EXISTS "{extension}"', ) except asyncpg.exceptions.InsufficientPrivilegeError: - print( - colored_string( - f"=> Unable to create {extension} extension - some " - "functionality may not behave as expected. Make sure " - "your database user has permission to create " - "extensions, or add it manually using " - f'`CREATE EXTENSION "{extension}";`', - level=Level.medium, - ) + colored_warning( + f"=> Unable to create {extension} extension - some " + "functionality may not behave as expected. Make sure " + "your database user has permission to create " + "extensions, or add it manually using " + f'`CREATE EXTENSION "{extension}";`', + level=Level.medium, ) ########################################################################### diff --git a/piccolo/main.py b/piccolo/main.py index 2f58f493c..0c3098cb3 100644 --- a/piccolo/main.py +++ b/piccolo/main.py @@ -24,7 +24,7 @@ from piccolo.apps.user.piccolo_app import APP_CONFIG as user_config from piccolo.conf.apps import AppRegistry, Finder from piccolo.utils.sync import run_sync -from piccolo.utils.warnings import Level, colored_string +from piccolo.utils.warnings import Level, colored_warning DIAGNOSE_FLAG = "--diagnose" @@ -117,18 +117,17 @@ def main(): if havent_ran_count == 1 else f"{havent_ran_count} migrations haven't" ) - print( - colored_string( - message=( - "=> {} been run - the app " - "might not behave as expected.\n" - "To check which use:\n" - " piccolo migrations check\n" - "To run all migrations:\n" - " piccolo migrations forwards all\n" - ).format(message), - level=Level.high, - ) + + colored_warning( + message=( + "=> {} been run - the app " + "might not behave as expected.\n" + "To check which use:\n" + " piccolo migrations check\n" + "To run all migrations:\n" + " piccolo migrations forwards all\n" + ).format(message), + level=Level.high, ) except Exception: pass diff --git a/tests/apps/schema/commands/test_graph.py b/tests/apps/schema/commands/test_graph.py new file mode 100644 index 000000000..16e541a43 --- /dev/null +++ b/tests/apps/schema/commands/test_graph.py @@ -0,0 +1,47 @@ +import os +import tempfile +import uuid +from unittest import TestCase +from unittest.mock import MagicMock, patch + +from piccolo.apps.schema.commands.graph import graph + + +class TestGraph(TestCase): + def _verify_contents(self, file_contents: str): + """ + Make sure the contents of the file are correct. + """ + # Make sure no extra content was output at the start. + self.assertTrue(file_contents.startswith("digraph model_graph")) + + # Make sure the tables are present + self.assertTrue("TABLE_Band [label" in file_contents) + self.assertTrue("TABLE_Manager [label" in file_contents) + + # Make sure a relation is present + self.assertTrue("TABLE_Concert -> TABLE_Band" in file_contents) + + @patch("piccolo.apps.schema.commands.graph.print") + def test_graph(self, print_: MagicMock): + """ + Make sure the file contents can be printed to stdout. + """ + graph() + file_contents = print_.call_args[0][0] + self._verify_contents(file_contents) + + def test_graph_to_file(self): + """ + Make sure the file contents can be written to disk. + """ + directory = tempfile.gettempdir() + path = os.path.join(directory, f"{uuid.uuid4()}.dot") + + graph(output=path) + + with open(path, "r") as f: + file_contents = f.read() + + self._verify_contents(file_contents) + os.unlink(path) From f1244ecc530029cba4c2d47b8f6bd7d13dcff7a8 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 9 Sep 2021 12:10:01 +0100 Subject: [PATCH 054/727] bumped version --- CHANGES | 12 ++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 9ae2a2e3f..f546b5d8a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,18 @@ Changes ======= +0.45.0 +------ +Added the ``piccolo schema graph`` command for visualising your database +structure, which outputs a Graphviz file. It can then be turned into an +image, for example: + +.. code-block:: bash + + piccolo schema map | dot -Tpdf -o graph.pdf + +Also made some minor changes to the ASGI templates, to reduce MyPy errors. + 0.44.1 ------ Updated ``to_dict`` so it works with nested objects, as introduced by the diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 0e1d75596..06e3e25dc 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.44.1" +__VERSION__ = "0.45.0" From ed1e50273499e317b39a35cee96092df961100aa Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 9 Sep 2021 12:28:00 +0100 Subject: [PATCH 055/727] mention Piccolo apps, and update example code to use `get` --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2c11c1ad9..3eb6732b4 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ Some of it’s stand out features are: - Tab completion support - works great with iPython and VSCode. - Batteries included - a User model, authentication, migrations, an [admin GUI](https://github.com/piccolo-orm/piccolo_admin), and more. - Modern Python - fully type annotated. +- Make your codebase modular and scalable with Piccolo apps (similar to Django apps). ## Syntax @@ -59,7 +60,7 @@ b = Band(name='C-Sharps', popularity=100) await b.save().run() # To fetch an object from the database, and update it: -b = await Band.objects().where(Band.name == 'Pythonistas').first().run() +b = await Band.objects().get(Band.name == 'Pythonistas').run() b.popularity = 10000 await b.save().run() From 85696253b52c60517f2d6ec0390ca09dc31ada91 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 10 Sep 2021 17:32:26 +0100 Subject: [PATCH 056/727] Schema generation fixes (#226) * fix a typo in the column import * sort the tables by their foreign keys * make sure there are no missing imports from column params * check the generated code is valid * use graphlib for table sorting Using just basic sort would fail on occasion * remove walrus operators * remove unnecessary imports from schema generation output * make graph creation recursive --- .../apps/migrations/auto/migration_manager.py | 77 ++++-- piccolo/apps/schema/commands/generate.py | 23 +- piccolo/utils/graphlib/__init__.py | 5 + piccolo/utils/graphlib/_graphlib.py | 255 ++++++++++++++++++ tests/apps/schema/commands/test_generate.py | 10 +- 5 files changed, 339 insertions(+), 31 deletions(-) create mode 100644 piccolo/utils/graphlib/__init__.py create mode 100644 piccolo/utils/graphlib/_graphlib.py diff --git a/piccolo/apps/migrations/auto/migration_manager.py b/piccolo/apps/migrations/auto/migration_manager.py index 0f6c01478..0b2d1e3f5 100644 --- a/piccolo/apps/migrations/auto/migration_manager.py +++ b/piccolo/apps/migrations/auto/migration_manager.py @@ -3,7 +3,6 @@ import inspect import typing as t from dataclasses import dataclass, field -from functools import cmp_to_key from piccolo.apps.migrations.auto.diffable_table import DiffableTable from piccolo.apps.migrations.auto.operations import ( @@ -16,6 +15,7 @@ from piccolo.columns import Column, column_types from piccolo.engine import engine_finder from piccolo.table import Table, create_table_class +from piccolo.utils.graphlib import TopologicalSorter @dataclass @@ -118,37 +118,48 @@ def table_class_names(self) -> t.List[str]: return list(set([i.table_class_name for i in self.alter_columns])) -def _compare_tables( - table_a: t.Type[Table], - table_b: t.Type[Table], +def _get_graph( + table_classes: t.List[t.Type[Table]], iterations: int = 0, - max_iterations=5, -) -> int: + max_iterations: int = 5, +) -> t.Dict[str, t.Set[str]]: """ - A comparison function, for sorting Table classes, based on their foreign - keys. + Analyses the tables based on their foreign keys, and returns a data + structure like: + + .. code-block:: python - :param iterations: - As this function is called recursively, we use this to limit the depth, - to prevent an infinite loop. + {'band': {'manager'}, 'concert': {'band', 'venue'}, 'manager': set()} + + The keys are tablenames, and the values are tablenames directly connected + to it via a foreign key. """ + output: t.Dict[str, t.Set[str]] = {} + if iterations >= max_iterations: - return 0 + return output - for fk_column in table_a._meta.foreign_key_columns: - references = fk_column._foreign_key_meta.resolved_references - if references._meta.tablename == table_b._meta.tablename: - return 1 - else: - for _fk_column in references._meta.foreign_key_columns: - _references = _fk_column._foreign_key_meta.resolved_references - if _compare_tables( - _references, table_b, iterations=iterations + 1 - ): - return 1 + for table_class in table_classes: + dependents: t.Set[str] = set() + for fk in table_class._meta.foreign_key_columns: + dependents.add( + fk._foreign_key_meta.resolved_references._meta.tablename + ) - return -1 + # We also recursively check the related tables to get a fuller + # picture of the schema and relationships. + referenced_table = fk._foreign_key_meta.resolved_references + output.update( + _get_graph( + [referenced_table], + iterations=iterations + 1, + ) + ) + + output[table_class._meta.tablename] = dependents + + return output def sort_table_classes( @@ -158,7 +169,23 @@ def sort_table_classes( Sort the table classes based on their foreign keys, so they can be created in the correct order. """ - return sorted(table_classes, key=cmp_to_key(_compare_tables)) + table_class_dict = { + table_class._meta.tablename: table_class + for table_class in table_classes + } + + graph = _get_graph(table_classes) + + sorter = TopologicalSorter(graph) + ordered_tablenames = tuple(sorter.static_order()) + + output: t.List[t.Type[Table]] = [] + for tablename in ordered_tablenames: + table_class = table_class_dict.get(tablename, None) + if table_class is not None: + output.append(table_class) + + return output @dataclass diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index a6042fdd9..8641568f8 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -6,6 +6,8 @@ import black from typing_extensions import Literal +from piccolo.apps.migrations.auto.migration_manager import sort_table_classes +from piccolo.apps.migrations.auto.serialisation import serialise_params from piccolo.columns.base import Column from piccolo.columns.column_types import ( JSON, @@ -336,18 +338,22 @@ class Schema(Table, db=engine): ) else: kwargs["references"] = ForeignKeyPlaceholder - imports.add( - "from piccolo.columns.base import OnDelete, OnUpdate" - ) imports.add( - "from piccolo.column_types import " + column_type.__name__ + "from piccolo.columns.column_types import " + + column_type.__name__ ) if column_type is Varchar: kwargs["length"] = pg_row_meta.character_maximum_length - columns[column_name] = column_type(**kwargs) + column = column_type(**kwargs) + + serialised_params = serialise_params(column._meta.params) + for extra_import in serialised_params.extra_imports: + imports.add(extra_import.__repr__()) + + columns[column_name] = column else: warnings.append(f"{tablename}.{column_name} ['{data_type}']") @@ -358,6 +364,13 @@ class Schema(Table, db=engine): ) tables.append(table) + # Sort the tables based on their ForeignKeys. + tables = sort_table_classes(tables) + + # We currently don't show the index argument for columns in the output, + # so we don't need this import for now: + imports.remove("from piccolo.columns.indexes import IndexMethod") + return OutputSchema( imports=sorted(list(imports)), warnings=warnings, tables=tables ) diff --git a/piccolo/utils/graphlib/__init__.py b/piccolo/utils/graphlib/__init__.py new file mode 100644 index 000000000..77a9e5774 --- /dev/null +++ b/piccolo/utils/graphlib/__init__.py @@ -0,0 +1,5 @@ +try: + from graphlib import CycleError, TopologicalSorter # type: ignore +except ImportError: + # For version < Python 3.9 + from ._graphlib import CycleError, TopologicalSorter diff --git a/piccolo/utils/graphlib/_graphlib.py b/piccolo/utils/graphlib/_graphlib.py new file mode 100644 index 000000000..1cc64d51d --- /dev/null +++ b/piccolo/utils/graphlib/_graphlib.py @@ -0,0 +1,255 @@ +""" +This is a backport of graphlib from Python 3.9. +""" + +# flake8: noqa + + +__all__ = ["TopologicalSorter", "CycleError"] + +_NODE_OUT = -1 +_NODE_DONE = -2 + + +class _NodeInfo: + __slots__ = "node", "npredecessors", "successors" + + def __init__(self, node): + # The node this class is augmenting. + self.node = node + + # Number of predecessors, generally >= 0. When this value falls to 0, + # and is returned by get_ready(), this is set to _NODE_OUT and when the + # node is marked done by a call to done(), set to _NODE_DONE. + self.npredecessors = 0 + + # List of successor nodes. The list can contain duplicated elements as + # long as they're all reflected in the successor's npredecessors attribute). + self.successors = [] + + +class CycleError(ValueError): + """Subclass of ValueError raised by TopologicalSorter.prepare if cycles + exist in the working graph. + + If multiple cycles exist, only one undefined choice among them will be reported + and included in the exception. The detected cycle can be accessed via the second + element in the *args* attribute of the exception instance and consists in a list + of nodes, such that each node is, in the graph, an immediate predecessor of the + next node in the list. In the reported list, the first and the last node will be + the same, to make it clear that it is cyclic. + """ + + pass + + +class TopologicalSorter: + """Provides functionality to topologically sort a graph of hashable nodes""" + + def __init__(self, graph=None): + self._node2info = {} + self._ready_nodes = None + self._npassedout = 0 + self._nfinished = 0 + + if graph is not None: + for node, predecessors in graph.items(): + self.add(node, *predecessors) + + def _get_nodeinfo(self, node): + result = self._node2info.get(node) + if result is None: + self._node2info[node] = result = _NodeInfo(node) + return result + + def add(self, node, *predecessors): + """Add a new node and its predecessors to the graph. + + Both the *node* and all elements in *predecessors* must be hashable. + + If called multiple times with the same node argument, the set of dependencies + will be the union of all dependencies passed in. + + It is possible to add a node with no dependencies (*predecessors* is not provided) + as well as provide a dependency twice. If a node that has not been provided before + is included among *predecessors* it will be automatically added to the graph with + no predecessors of its own. + + Raises ValueError if called after "prepare". + """ + if self._ready_nodes is not None: + raise ValueError("Nodes cannot be added after a call to prepare()") + + # Create the node -> predecessor edges + nodeinfo = self._get_nodeinfo(node) + nodeinfo.npredecessors += len(predecessors) + + # Create the predecessor -> node edges + for pred in predecessors: + pred_info = self._get_nodeinfo(pred) + pred_info.successors.append(node) + + def prepare(self): + """Mark the graph as finished and check for cycles in the graph. + + If any cycle is detected, "CycleError" will be raised, but "get_ready" can + still be used to obtain as many nodes as possible until cycles block more + progress. After a call to this function, the graph cannot be modified and + therefore no more nodes can be added using "add". + """ + if self._ready_nodes is not None: + raise ValueError("cannot prepare() more than once") + + self._ready_nodes = [ + i.node for i in self._node2info.values() if i.npredecessors == 0 + ] + # ready_nodes is set before we look for cycles on purpose: + # if the user wants to catch the CycleError, that's fine, + # they can continue using the instance to grab as many + # nodes as possible before cycles block more progress + cycle = self._find_cycle() + if cycle: + raise CycleError(f"nodes are in a cycle", cycle) + + def get_ready(self): + """Return a tuple of all the nodes that are ready. + + Initially it returns all nodes with no predecessors; once those are marked + as processed by calling "done", further calls will return all new nodes that + have all their predecessors already processed. Once no more progress can be made, + empty tuples are returned. + + Raises ValueError if called without calling "prepare" previously. + """ + if self._ready_nodes is None: + raise ValueError("prepare() must be called first") + + # Get the nodes that are ready and mark them + result = tuple(self._ready_nodes) + n2i = self._node2info + for node in result: + n2i[node].npredecessors = _NODE_OUT + + # Clean the list of nodes that are ready and update + # the counter of nodes that we have returned. + self._ready_nodes.clear() + self._npassedout += len(result) + + return result + + def is_active(self): + """Return ``True`` if more progress can be made and ``False`` otherwise. + + Progress can be made if cycles do not block the resolution and either there + are still nodes ready that haven't yet been returned by "get_ready" or the + number of nodes marked "done" is less than the number that have been returned + by "get_ready". + + Raises ValueError if called without calling "prepare" previously. + """ + if self._ready_nodes is None: + raise ValueError("prepare() must be called first") + return self._nfinished < self._npassedout or bool(self._ready_nodes) + + def __bool__(self): + return self.is_active() + + def done(self, *nodes): + """Marks a set of nodes returned by "get_ready" as processed. + + This method unblocks any successor of each node in *nodes* for being returned + in the future by a call to "get_ready". + + Raises :exec:`ValueError` if any node in *nodes* has already been marked as + processed by a previous call to this method, if a node was not added to the + graph by using "add" or if called without calling "prepare" previously or if + node has not yet been returned by "get_ready". + """ + + if self._ready_nodes is None: + raise ValueError("prepare() must be called first") + + n2i = self._node2info + + for node in nodes: + + # Check if we know about this node (it was added previously using add() + nodeinfo = n2i.get(node) + if nodeinfo is None: + raise ValueError(f"node {node!r} was not added using add()") + + # If the node has not being returned (marked as ready) previously, inform the user. + stat = nodeinfo.npredecessors + if stat != _NODE_OUT: + if stat >= 0: + raise ValueError( + f"node {node!r} was not passed out (still not ready)" + ) + elif stat == _NODE_DONE: + raise ValueError(f"node {node!r} was already marked done") + else: + assert False, f"node {node!r}: unknown status {stat}" + + # Mark the node as processed + nodeinfo.npredecessors = _NODE_DONE + + # Go to all the successors and reduce the number of predecessors, collecting all the ones + # that are ready to be returned in the next get_ready() call. + for successor in nodeinfo.successors: + successor_info = n2i[successor] + successor_info.npredecessors -= 1 + if successor_info.npredecessors == 0: + self._ready_nodes.append(successor) + self._nfinished += 1 + + def _find_cycle(self): + n2i = self._node2info + stack = [] + itstack = [] + seen = set() + node2stacki = {} + + for node in n2i: + if node in seen: + continue + + while True: + if node in seen: + # If we have seen already the node and is in the + # current stack we have found a cycle. + if node in node2stacki: + return stack[node2stacki[node] :] + [node] + # else go on to get next successor + else: + seen.add(node) + itstack.append(iter(n2i[node].successors).__next__) + node2stacki[node] = len(stack) + stack.append(node) + + # Backtrack to the topmost stack entry with + # at least another successor. + while stack: + try: + node = itstack[-1]() + break + except StopIteration: + del node2stacki[stack.pop()] + itstack.pop() + else: + break + return None + + def static_order(self): + """Returns an iterable of nodes in a topological order. + + The particular order that is returned may depend on the specific + order in which the items were inserted in the graph. + + Using this method does not require to call "prepare" or "done". If any + cycle is detected, :exc:`CycleError` will be raised. + """ + self.prepare() + while self.is_active(): + node_group = self.get_ready() + yield from node_group + self.done(*node_group) diff --git a/tests/apps/schema/commands/test_generate.py b/tests/apps/schema/commands/test_generate.py index db3ee04db..781660ad4 100644 --- a/tests/apps/schema/commands/test_generate.py +++ b/tests/apps/schema/commands/test_generate.py @@ -1,7 +1,9 @@ from __future__ import annotations +import ast import typing as t from unittest import TestCase +from unittest.mock import MagicMock, patch from piccolo.apps.schema.commands.generate import ( OutputSchema, @@ -117,8 +119,14 @@ def test_get_output_schema(self): SmallTable_ = output_schema.get_table_with_name("SmallTable") self._compare_table_columns(SmallTable, SmallTable_) - def test_generate(self): + @patch("piccolo.apps.schema.commands.generate.print") + def test_generate(self, print_: MagicMock): """ Test the main generate command runs without errors. """ run_sync(generate()) + file_contents = print_.call_args[0][0] + + # Make sure the output is valid Python code (will raise a SyntaxError + # exception otherwise). + ast.parse(file_contents) From 27d6f5b344c0ef3c63d0a72afffc2345c883861b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 10 Sep 2021 17:36:12 +0100 Subject: [PATCH 057/727] bumped version --- CHANGES | 7 +++++++ piccolo/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index f546b5d8a..7d903dcf6 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,13 @@ Changes ======= +0.45.1 +------ +Improvements to ``piccolo schema generate``. It's now smarter about which +imports to include. Also, the ``Table`` classes output will now be sorted based +on their ``ForeignKey`` columns. Internally the sorting algorithm has been +changed to use the ``graphlib`` module, which was added in Python 3.9. + 0.45.0 ------ Added the ``piccolo schema graph`` command for visualising your database diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 06e3e25dc..bb956ef59 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.45.0" +__VERSION__ = "0.45.1" From e9cf6c4232737a2b4994b3c63e2ec210430b7a12 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 10 Sep 2021 19:09:50 +0100 Subject: [PATCH 058/727] extra test for table sorting (#227) * extra test for table sorting * add a test with only a single table --- .../migrations/auto/test_migration_manager.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/apps/migrations/auto/test_migration_manager.py b/tests/apps/migrations/auto/test_migration_manager.py index 0eabe7651..80f19fd89 100644 --- a/tests/apps/migrations/auto/test_migration_manager.py +++ b/tests/apps/migrations/auto/test_migration_manager.py @@ -11,6 +11,7 @@ from piccolo.columns.base import OnDelete, OnUpdate from piccolo.columns.column_types import ForeignKey from piccolo.conf.apps import AppConfig +from piccolo.table import Table from piccolo.utils.lazy_loader import LazyLoader from tests.base import DBTestCase, postgres_only, set_mock_return_value from tests.example_app.tables import Band, Concert, Manager, Venue @@ -34,6 +35,29 @@ def test_sort_table_classes(self): sorted_tables.index(Band) < sorted_tables.index(Concert) ) + def test_sort_unrelated_tables(self): + """ + Make sure there are no weird edge cases with tables with no foreign + key relationships with each other. + """ + + class TableA(Table): + pass + + class TableB(Table): + pass + + self.assertEqual( + sort_table_classes([TableA, TableB]), [TableA, TableB] + ) + + def test_single_table(self): + """ + Make sure that sorting a list with only a single table in it still + works. + """ + self.assertEqual(sort_table_classes([Band]), [Band]) + class TestMigrationManager(DBTestCase): @postgres_only From 104c1a5ae9f534c4e1642b8ae79340c7f5b6cceb Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 11 Sep 2021 09:37:10 +0100 Subject: [PATCH 059/727] add `mirror` as an alias for `piccolo schema generate` --- piccolo/apps/schema/piccolo_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piccolo/apps/schema/piccolo_app.py b/piccolo/apps/schema/piccolo_app.py index 6f8ab0b1d..ae5449d65 100644 --- a/piccolo/apps/schema/piccolo_app.py +++ b/piccolo/apps/schema/piccolo_app.py @@ -7,7 +7,7 @@ app_name="schema", migrations_folder_path="", commands=[ - Command(callable=generate, aliases=["gen", "create", "new"]), + Command(callable=generate, aliases=["gen", "create", "new", "mirror"]), Command( callable=graph, aliases=["map", "visualise", "vizualise", "viz", "vis"], From 48768ba27b4348026cd2b1e65350aad2a46e9767 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 14 Sep 2021 09:12:40 +0100 Subject: [PATCH 060/727] Fixtures app (#229) * prototype for fixture dumping * add pretty option to dump_json * add fixture loading * added docs * improved docs - mentioned how multiple apps / tables can be specified * renamed app (fixtures -> fixture) * moved `MegaTable` into it's own file for reuse * refactored test file, to support multiple example apps * fix typo * added `digits` arg to `MegaTable.numeric_col` * use pydantic to serialise / deserialise json * tweaked docs * fix test * conditional asyncpg import * add missing docstrings * make TestDumpLoad work with sqlite * added tests for `create_pydantic_model` --- .../projects_and_apps/included_apps.rst | 48 ++- .../apps/fixture}/__init__.py | 0 piccolo/apps/fixture/commands/__init__.py | 0 piccolo/apps/fixture/commands/dump.py | 134 +++++++ piccolo/apps/fixture/commands/load.py | 72 ++++ piccolo/apps/fixture/commands/shared.py | 57 +++ piccolo/apps/fixture/piccolo_app.py | 11 + piccolo/engine/base.py | 4 + piccolo/engine/sqlite.py | 14 +- piccolo/main.py | 2 + piccolo/utils/encoding.py | 14 +- piccolo/utils/pydantic.py | 192 ++++++++++ requirements/requirements.txt | 1 + tests/apps/app/commands/test_show_all.py | 6 +- tests/apps/fixture/__init__.py | 0 tests/apps/fixture/commands/__init__.py | 0 tests/apps/fixture/commands/test_dump_load.py | 131 +++++++ tests/apps/fixture/commands/test_shared.py | 60 ++++ .../migrations/auto/test_migration_manager.py | 2 +- .../migrations/auto/test_serialisation.py | 2 +- .../commands/test_forwards_backwards.py | 2 +- tests/apps/migrations/commands/test_new.py | 2 +- tests/apps/schema/commands/test_generate.py | 53 +-- tests/apps/shell/commands/test_run.py | 3 + tests/columns/test_choices.py | 2 +- tests/columns/test_combination.py | 2 +- tests/columns/test_foreignkey.py | 2 +- tests/columns/test_reference.py | 8 +- tests/conf/test_apps.py | 13 +- tests/engine/test_nested_transaction.py | 2 +- tests/engine/test_pool.py | 2 +- tests/engine/test_transaction.py | 3 +- tests/example_apps/__init__.py | 0 tests/example_apps/mega/__init__.py | 0 tests/example_apps/mega/piccolo_app.py | 17 + .../mega/piccolo_migrations/__init__.py | 0 tests/example_apps/mega/tables.py | 56 +++ tests/example_apps/music/__init__.py | 0 .../music}/piccolo_app.py | 0 .../piccolo_migrations/2020-12-17T18-44-30.py | 0 .../piccolo_migrations/2020-12-17T18-44-39.py | 0 .../piccolo_migrations/2020-12-17T18-44-44.py | 0 .../2021-07-25T22-38-48-009306.py | 0 .../2021-09-06T13-58-23-024723.py | 0 .../music}/tables.py | 0 tests/postgres_conf.py | 7 +- tests/query/test_await.py | 3 +- tests/query/test_freeze.py | 2 +- tests/query/test_mixins.py | 2 +- tests/query/test_slots.py | 3 +- tests/sqlite_conf.py | 7 +- tests/table/instance/test_get_related.py | 2 +- .../instance/test_get_related_readable.py | 2 +- tests/table/instance/test_instantiate.py | 2 +- tests/table/instance/test_remove.py | 2 +- tests/table/instance/test_save.py | 2 +- tests/table/instance/test_to_dict.py | 2 +- tests/table/test_all_columns.py | 2 +- tests/table/test_alter.py | 2 +- tests/table/test_batch.py | 3 +- tests/table/test_count.py | 3 +- tests/table/test_create.py | 3 +- tests/table/test_delete.py | 2 +- tests/table/test_exists.py | 3 +- tests/table/test_indexes.py | 3 +- tests/table/test_insert.py | 3 +- tests/table/test_join.py | 8 +- tests/table/test_metaclass.py | 2 +- tests/table/test_objects.py | 2 +- tests/table/test_output.py | 2 +- tests/table/test_raw.py | 3 +- tests/table/test_ref.py | 2 +- tests/table/test_repr.py | 3 +- tests/table/test_select.py | 2 +- tests/table/test_str.py | 2 +- tests/table/test_table_exists.py | 2 +- tests/table/test_update.py | 3 +- tests/testing/test_model_builder.py | 3 +- tests/utils/test_pydantic.py | 331 ++++++++++++++++++ 79 files changed, 1228 insertions(+), 114 deletions(-) rename {tests/example_app => piccolo/apps/fixture}/__init__.py (100%) create mode 100644 piccolo/apps/fixture/commands/__init__.py create mode 100644 piccolo/apps/fixture/commands/dump.py create mode 100644 piccolo/apps/fixture/commands/load.py create mode 100644 piccolo/apps/fixture/commands/shared.py create mode 100644 piccolo/apps/fixture/piccolo_app.py create mode 100644 piccolo/utils/pydantic.py create mode 100644 tests/apps/fixture/__init__.py create mode 100644 tests/apps/fixture/commands/__init__.py create mode 100644 tests/apps/fixture/commands/test_dump_load.py create mode 100644 tests/apps/fixture/commands/test_shared.py create mode 100644 tests/example_apps/__init__.py create mode 100644 tests/example_apps/mega/__init__.py create mode 100644 tests/example_apps/mega/piccolo_app.py create mode 100644 tests/example_apps/mega/piccolo_migrations/__init__.py create mode 100644 tests/example_apps/mega/tables.py create mode 100644 tests/example_apps/music/__init__.py rename tests/{example_app => example_apps/music}/piccolo_app.py (100%) rename tests/{example_app => example_apps/music}/piccolo_migrations/2020-12-17T18-44-30.py (100%) rename tests/{example_app => example_apps/music}/piccolo_migrations/2020-12-17T18-44-39.py (100%) rename tests/{example_app => example_apps/music}/piccolo_migrations/2020-12-17T18-44-44.py (100%) rename tests/{example_app => example_apps/music}/piccolo_migrations/2021-07-25T22-38-48-009306.py (100%) rename tests/{example_app => example_apps/music}/piccolo_migrations/2021-09-06T13-58-23-024723.py (100%) rename tests/{example_app => example_apps/music}/tables.py (100%) create mode 100644 tests/utils/test_pydantic.py diff --git a/docs/src/piccolo/projects_and_apps/included_apps.rst b/docs/src/piccolo/projects_and_apps/included_apps.rst index c2f7cc54c..89431692d 100644 --- a/docs/src/piccolo/projects_and_apps/included_apps.rst +++ b/docs/src/piccolo/projects_and_apps/included_apps.rst @@ -4,10 +4,15 @@ Included Apps Just as you can modularise your own code using :ref:`apps`, Piccolo itself ships with several builtin apps, which provide a lot of its functionality. +------------------------------------------------------------------------------- + Auto includes ------------- -The following are registered with your :ref:`AppRegistry` automatically: +The following are registered with your :ref:`AppRegistry` automatically. + +.. hint:: To find out more about each of these commands you can use the + ``--help`` flag on the command line. For example ``piccolo app new --help``. ------------------------------------------------------------------------------- @@ -33,6 +38,47 @@ Lets you scaffold an ASGI web app. See :ref:`ASGICommand`. ------------------------------------------------------------------------------- +fixture +~~~~~~~ + +Fixtures are used when you want to seed your database with essential data (for +example, country names). + +Once you have created a fixture, it can be used by your colleagues when setting +up an application on their local machines, or when deploying to a new +environment. + +Databases such as Postgres have inbuilt ways of dumping and restoring data +(via ``pg_dump`` and ``pg_restore``). Some reasons to use the fixtures app +instead: + + * When you want the data to be loadable in a range of database versions. + * Fixtures are stored in JSON, which are a bit friendlier for source control. + +To dump the data into a new fixture file: + +.. code-block:: bash + + piccolo fixtures dump > fixtures.json + +By default, the fixture contains data from all apps and tables. You can specify +a subset of apps and tables instead, for example: + +.. code-block:: bash + + piccolo fixture dump --apps=blog --tables=Post > fixtures.json + + # Or for multiple apps / tables + piccolo fixtures dump --apps=blog,shop --tables=Post,Product > fixtures.json + +To load the fixture: + +.. code-block:: bash + + piccolo fixture load fixtures.json + +------------------------------------------------------------------------------- + meta ~~~~ diff --git a/tests/example_app/__init__.py b/piccolo/apps/fixture/__init__.py similarity index 100% rename from tests/example_app/__init__.py rename to piccolo/apps/fixture/__init__.py diff --git a/piccolo/apps/fixture/commands/__init__.py b/piccolo/apps/fixture/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/piccolo/apps/fixture/commands/dump.py b/piccolo/apps/fixture/commands/dump.py new file mode 100644 index 000000000..1f13ed199 --- /dev/null +++ b/piccolo/apps/fixture/commands/dump.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import typing as t + +from piccolo.apps.fixture.commands.shared import ( + FixtureConfig, + create_pydantic_fixture_model, +) +from piccolo.apps.migrations.auto.migration_manager import sort_table_classes +from piccolo.conf.apps import Finder + + +async def get_dump( + fixture_configs: t.List[FixtureConfig], +) -> t.Dict[str, t.Any]: + """ + Gets the data for each table specified and returns a data structure like: + + .. code-block:: python + + { + 'my_app_name': { + 'MyTableName': [ + { + 'id': 1, + 'my_column_name': 'foo' + } + ] + } + } + + """ + finder = Finder() + + output: t.Dict[str, t.Any] = {} + + for fixture_config in fixture_configs: + app_config = finder.get_app_config(app_name=fixture_config.app_name) + table_classes = [ + i + for i in app_config.table_classes + if i.__name__ in fixture_config.table_class_names + ] + sorted_table_classes = sort_table_classes(table_classes) + + output[fixture_config.app_name] = {} + + for table_class in sorted_table_classes: + data = await table_class.select().run() + output[fixture_config.app_name][table_class.__name__] = data + + return output + + +async def dump_to_json_string( + fixture_configs: t.List[FixtureConfig], +) -> str: + """ + Dumps all of the data for the given tables into a JSON string. + """ + dump = await get_dump(fixture_configs=fixture_configs) + pydantic_model = create_pydantic_fixture_model( + fixture_configs=fixture_configs + ) + json_output = pydantic_model(**dump).json() + return json_output + + +def parse_args(apps: str, tables: str) -> t.List[FixtureConfig]: + """ + Works out which apps and tables the user is referring to. + """ + finder = Finder() + app_names = [] + + if apps == "all": + app_names = finder.get_sorted_app_names() + elif "," in apps: + app_names = apps.split(",") + else: + # Must be a single app name + app_names.append(apps) + + table_class_names: t.Optional[t.List[str]] = None + + if tables != "all": + if "," in tables: + table_class_names = tables.split(",") + else: + # Must be a single table class name + table_class_names = [tables] + + output: t.List[FixtureConfig] = [] + + for app_name in app_names: + app_config = finder.get_app_config(app_name=app_name) + table_classes = app_config.table_classes + + if table_class_names is None: + fixture_configs = [i.__name__ for i in table_classes] + else: + fixture_configs = [ + i.__name__ + for i in table_classes + if i.__name__ in table_class_names + ] + output.append( + FixtureConfig( + app_name=app_name, + table_class_names=fixture_configs, + ) + ) + + return output + + +async def dump(apps: str = "all", tables: str = "all"): + """ + Serialises the data from the given Piccolo apps / tables, and prints it + out. + + :param apps: + For all apps, specify `all`. For specific apps, pass in a comma + separated list e.g. `blog,profiles,billing`. For a single app, just + pass in the name of that app, e.g. `blog`. + :param tables: + For all tables, specify `all`. For specific tables, pass in a comma + separated list e.g. `Post,Tag`. For a single app, just + pass in the name of that app, e.g. `Post`. + + """ + fixture_configs = parse_args(apps=apps, tables=tables) + json_string = await dump_to_json_string(fixture_configs=fixture_configs) + print(json_string) diff --git a/piccolo/apps/fixture/commands/load.py b/piccolo/apps/fixture/commands/load.py new file mode 100644 index 000000000..64cfc2ba0 --- /dev/null +++ b/piccolo/apps/fixture/commands/load.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from piccolo.apps.fixture.commands.shared import ( + FixtureConfig, + create_pydantic_fixture_model, +) +from piccolo.conf.apps import Finder +from piccolo.engine import engine_finder +from piccolo.utils.encoding import load_json + + +async def load_json_string(json_string: str): + """ + Parses the JSON string, and inserts the parsed data into the database. + """ + # We have to deserialise the JSON to find out which apps and tables it + # contains, so we can create a Pydantic model. + # Then we let Pydantic do the proper deserialisation, as it does a much + # better job of deserialising dates, datetimes, bytes etc. + deserialised_contents = load_json(json_string) + + app_names = deserialised_contents.keys() + + fixture_configs = [ + FixtureConfig( + app_name=app_name, + table_class_names=[ + i for i in deserialised_contents[app_name].keys() + ], + ) + for app_name in app_names + ] + pydantic_model_class = create_pydantic_fixture_model( + fixture_configs=fixture_configs + ) + + fixture_pydantic_model = pydantic_model_class.parse_raw(json_string) + + finder = Finder() + engine = engine_finder() + + if not engine: + raise Exception("Unable to find the engine.") + + async with engine.transaction(): + for app_name in app_names: + app_model = getattr(fixture_pydantic_model, app_name) + + for ( + table_class_name, + model_instance_list, + ) in app_model.__dict__.items(): + table_class = finder.get_table_with_name( + app_name, table_class_name + ) + + await table_class.insert( + *[ + table_class(**row.__dict__) + for row in model_instance_list + ] + ).run() + + +async def load(path: str = "fixture.json"): + """ + Reads the fixture file, and loads the contents into the database. + """ + with open(path, "r") as f: + contents = f.read() + + await load_json_string(contents) diff --git a/piccolo/apps/fixture/commands/shared.py b/piccolo/apps/fixture/commands/shared.py new file mode 100644 index 000000000..c2e2c39f8 --- /dev/null +++ b/piccolo/apps/fixture/commands/shared.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import typing as t +from dataclasses import dataclass + +import pydantic + +from piccolo.conf.apps import Finder +from piccolo.utils.pydantic import create_pydantic_model + +if t.TYPE_CHECKING: + from piccolo.table import Table + + +@dataclass +class FixtureConfig: + app_name: str + table_class_names: t.List[str] + + +def create_pydantic_fixture_model(fixture_configs: t.List[FixtureConfig]): + """ + Returns a nested Pydantic model for serialising and deserialising fixtures. + """ + columns: t.Dict[str, t.Any] = {} + + finder = Finder() + + for fixture_config in fixture_configs: + + app_columns: t.Dict[str, t.Any] = {} + + for table_class_name in fixture_config.table_class_names: + table_class: t.Type[Table] = finder.get_table_with_name( + app_name=fixture_config.app_name, + table_class_name=table_class_name, + ) + app_columns[table_class_name] = ( + t.List[ # type: ignore + create_pydantic_model( + table_class, include_default_columns=True + ) + ], + ..., + ) + + app_model: t.Any = pydantic.create_model( + f"{fixture_config.app_name.title()}Model", **app_columns + ) + + columns[fixture_config.app_name] = (app_model, ...) + + model: t.Type[pydantic.BaseModel] = pydantic.create_model( + "FixtureModel", **columns + ) + + return model diff --git a/piccolo/apps/fixture/piccolo_app.py b/piccolo/apps/fixture/piccolo_app.py new file mode 100644 index 000000000..79e7665cc --- /dev/null +++ b/piccolo/apps/fixture/piccolo_app.py @@ -0,0 +1,11 @@ +from piccolo.conf.apps import AppConfig + +from .commands.dump import dump + +APP_CONFIG = AppConfig( + app_name="fixtures", + migrations_folder_path="", + table_classes=[], + migration_dependencies=[], + commands=[dump], +) diff --git a/piccolo/engine/base.py b/piccolo/engine/base.py index 657e6145e..3da74b66d 100644 --- a/piccolo/engine/base.py +++ b/piccolo/engine/base.py @@ -54,6 +54,10 @@ async def run_querystring(self, querystring: QueryString, in_pool: bool): async def run_ddl(self, ddl: str, in_pool: bool = True): pass + @abstractmethod + def transaction(self): + pass + async def check_version(self): """ Warn if the database version isn't supported. diff --git a/piccolo/engine/sqlite.py b/piccolo/engine/sqlite.py index cb84f3148..538440077 100644 --- a/piccolo/engine/sqlite.py +++ b/piccolo/engine/sqlite.py @@ -141,12 +141,21 @@ def convert_boolean_out(value: bytes) -> bool: return _value == "1" +def convert_timestamp_out(value: bytes) -> datetime.datetime: + """ + If the value is from a timestamp column, convert it to a datetime value. + """ + return datetime.datetime.fromisoformat(value.decode("utf8")) + + def convert_timestamptz_out(value: bytes) -> datetime.datetime: """ - If the value is from a timstamptz column, convert it to a datetime value, + If the value is from a timestamptz column, convert it to a datetime value, with a timezone of UTC. """ - return datetime.datetime.fromisoformat(value.decode("utf8")) + _value = datetime.datetime.fromisoformat(value.decode("utf8")) + _value = _value.replace(tzinfo=datetime.timezone.utc) + return _value def convert_array_out(value: bytes) -> t.List: @@ -164,6 +173,7 @@ def convert_array_out(value: bytes) -> t.List: sqlite3.register_converter("Time", convert_time_out) sqlite3.register_converter("Seconds", convert_seconds_out) sqlite3.register_converter("Boolean", convert_boolean_out) +sqlite3.register_converter("Timestamp", convert_timestamp_out) sqlite3.register_converter("Timestamptz", convert_timestamptz_out) sqlite3.register_converter("Array", convert_array_out) diff --git a/piccolo/main.py b/piccolo/main.py index 0c3098cb3..f0bc5f9bb 100644 --- a/piccolo/main.py +++ b/piccolo/main.py @@ -12,6 +12,7 @@ from piccolo.apps.app.piccolo_app import APP_CONFIG as app_config from piccolo.apps.asgi.piccolo_app import APP_CONFIG as asgi_config +from piccolo.apps.fixture.piccolo_app import APP_CONFIG as fixture_config from piccolo.apps.meta.piccolo_app import APP_CONFIG as meta_config from piccolo.apps.migrations.commands.check import CheckMigrationManager from piccolo.apps.migrations.piccolo_app import APP_CONFIG as migrations_config @@ -60,6 +61,7 @@ def main(): for _app_config in [ app_config, asgi_config, + fixture_config, meta_config, migrations_config, playground_config, diff --git a/piccolo/utils/encoding.py b/piccolo/utils/encoding.py index e84b1b015..3feda117f 100644 --- a/piccolo/utils/encoding.py +++ b/piccolo/utils/encoding.py @@ -12,11 +12,19 @@ ORJSON = False -def dump_json(data: t.Any) -> str: +def dump_json(data: t.Any, pretty: bool = False) -> str: if ORJSON: - return orjson.dumps(data, default=str).decode("utf8") + orjson_params: t.Dict[str, t.Any] = {"default": str} + if pretty: + orjson_params["option"] = ( + orjson.OPT_INDENT_2 | orjson.OPT_APPEND_NEWLINE # type: ignore + ) + return orjson.dumps(data, **orjson_params).decode("utf8") else: - return json.dumps(data, default=str) + params: t.Dict[str, t.Any] = {"default": str} + if pretty: + params["indent"] = 2 + return json.dumps(data, **params) def load_json(data: str) -> t.Any: diff --git a/piccolo/utils/pydantic.py b/piccolo/utils/pydantic.py new file mode 100644 index 000000000..889c7cf8f --- /dev/null +++ b/piccolo/utils/pydantic.py @@ -0,0 +1,192 @@ +from __future__ import annotations + +import json +import typing as t +import uuid +from functools import lru_cache + +import pydantic + +from piccolo.columns import Column +from piccolo.columns.column_types import ( + JSON, + JSONB, + Array, + Decimal, + ForeignKey, + Numeric, + Secret, + Text, + Varchar, +) +from piccolo.table import Table +from piccolo.utils.encoding import load_json + +try: + from asyncpg.pgproto.pgproto import UUID # type: ignore +except ImportError: + JSON_ENCODERS = {uuid.UUID: lambda i: str(i)} +else: + JSON_ENCODERS = {uuid.UUID: lambda i: str(i), UUID: lambda i: str(i)} + + +class Config(pydantic.BaseConfig): + json_encoders = JSON_ENCODERS + arbitrary_types_allowed = True + + +def pydantic_json_validator(cls, value): + try: + load_json(value) + except json.JSONDecodeError: + raise ValueError("Unable to parse the JSON.") + else: + return value + + +@lru_cache() +def create_pydantic_model( + table: t.Type[Table], + nested: bool = False, + exclude_columns: t.Tuple[Column, ...] = (), + include_default_columns: bool = False, + include_readable: bool = False, + all_optional: bool = False, + model_name: t.Optional[str] = None, + deserialize_json: bool = False, +) -> t.Type[pydantic.BaseModel]: + """ + Create a Pydantic model representing a table. + + :param table: + The Piccolo ``Table`` you want to create a Pydantic serialiser model + for. + :param nested: + Whether ``ForeignKey`` columns are converted to nested Pydantic models. + :param exclude_columns: + A tuple of ``Column`` instances that should be excluded from the + Pydantic model. + :param include_default_columns: + Whether to include columns like ``id`` in the serialiser. You will + typically include these columns in GET requests, but don't require + them in POST requests. + :param include_readable: + Whether to include 'readable' columns, which give a string + representation of a foreign key. + :param all_optional: + If True, all fields are optional. Useful for filters etc. + :param model_name: + By default, the classname of the Piccolo ``Table`` will be used, but + you can override it if you want multiple Pydantic models based off the + same Piccolo table. + :param deserialize_json: + By default, the values of any Piccolo JSON or JSONB columns are + returned as strings. By setting this parameter to True, they will be + returned as objects. + :returns: + A Pydantic model. + + """ + columns: t.Dict[str, t.Any] = {} + validators: t.Dict[str, classmethod] = {} + piccolo_columns = ( + table._meta.columns + if include_default_columns + else table._meta.non_default_columns + ) + + if not all( + isinstance(column, Column) + # Make sure that every column is tied to the current Table + and column._meta.table is table + for column in exclude_columns + ): + raise ValueError(f"Exclude columns ({exclude_columns!r}) are invalid.") + + for column in piccolo_columns: + # normal __contains__ checks __eq__ as well which returns ``Where`` + # instance which always evaluates to ``True`` + if any(column is obj for obj in exclude_columns): + continue + + column_name = column._meta.name + + is_optional = True if all_optional else not column._meta.required + + ####################################################################### + + # Work out the column type + + if isinstance(column, (Decimal, Numeric)): + value_type: t.Type = pydantic.condecimal( + max_digits=column.precision, decimal_places=column.scale + ) + elif isinstance(column, Varchar): + value_type = pydantic.constr(max_length=column.length) + elif isinstance(column, Array): + value_type = t.List[column.base_column.value_type] # type: ignore + elif isinstance(column, (JSON, JSONB)): + if deserialize_json: + value_type = pydantic.Json + else: + value_type = column.value_type + validators[f"{column_name}_is_json"] = pydantic.validator( + column_name, allow_reuse=True + )(pydantic_json_validator) + else: + value_type = column.value_type + + _type = t.Optional[value_type] if is_optional else value_type + + ####################################################################### + + params: t.Dict[str, t.Any] = { + "default": None if is_optional else ..., + "nullable": column._meta.null, + } + + extra = { + "help_text": column._meta.help_text, + "choices": column._meta.get_choices_dict(), + } + + if isinstance(column, ForeignKey): + if nested: + _type = create_pydantic_model( + table=column._foreign_key_meta.resolved_references, + nested=True, + include_default_columns=include_default_columns, + include_readable=include_readable, + all_optional=all_optional, + deserialize_json=deserialize_json, + ) + + tablename = ( + column._foreign_key_meta.resolved_references._meta.tablename + ) + field = pydantic.Field( + extra={"foreign_key": True, "to": tablename, **extra}, + **params, + ) + if include_readable: + columns[f"{column_name}_readable"] = (str, None) + elif isinstance(column, Text): + field = pydantic.Field(format="text-area", extra=extra, **params) + elif isinstance(column, Secret): + field = pydantic.Field(extra={"secret": True, **extra}) + else: + field = pydantic.Field(extra=extra, **params) + + columns[column_name] = (_type, field) + + model_name = model_name or table.__name__ + + class CustomConfig(Config): + schema_extra = {"help_text": table._meta.help_text} + + return pydantic.create_model( + model_name, + __config__=CustomConfig, + __validators__=validators, + **columns, + ) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 0838d37ea..46f200de6 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -4,3 +4,4 @@ Jinja2>=2.11.0 targ>=0.3.3 inflection>=0.5.1 typing-extensions>=3.10.0.0 +pydantic>=1.6 diff --git a/tests/apps/app/commands/test_show_all.py b/tests/apps/app/commands/test_show_all.py index 0800c042d..92404a412 100644 --- a/tests/apps/app/commands/test_show_all.py +++ b/tests/apps/app/commands/test_show_all.py @@ -11,5 +11,9 @@ def test_show_all(self, print_: MagicMock): self.assertEqual( print_.mock_calls, - [call("Registered apps:"), call("tests.example_app.piccolo_app")], + [ + call("Registered apps:"), + call("tests.example_apps.music.piccolo_app"), + call("tests.example_apps.mega.piccolo_app"), + ], ) diff --git a/tests/apps/fixture/__init__.py b/tests/apps/fixture/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/apps/fixture/commands/__init__.py b/tests/apps/fixture/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/apps/fixture/commands/test_dump_load.py b/tests/apps/fixture/commands/test_dump_load.py new file mode 100644 index 000000000..9f4b6693f --- /dev/null +++ b/tests/apps/fixture/commands/test_dump_load.py @@ -0,0 +1,131 @@ +import datetime +import decimal +import uuid +from unittest import TestCase + +from piccolo.apps.fixture.commands.dump import ( + FixtureConfig, + dump_to_json_string, +) +from piccolo.apps.fixture.commands.load import load_json_string +from piccolo.utils.sync import run_sync +from tests.example_apps.mega.tables import MegaTable, SmallTable + + +class TestDumpLoad(TestCase): + """ + Test the fixture dump and load commands - makes sense to test them + together. + """ + + def setUp(self): + for table_class in (SmallTable, MegaTable): + table_class.create_table().run_sync() + + def tearDown(self): + for table_class in (MegaTable, SmallTable): + table_class.alter().drop_table().run_sync() + + def insert_row(self): + small_table = SmallTable(varchar_col="Test") + small_table.save().run_sync() + + mega_table = MegaTable( + bigint_col=1, + boolean_col=True, + bytea_col="hello".encode("utf8"), + date_col=datetime.date(year=2021, month=1, day=1), + foreignkey_col=small_table, + integer_col=1, + interval_col=datetime.timedelta(seconds=10), + json_col={"a": 1}, + jsonb_col={"a": 1}, + numeric_col=decimal.Decimal("1.1"), + real_col=1.1, + smallint_col=1, + text_col="hello", + timestamp_col=datetime.datetime(year=2021, month=1, day=1), + timestamptz_col=datetime.datetime(year=2021, month=1, day=1), + uuid_col=uuid.UUID("12783854-c012-4c15-8183-8eecb46f2c4e"), + varchar_col="hello", + unique_col="hello", + null_col=None, + not_null_col="hello", + ) + mega_table.save().run_sync() + + def test_dump_load(self): + """ + Make sure we can dump some rows into a JSON fixture, then load them + back into the database. + """ + self.insert_row() + + json_string = run_sync( + dump_to_json_string( + fixture_configs=[ + FixtureConfig( + app_name="mega", + table_class_names=["SmallTable", "MegaTable"], + ) + ] + ) + ) + + # We need to clear the data out now, otherwise when loading the data + # back in, there will be a constraint errors over clashing primary + # keys. + SmallTable.delete(force=True).run_sync() + MegaTable.delete(force=True).run_sync() + + run_sync(load_json_string(json_string)) + + self.assertEqual( + SmallTable.select().run_sync(), + [{"id": 1, "varchar_col": "Test"}], + ) + + mega_table_data = MegaTable.select().run_sync() + + # Real numbers don't have perfect precision when coming back from the + # database, so we need to round them to be able to compare them. + mega_table_data[0]["real_col"] = round( + mega_table_data[0]["real_col"], 1 + ) + + # Remove white space from the JSON values + for col_name in ("json_col", "jsonb_col"): + mega_table_data[0][col_name] = mega_table_data[0][ + col_name + ].replace(" ", "") + + self.assertTrue(len(mega_table_data) == 1) + + self.assertDictEqual( + mega_table_data[0], + { + "id": 1, + "bigint_col": 1, + "boolean_col": True, + "bytea_col": b"hello", + "date_col": datetime.date(2021, 1, 1), + "foreignkey_col": 1, + "integer_col": 1, + "interval_col": datetime.timedelta(seconds=10), + "json_col": '{"a":1}', + "jsonb_col": '{"a":1}', + "numeric_col": decimal.Decimal("1.1"), + "real_col": 1.1, + "smallint_col": 1, + "text_col": "hello", + "timestamp_col": datetime.datetime(2021, 1, 1, 0, 0), + "timestamptz_col": datetime.datetime( + 2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc + ), + "uuid_col": uuid.UUID("12783854-c012-4c15-8183-8eecb46f2c4e"), + "varchar_col": "hello", + "unique_col": "hello", + "null_col": None, + "not_null_col": "hello", + }, + ) diff --git a/tests/apps/fixture/commands/test_shared.py b/tests/apps/fixture/commands/test_shared.py new file mode 100644 index 000000000..7b56dabd1 --- /dev/null +++ b/tests/apps/fixture/commands/test_shared.py @@ -0,0 +1,60 @@ +import datetime +import decimal +import uuid +from unittest import TestCase + +from piccolo.apps.fixture.commands.shared import ( + FixtureConfig, + create_pydantic_fixture_model, +) + + +class TestShared(TestCase): + def test_shared(self): + pydantic_model = create_pydantic_fixture_model( + fixture_configs=[ + FixtureConfig( + app_name="mega", + table_class_names=["MegaTable", "SmallTable"], + ) + ] + ) + + data = { + "mega": { + "SmallTable": [{"id": 1, "varchar_col": "Test"}], + "MegaTable": [ + { + "id": 1, + "bigint_col": 1, + "boolean_col": True, + "bytea_col": b"hello", + "date_col": datetime.date(2021, 1, 1), + "foreignkey_col": 1, + "integer_col": 1, + "interval_col": datetime.timedelta(seconds=10), + "json_col": '{"a":1}', + "jsonb_col": '{"a": 1}', + "numeric_col": decimal.Decimal("1.10"), + "real_col": 1.100000023841858, + "smallint_col": 1, + "text_col": "hello", + "timestamp_col": datetime.datetime(2021, 1, 1, 0, 0), + "timestamptz_col": datetime.datetime( + 2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc + ), + "uuid_col": uuid.UUID( + "12783854-c012-4c15-8183-8eecb46f2c4e" + ), + "varchar_col": "hello", + "unique_col": "hello", + "null_col": None, + "not_null_col": "hello", + } + ], + } + } + + model = pydantic_model(**data) + self.assertEqual(model.mega.SmallTable[0].id, 1) + self.assertEqual(model.mega.MegaTable[0].id, 1) diff --git a/tests/apps/migrations/auto/test_migration_manager.py b/tests/apps/migrations/auto/test_migration_manager.py index 80f19fd89..124ed1ba4 100644 --- a/tests/apps/migrations/auto/test_migration_manager.py +++ b/tests/apps/migrations/auto/test_migration_manager.py @@ -14,7 +14,7 @@ from piccolo.table import Table from piccolo.utils.lazy_loader import LazyLoader from tests.base import DBTestCase, postgres_only, set_mock_return_value -from tests.example_app.tables import Band, Concert, Manager, Venue +from tests.example_apps.music.tables import Band, Concert, Manager, Venue asyncpg = LazyLoader("asyncpg", globals(), "asyncpg") diff --git a/tests/apps/migrations/auto/test_serialisation.py b/tests/apps/migrations/auto/test_serialisation.py index 8cc0639cf..c97416920 100644 --- a/tests/apps/migrations/auto/test_serialisation.py +++ b/tests/apps/migrations/auto/test_serialisation.py @@ -45,7 +45,7 @@ def test_lazy_table_reference(self): ), LazyTableReference( table_class_name="Manager", - module_path="tests.example_app.tables", + module_path="tests.example_apps.music.tables", ), ] diff --git a/tests/apps/migrations/commands/test_forwards_backwards.py b/tests/apps/migrations/commands/test_forwards_backwards.py index fa205eaf9..6449c05da 100644 --- a/tests/apps/migrations/commands/test_forwards_backwards.py +++ b/tests/apps/migrations/commands/test_forwards_backwards.py @@ -10,7 +10,7 @@ from piccolo.apps.migrations.tables import Migration from piccolo.utils.sync import run_sync from tests.base import postgres_only -from tests.example_app.tables import ( +from tests.example_apps.music.tables import ( Band, Concert, Manager, diff --git a/tests/apps/migrations/commands/test_new.py b/tests/apps/migrations/commands/test_new.py index 5d7fa266f..bb212db80 100644 --- a/tests/apps/migrations/commands/test_new.py +++ b/tests/apps/migrations/commands/test_new.py @@ -12,7 +12,7 @@ from piccolo.conf.apps import AppConfig from piccolo.utils.sync import run_sync from tests.base import postgres_only -from tests.example_app.tables import Manager +from tests.example_apps.music.tables import Manager class TestNewMigrationCommand(TestCase): diff --git a/tests/apps/schema/commands/test_generate.py b/tests/apps/schema/commands/test_generate.py index 781660ad4..afdfb5256 100644 --- a/tests/apps/schema/commands/test_generate.py +++ b/tests/apps/schema/commands/test_generate.py @@ -10,60 +10,11 @@ generate, get_output_schema, ) -from piccolo.columns.column_types import ( - JSON, - JSONB, - UUID, - BigInt, - Boolean, - Bytea, - Date, - ForeignKey, - Integer, - Interval, - Numeric, - Real, - SmallInt, - Text, - Timestamp, - Timestamptz, - Varchar, -) +from piccolo.columns.column_types import Varchar from piccolo.table import Table from piccolo.utils.sync import run_sync from tests.base import postgres_only - - -class SmallTable(Table): - varchar_col = Varchar() - - -class MegaTable(Table): - """ - A table containing all of the column types, and different column kwargs. - """ - - bigint_col = BigInt() - boolean_col = Boolean() - bytea_col = Bytea() - date_col = Date() - foreignkey_col = ForeignKey(SmallTable) - integer_col = Integer() - interval_col = Interval() - json_col = JSON() - jsonb_col = JSONB() - numeric_col = Numeric() - real_col = Real() - smallint_col = SmallInt() - text_col = Text() - timestamp_col = Timestamp() - timestamptz_col = Timestamptz() - uuid_col = UUID() - varchar_col = Varchar() - - unique_col = Varchar(unique=True) - null_col = Varchar(null=True) - not_null_col = Varchar(null=False) +from tests.example_apps.mega.tables import MegaTable, SmallTable @postgres_only diff --git a/tests/apps/shell/commands/test_run.py b/tests/apps/shell/commands/test_run.py index f2ec08a68..8a9dffb0f 100644 --- a/tests/apps/shell/commands/test_run.py +++ b/tests/apps/shell/commands/test_run.py @@ -26,6 +26,9 @@ def test_run(self, print_: MagicMock, start_ipython_shell: MagicMock): call("- Shirt"), call("- Ticket"), call("- Venue"), + call("Importing mega tables:"), + call("- MegaTable"), + call("- SmallTable"), call("-------"), ], ) diff --git a/tests/columns/test_choices.py b/tests/columns/test_choices.py index 4be9f9921..5940ce65a 100644 --- a/tests/columns/test_choices.py +++ b/tests/columns/test_choices.py @@ -1,5 +1,5 @@ from tests.base import DBTestCase -from tests.example_app.tables import Shirt +from tests.example_apps.music.tables import Shirt class TestChoices(DBTestCase): diff --git a/tests/columns/test_combination.py b/tests/columns/test_combination.py index c580e598e..4f756192e 100644 --- a/tests/columns/test_combination.py +++ b/tests/columns/test_combination.py @@ -1,6 +1,6 @@ import unittest -from ..example_app.tables import Band +from tests.example_apps.music.tables import Band class TestWhere(unittest.TestCase): diff --git a/tests/columns/test_foreignkey.py b/tests/columns/test_foreignkey.py index ac26c24b4..c1708becc 100644 --- a/tests/columns/test_foreignkey.py +++ b/tests/columns/test_foreignkey.py @@ -3,7 +3,7 @@ from piccolo.columns import Column, ForeignKey, LazyTableReference, Varchar from piccolo.table import Table -from tests.example_app.tables import Band, Concert, Manager, Ticket +from tests.example_apps.music.tables import Band, Concert, Manager, Ticket class Manager1(Table, tablename="manager"): diff --git a/tests/columns/test_reference.py b/tests/columns/test_reference.py index 391592a84..b9ffe85a7 100644 --- a/tests/columns/test_reference.py +++ b/tests/columns/test_reference.py @@ -20,7 +20,7 @@ def test_init(self): LazyTableReference( table_class_name="Manager", app_name="example_app", - module_path="tests.example_app.tables", + module_path="tests.example_apps.music.tables", ) # Shouldn't raise exceptions: @@ -30,7 +30,7 @@ def test_init(self): ) LazyTableReference( table_class_name="Manager", - module_path="tests.example_app.tables", + module_path="tests.example_apps.music.tables", ) def test_str(self): @@ -45,7 +45,7 @@ def test_str(self): self.assertEqual( LazyTableReference( table_class_name="Manager", - module_path="tests.example_app.tables", + module_path="tests.example_apps.music.tables", ).__str__(), - "Module tests.example_app.tables.Manager", + "Module tests.example_apps.music.tables.Manager", ) diff --git a/tests/conf/test_apps.py b/tests/conf/test_apps.py index 737d8b6ca..f613a453c 100644 --- a/tests/conf/test_apps.py +++ b/tests/conf/test_apps.py @@ -2,8 +2,7 @@ from piccolo.apps.user.tables import BaseUser from piccolo.conf.apps import AppConfig, AppRegistry, table_finder - -from ..example_app.tables import Manager +from tests.example_apps.music.tables import Manager class TestAppRegistry(TestCase): @@ -58,7 +57,7 @@ def test_table_finder(self): """ Should return all Table subclasses. """ - tables = table_finder(modules=["tests.example_app.tables"]) + tables = table_finder(modules=["tests.example_apps.music.tables"]) table_class_names = [i.__name__ for i in tables] table_class_names.sort() @@ -84,7 +83,7 @@ def test_table_finder_coercion(self): """ Should convert a string argument to a list. """ - tables = table_finder(modules="tests.example_app.tables") + tables = table_finder(modules="tests.example_apps.music.tables") table_class_names = [i.__name__ for i in tables] table_class_names.sort() @@ -108,7 +107,8 @@ def test_include_tags(self): Should return all Table subclasses with a matching tag. """ tables = table_finder( - modules=["tests.example_app.tables"], include_tags=["special"] + modules=["tests.example_apps.music.tables"], + include_tags=["special"], ) table_class_names = [i.__name__ for i in tables] @@ -124,7 +124,8 @@ def test_exclude_tags(self): Should return all Table subclasses without the specified tags. """ tables = table_finder( - modules=["tests.example_app.tables"], exclude_tags=["special"] + modules=["tests.example_apps.music.tables"], + exclude_tags=["special"], ) table_class_names = [i.__name__ for i in tables] diff --git a/tests/engine/test_nested_transaction.py b/tests/engine/test_nested_transaction.py index 3e0443658..999870ed7 100644 --- a/tests/engine/test_nested_transaction.py +++ b/tests/engine/test_nested_transaction.py @@ -5,9 +5,9 @@ from piccolo.engine.exceptions import TransactionError from piccolo.engine.sqlite import SQLiteEngine from piccolo.table import Table +from tests.example_apps.music.tables import Manager from ..base import DBTestCase, sqlite_only -from ..example_app.tables import Manager ENGINE_1 = SQLiteEngine(path="engine1.sqlite") ENGINE_2 = SQLiteEngine(path="engine2.sqlite") diff --git a/tests/engine/test_pool.py b/tests/engine/test_pool.py index e39c66379..b91ade933 100644 --- a/tests/engine/test_pool.py +++ b/tests/engine/test_pool.py @@ -1,9 +1,9 @@ import asyncio from piccolo.engine.postgres import PostgresEngine +from tests.example_apps.music.tables import Manager from ..base import DBTestCase, postgres_only -from ..example_app.tables import Manager @postgres_only diff --git a/tests/engine/test_transaction.py b/tests/engine/test_transaction.py index 0f248a943..a2cd38a43 100644 --- a/tests/engine/test_transaction.py +++ b/tests/engine/test_transaction.py @@ -1,8 +1,9 @@ import asyncio from unittest import TestCase +from tests.example_apps.music.tables import Band, Manager + from ..base import postgres_only -from ..example_app.tables import Band, Manager class TestAtomic(TestCase): diff --git a/tests/example_apps/__init__.py b/tests/example_apps/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/example_apps/mega/__init__.py b/tests/example_apps/mega/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/example_apps/mega/piccolo_app.py b/tests/example_apps/mega/piccolo_app.py new file mode 100644 index 000000000..f565bb5aa --- /dev/null +++ b/tests/example_apps/mega/piccolo_app.py @@ -0,0 +1,17 @@ +import os + +from piccolo.conf.apps import AppConfig + +from .tables import MegaTable, SmallTable + +CURRENT_DIRECTORY = os.path.dirname(os.path.abspath(__file__)) + + +APP_CONFIG = AppConfig( + app_name="mega", + table_classes=[MegaTable, SmallTable], + migrations_folder_path=os.path.join( + CURRENT_DIRECTORY, "piccolo_migrations" + ), + commands=[], +) diff --git a/tests/example_apps/mega/piccolo_migrations/__init__.py b/tests/example_apps/mega/piccolo_migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/example_apps/mega/tables.py b/tests/example_apps/mega/tables.py new file mode 100644 index 000000000..6ecb17d94 --- /dev/null +++ b/tests/example_apps/mega/tables.py @@ -0,0 +1,56 @@ +""" +This is a useful table when we want to test all possible column types. +""" + +from piccolo.columns.column_types import ( + JSON, + JSONB, + UUID, + BigInt, + Boolean, + Bytea, + Date, + ForeignKey, + Integer, + Interval, + Numeric, + Real, + SmallInt, + Text, + Timestamp, + Timestamptz, + Varchar, +) +from piccolo.table import Table + + +class SmallTable(Table): + varchar_col = Varchar() + + +class MegaTable(Table): + """ + A table containing all of the column types, and different column kwargs. + """ + + bigint_col = BigInt() + boolean_col = Boolean() + bytea_col = Bytea() + date_col = Date() + foreignkey_col = ForeignKey(SmallTable) + integer_col = Integer() + interval_col = Interval() + json_col = JSON() + jsonb_col = JSONB() + numeric_col = Numeric(digits=(5, 2)) + real_col = Real() + smallint_col = SmallInt() + text_col = Text() + timestamp_col = Timestamp() + timestamptz_col = Timestamptz() + uuid_col = UUID() + varchar_col = Varchar() + + unique_col = Varchar(unique=True) + null_col = Varchar(null=True) + not_null_col = Varchar(null=False) diff --git a/tests/example_apps/music/__init__.py b/tests/example_apps/music/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/example_app/piccolo_app.py b/tests/example_apps/music/piccolo_app.py similarity index 100% rename from tests/example_app/piccolo_app.py rename to tests/example_apps/music/piccolo_app.py diff --git a/tests/example_app/piccolo_migrations/2020-12-17T18-44-30.py b/tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-30.py similarity index 100% rename from tests/example_app/piccolo_migrations/2020-12-17T18-44-30.py rename to tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-30.py diff --git a/tests/example_app/piccolo_migrations/2020-12-17T18-44-39.py b/tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-39.py similarity index 100% rename from tests/example_app/piccolo_migrations/2020-12-17T18-44-39.py rename to tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-39.py diff --git a/tests/example_app/piccolo_migrations/2020-12-17T18-44-44.py b/tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-44.py similarity index 100% rename from tests/example_app/piccolo_migrations/2020-12-17T18-44-44.py rename to tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-44.py diff --git a/tests/example_app/piccolo_migrations/2021-07-25T22-38-48-009306.py b/tests/example_apps/music/piccolo_migrations/2021-07-25T22-38-48-009306.py similarity index 100% rename from tests/example_app/piccolo_migrations/2021-07-25T22-38-48-009306.py rename to tests/example_apps/music/piccolo_migrations/2021-07-25T22-38-48-009306.py diff --git a/tests/example_app/piccolo_migrations/2021-09-06T13-58-23-024723.py b/tests/example_apps/music/piccolo_migrations/2021-09-06T13-58-23-024723.py similarity index 100% rename from tests/example_app/piccolo_migrations/2021-09-06T13-58-23-024723.py rename to tests/example_apps/music/piccolo_migrations/2021-09-06T13-58-23-024723.py diff --git a/tests/example_app/tables.py b/tests/example_apps/music/tables.py similarity index 100% rename from tests/example_app/tables.py rename to tests/example_apps/music/tables.py diff --git a/tests/postgres_conf.py b/tests/postgres_conf.py index 5a795c4fd..af21dcbc5 100644 --- a/tests/postgres_conf.py +++ b/tests/postgres_conf.py @@ -14,4 +14,9 @@ ) -APP_REGISTRY = AppRegistry(apps=["tests.example_app.piccolo_app"]) +APP_REGISTRY = AppRegistry( + apps=[ + "tests.example_apps.music.piccolo_app", + "tests.example_apps.mega.piccolo_app", + ] +) diff --git a/tests/query/test_await.py b/tests/query/test_await.py index 867ba7b98..fe011b52f 100644 --- a/tests/query/test_await.py +++ b/tests/query/test_await.py @@ -1,7 +1,8 @@ import asyncio +from tests.example_apps.music.tables import Band + from ..base import DBTestCase -from ..example_app.tables import Band class TestAwait(DBTestCase): diff --git a/tests/query/test_freeze.py b/tests/query/test_freeze.py index 7c5d3e744..255d27a9f 100644 --- a/tests/query/test_freeze.py +++ b/tests/query/test_freeze.py @@ -4,7 +4,7 @@ from piccolo.query.base import Query from tests.base import DBTestCase, sqlite_only -from tests.example_app.tables import Band +from tests.example_apps.music.tables import Band @dataclass diff --git a/tests/query/test_mixins.py b/tests/query/test_mixins.py index f03dbefa4..bd358fadf 100644 --- a/tests/query/test_mixins.py +++ b/tests/query/test_mixins.py @@ -1,7 +1,7 @@ from unittest import TestCase from piccolo.query.mixins import ColumnsDelegate -from tests.example_app.tables import Band +from tests.example_apps.music.tables import Band class TestColumnsDelegate(TestCase): diff --git a/tests/query/test_slots.py b/tests/query/test_slots.py index 7777f4158..971019910 100644 --- a/tests/query/test_slots.py +++ b/tests/query/test_slots.py @@ -13,8 +13,7 @@ TableExists, Update, ) - -from ..example_app.tables import Manager +from tests.example_apps.music.tables import Manager class TestSlots(TestCase): diff --git a/tests/sqlite_conf.py b/tests/sqlite_conf.py index 6564b4b94..afcc6185e 100644 --- a/tests/sqlite_conf.py +++ b/tests/sqlite_conf.py @@ -4,4 +4,9 @@ DB = SQLiteEngine(path="test.sqlite") -APP_REGISTRY = AppRegistry(apps=["tests.example_app.piccolo_app"]) +APP_REGISTRY = AppRegistry( + apps=[ + "tests.example_apps.music.piccolo_app", + "tests.example_apps.mega.piccolo_app", + ] +) diff --git a/tests/table/instance/test_get_related.py b/tests/table/instance/test_get_related.py index 6e5025602..e5a03c643 100644 --- a/tests/table/instance/test_get_related.py +++ b/tests/table/instance/test_get_related.py @@ -1,6 +1,6 @@ from unittest import TestCase -from tests.example_app.tables import Band, Manager +from tests.example_apps.music.tables import Band, Manager TABLES = [Manager, Band] diff --git a/tests/table/instance/test_get_related_readable.py b/tests/table/instance/test_get_related_readable.py index 649b10099..4c0467be7 100644 --- a/tests/table/instance/test_get_related_readable.py +++ b/tests/table/instance/test_get_related_readable.py @@ -1,5 +1,5 @@ from tests.base import DBTestCase -from tests.example_app.tables import Band +from tests.example_apps.music.tables import Band class TestGetRelatedReadable(DBTestCase): diff --git a/tests/table/instance/test_instantiate.py b/tests/table/instance/test_instantiate.py index 154b18df1..43b84f49d 100644 --- a/tests/table/instance/test_instantiate.py +++ b/tests/table/instance/test_instantiate.py @@ -1,5 +1,5 @@ from tests.base import DBTestCase, postgres_only, sqlite_only -from tests.example_app.tables import Band +from tests.example_apps.music.tables import Band class TestInstance(DBTestCase): diff --git a/tests/table/instance/test_remove.py b/tests/table/instance/test_remove.py index 18045f2ab..be9f7b511 100644 --- a/tests/table/instance/test_remove.py +++ b/tests/table/instance/test_remove.py @@ -1,6 +1,6 @@ from unittest import TestCase -from tests.example_app.tables import Manager +from tests.example_apps.music.tables import Manager class TestRemove(TestCase): diff --git a/tests/table/instance/test_save.py b/tests/table/instance/test_save.py index f5c17b73f..ba6bee5d4 100644 --- a/tests/table/instance/test_save.py +++ b/tests/table/instance/test_save.py @@ -1,6 +1,6 @@ from unittest import TestCase -from tests.example_app.tables import Manager +from tests.example_apps.music.tables import Manager class TestSave(TestCase): diff --git a/tests/table/instance/test_to_dict.py b/tests/table/instance/test_to_dict.py index cbafa8812..4142974b7 100644 --- a/tests/table/instance/test_to_dict.py +++ b/tests/table/instance/test_to_dict.py @@ -1,5 +1,5 @@ from tests.base import DBTestCase -from tests.example_app.tables import Band, Manager +from tests.example_apps.music.tables import Band, Manager class TestToDict(DBTestCase): diff --git a/tests/table/test_all_columns.py b/tests/table/test_all_columns.py index 1ff7d1283..cdbc0d98a 100644 --- a/tests/table/test_all_columns.py +++ b/tests/table/test_all_columns.py @@ -1,6 +1,6 @@ from unittest import TestCase -from tests.example_app.tables import Band +from tests.example_apps.music.tables import Band class TestAllColumns(TestCase): diff --git a/tests/table/test_alter.py b/tests/table/test_alter.py index a8cfa0387..684340f58 100644 --- a/tests/table/test_alter.py +++ b/tests/table/test_alter.py @@ -2,9 +2,9 @@ from piccolo.columns import BigInt, Integer, Numeric, Varchar from piccolo.table import Table +from tests.example_apps.music.tables import Band, Manager from ..base import DBTestCase, postgres_only -from ..example_app.tables import Band, Manager class TestRenameColumn(DBTestCase): diff --git a/tests/table/test_batch.py b/tests/table/test_batch.py index a67c131d1..b06238880 100644 --- a/tests/table/test_batch.py +++ b/tests/table/test_batch.py @@ -1,8 +1,9 @@ import asyncio import math +from tests.example_apps.music.tables import Manager + from ..base import DBTestCase -from ..example_app.tables import Manager class TestBatchSelect(DBTestCase): diff --git a/tests/table/test_count.py b/tests/table/test_count.py index 037fab5bc..45f33f2f5 100644 --- a/tests/table/test_count.py +++ b/tests/table/test_count.py @@ -1,5 +1,6 @@ +from tests.example_apps.music.tables import Band + from ..base import DBTestCase -from ..example_app.tables import Band class TestCount(DBTestCase): diff --git a/tests/table/test_create.py b/tests/table/test_create.py index 5ef01335e..521c4f7d6 100644 --- a/tests/table/test_create.py +++ b/tests/table/test_create.py @@ -2,8 +2,7 @@ from piccolo.columns import Varchar from piccolo.table import Table - -from ..example_app.tables import Manager +from tests.example_apps.music.tables import Manager class TestCreate(TestCase): diff --git a/tests/table/test_delete.py b/tests/table/test_delete.py index 0fd8c24ba..15d3026b9 100644 --- a/tests/table/test_delete.py +++ b/tests/table/test_delete.py @@ -1,7 +1,7 @@ from piccolo.query.methods.delete import DeletionError +from tests.example_apps.music.tables import Band from ..base import DBTestCase -from ..example_app.tables import Band class TestDelete(DBTestCase): diff --git a/tests/table/test_exists.py b/tests/table/test_exists.py index b261cdb0e..59f7c1382 100644 --- a/tests/table/test_exists.py +++ b/tests/table/test_exists.py @@ -1,5 +1,6 @@ +from tests.example_apps.music.tables import Band + from ..base import DBTestCase -from ..example_app.tables import Band class TestExists(DBTestCase): diff --git a/tests/table/test_indexes.py b/tests/table/test_indexes.py index a4e40f652..ba6cb9aa1 100644 --- a/tests/table/test_indexes.py +++ b/tests/table/test_indexes.py @@ -1,7 +1,8 @@ from unittest import TestCase +from tests.example_apps.music.tables import Manager + from ..base import DBTestCase -from ..example_app.tables import Manager class TestIndexes(DBTestCase): diff --git a/tests/table/test_insert.py b/tests/table/test_insert.py index 45eb77bd3..66e90350d 100644 --- a/tests/table/test_insert.py +++ b/tests/table/test_insert.py @@ -1,5 +1,6 @@ +from tests.example_apps.music.tables import Band, Manager + from ..base import DBTestCase -from ..example_app.tables import Band, Manager class TestInsert(DBTestCase): diff --git a/tests/table/test_join.py b/tests/table/test_join.py index 2468b0c57..d89dc2402 100644 --- a/tests/table/test_join.py +++ b/tests/table/test_join.py @@ -1,7 +1,13 @@ import decimal from unittest import TestCase -from tests.example_app.tables import Band, Concert, Manager, Ticket, Venue +from tests.example_apps.music.tables import ( + Band, + Concert, + Manager, + Ticket, + Venue, +) TABLES = [Manager, Band, Venue, Concert] diff --git a/tests/table/test_metaclass.py b/tests/table/test_metaclass.py index f6a274eac..0afff0290 100644 --- a/tests/table/test_metaclass.py +++ b/tests/table/test_metaclass.py @@ -3,7 +3,7 @@ from piccolo.columns import Secret from piccolo.columns.column_types import JSON, JSONB, ForeignKey from piccolo.table import Table -from tests.example_app.tables import Band +from tests.example_apps.music.tables import Band class TestMetaClass(TestCase): diff --git a/tests/table/test_objects.py b/tests/table/test_objects.py index 2f602620a..0a0a6fbaa 100644 --- a/tests/table/test_objects.py +++ b/tests/table/test_objects.py @@ -1,5 +1,5 @@ from tests.base import DBTestCase, postgres_only, sqlite_only -from tests.example_app.tables import Band, Manager +from tests.example_apps.music.tables import Band, Manager class TestObjects(DBTestCase): diff --git a/tests/table/test_output.py b/tests/table/test_output.py index d19361d3d..e4d21ff12 100644 --- a/tests/table/test_output.py +++ b/tests/table/test_output.py @@ -2,7 +2,7 @@ from unittest import TestCase from tests.base import DBTestCase -from tests.example_app.tables import Band, RecordingStudio +from tests.example_apps.music.tables import Band, RecordingStudio class TestOutputList(DBTestCase): diff --git a/tests/table/test_raw.py b/tests/table/test_raw.py index dc6ef530b..9da3b919c 100644 --- a/tests/table/test_raw.py +++ b/tests/table/test_raw.py @@ -1,5 +1,6 @@ +from tests.example_apps.music.tables import Band + from ..base import DBTestCase -from ..example_app.tables import Band class TestRaw(DBTestCase): diff --git a/tests/table/test_ref.py b/tests/table/test_ref.py index 875fa74a2..fb973f086 100644 --- a/tests/table/test_ref.py +++ b/tests/table/test_ref.py @@ -1,7 +1,7 @@ from unittest import TestCase from piccolo.columns.column_types import Varchar -from tests.example_app.tables import Band +from tests.example_apps.music.tables import Band class TestRef(TestCase): diff --git a/tests/table/test_repr.py b/tests/table/test_repr.py index 055637d31..240bfc53d 100644 --- a/tests/table/test_repr.py +++ b/tests/table/test_repr.py @@ -1,5 +1,6 @@ +from tests.example_apps.music.tables import Manager + from ..base import DBTestCase, postgres_only, sqlite_only -from ..example_app.tables import Manager class TestTableRepr(DBTestCase): diff --git a/tests/table/test_select.py b/tests/table/test_select.py index 958997f50..f2d9f0ace 100644 --- a/tests/table/test_select.py +++ b/tests/table/test_select.py @@ -3,9 +3,9 @@ from piccolo.apps.user.tables import BaseUser from piccolo.columns.combination import WhereRaw from piccolo.query.methods.select import Avg, Count, Max, Min, Sum +from tests.example_apps.music.tables import Band, Concert, Manager from ..base import DBTestCase, postgres_only, sqlite_only -from ..example_app.tables import Band, Concert, Manager class TestSelect(DBTestCase): diff --git a/tests/table/test_str.py b/tests/table/test_str.py index c53289c58..82325e098 100644 --- a/tests/table/test_str.py +++ b/tests/table/test_str.py @@ -1,6 +1,6 @@ from unittest import TestCase -from ..example_app.tables import Manager +from tests.example_apps.music.tables import Manager class TestTableStr(TestCase): diff --git a/tests/table/test_table_exists.py b/tests/table/test_table_exists.py index 1ee789e19..e35af2288 100644 --- a/tests/table/test_table_exists.py +++ b/tests/table/test_table_exists.py @@ -1,6 +1,6 @@ from unittest import TestCase -from ..example_app.tables import Manager +from tests.example_apps.music.tables import Manager class TestTableExists(TestCase): diff --git a/tests/table/test_update.py b/tests/table/test_update.py index ba5b33a56..d3d27715a 100644 --- a/tests/table/test_update.py +++ b/tests/table/test_update.py @@ -1,5 +1,6 @@ +from tests.example_apps.music.tables import Band, Poster + from ..base import DBTestCase -from ..example_app.tables import Band, Poster class TestUpdate(DBTestCase): diff --git a/tests/testing/test_model_builder.py b/tests/testing/test_model_builder.py index bb13b534f..16ce96aec 100644 --- a/tests/testing/test_model_builder.py +++ b/tests/testing/test_model_builder.py @@ -2,8 +2,7 @@ import unittest from piccolo.testing.model_builder import ModelBuilder - -from ..example_app.tables import ( +from tests.example_apps.music.tables import ( Band, Concert, Manager, diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py new file mode 100644 index 000000000..8c3f626be --- /dev/null +++ b/tests/utils/test_pydantic.py @@ -0,0 +1,331 @@ +import decimal +from unittest import TestCase + +import pydantic +from pydantic import ValidationError + +from piccolo.columns import JSON, JSONB, Array, Numeric, Secret, Text, Varchar +from piccolo.columns.column_types import ForeignKey +from piccolo.table import Table +from piccolo.utils.pydantic import create_pydantic_model + + +class TestVarcharColumn(TestCase): + def test_varchar_length(self): + class Director(Table): + name = Varchar(length=10) + + pydantic_model = create_pydantic_model(table=Director) + + with self.assertRaises(ValidationError): + pydantic_model(name="This is a really long name") + + pydantic_model(name="short name") + + +class TestNumericColumn(TestCase): + """ + Numeric and Decimal are the same - so we'll just Numeric. + """ + + def test_numeric_digits(self): + class Movie(Table): + box_office = Numeric(digits=(5, 1)) + + pydantic_model = create_pydantic_model(table=Movie) + + with self.assertRaises(ValidationError): + # This should fail as there are too much numbers after the decimal + # point + pydantic_model(box_office=decimal.Decimal("1.11")) + + with self.assertRaises(ValidationError): + # This should fail as there are too much numbers in total + pydantic_model(box_office=decimal.Decimal("11111.1")) + + pydantic_model(box_office=decimal.Decimal("1.0")) + + +class TestSecretColumn(TestCase): + def test_secret_param(self): + class TopSecret(Table): + confidential = Secret() + + pydantic_model = create_pydantic_model(table=TopSecret) + self.assertEqual( + pydantic_model.schema()["properties"]["confidential"]["extra"][ + "secret" + ], + True, + ) + + +class TestArrayColumn(TestCase): + def test_array_param(self): + class Band(Table): + members = Array(base_column=Varchar(length=16)) + + pydantic_model = create_pydantic_model(table=Band) + + self.assertEqual( + pydantic_model.schema()["properties"]["members"]["items"]["type"], + "string", + ) + + +class TestTextColumn(TestCase): + def test_text_format(self): + class Band(Table): + bio = Text() + + pydantic_model = create_pydantic_model(table=Band) + + self.assertEqual( + pydantic_model.schema()["properties"]["bio"]["format"], + "text-area", + ) + + +class TestColumnHelpText(TestCase): + """ + Make sure that columns with `help_text` attribute defined have the + relevant text appear in the schema. + """ + + def test_help_text_present(self): + + help_text = "In millions of US dollars." + + class Movie(Table): + box_office = Numeric(digits=(5, 1), help_text=help_text) + + pydantic_model = create_pydantic_model(table=Movie) + self.assertEqual( + pydantic_model.schema()["properties"]["box_office"]["extra"][ + "help_text" + ], + help_text, + ) + + +class TestTableHelpText(TestCase): + """ + Make sure that tables with `help_text` attribute defined have the + relevant text appear in the schema. + """ + + def test_help_text_present(self): + + help_text = "Movies which were released in cinemas." + + class Movie(Table, help_text=help_text): + name = Varchar() + + pydantic_model = create_pydantic_model(table=Movie) + self.assertEqual( + pydantic_model.schema()["help_text"], + help_text, + ) + + +class TestJSONColumn(TestCase): + def test_default(self): + class Movie(Table): + meta = JSON() + meta_b = JSONB() + + pydantic_model = create_pydantic_model(table=Movie) + + json_string = '{"code": 12345}' + + model_instance = pydantic_model(meta=json_string, meta_b=json_string) + self.assertEqual(model_instance.meta, json_string) + self.assertEqual(model_instance.meta_b, json_string) + + def test_deserialize_json(self): + class Movie(Table): + meta = JSON() + meta_b = JSONB() + + pydantic_model = create_pydantic_model( + table=Movie, deserialize_json=True + ) + + json_string = '{"code": 12345}' + output = {"code": 12345} + + model_instance = pydantic_model(meta=json_string, meta_b=json_string) + self.assertEqual(model_instance.meta, output) + self.assertEqual(model_instance.meta_b, output) + + def test_validation(self): + class Movie(Table): + meta = JSON() + meta_b = JSONB() + + for deserialize_json in (True, False): + pydantic_model = create_pydantic_model( + table=Movie, deserialize_json=deserialize_json + ) + + json_string = "error" + + with self.assertRaises(pydantic.ValidationError): + pydantic_model(meta=json_string, meta_b=json_string) + + +class TestExcludeColumn(TestCase): + def test_all(self): + class Computer(Table): + CPU = Varchar() + GPU = Varchar() + + pydantic_model = create_pydantic_model(Computer, exclude_columns=()) + + properties = pydantic_model.schema()["properties"] + self.assertIsInstance(properties["GPU"], dict) + self.assertIsInstance(properties["CPU"], dict) + + def test_exclude(self): + class Computer(Table): + CPU = Varchar() + GPU = Varchar() + + pydantic_model = create_pydantic_model( + Computer, + exclude_columns=(Computer.CPU,), + ) + + properties = pydantic_model.schema()["properties"] + self.assertIsInstance(properties.get("GPU"), dict) + self.assertIsNone(properties.get("CPU")) + + def test_exclude_all_manually(self): + class Computer(Table): + GPU = Varchar() + CPU = Varchar() + + pydantic_model = create_pydantic_model( + Computer, + exclude_columns=(Computer.GPU, Computer.CPU), + ) + + self.assertEqual(pydantic_model.schema()["properties"], {}) + + def test_exclude_all_meta(self): + class Computer(Table): + GPU = Varchar() + CPU = Varchar() + + pydantic_model = create_pydantic_model( + Computer, + exclude_columns=tuple(Computer._meta.columns), + ) + + self.assertEqual(pydantic_model.schema()["properties"], {}) + + def test_invalid_column_str(self): + class Computer(Table): + CPU = Varchar() + GPU = Varchar() + + with self.assertRaises(ValueError): + create_pydantic_model( + Computer, + exclude_columns=("CPU",), + ) + + def test_invalid_column_different_table(self): + class Computer(Table): + CPU = Varchar() + GPU = Varchar() + + class Computer2(Table): + SSD = Varchar() + + with self.assertRaises(ValueError): + create_pydantic_model(Computer, exclude_columns=(Computer2.SSD,)) + + def test_invalid_column_different_table_same_type(self): + class Computer(Table): + CPU = Varchar() + GPU = Varchar() + + class Computer2(Table): + CPU = Varchar() + + with self.assertRaises(ValueError): + create_pydantic_model(Computer, exclude_columns=(Computer2.CPU,)) + + +class TestNestedModel(TestCase): + def test_nested_models(self): + class Country(Table): + name = Varchar(length=10) + + class Director(Table): + name = Varchar(length=10) + country = ForeignKey(Country) + + class Movie(Table): + name = Varchar(length=10) + director = ForeignKey(Director) + + MovieModel = create_pydantic_model(table=Movie, nested=True) + + ####################################################################### + + DirectorModel = MovieModel.__fields__["director"].type_ + + self.assertTrue(issubclass(DirectorModel, pydantic.BaseModel)) + + director_model_keys = [i for i in DirectorModel.__fields__.keys()] + self.assertEqual(director_model_keys, ["name", "country"]) + + ####################################################################### + + CountryModel = DirectorModel.__fields__["country"].type_ + + self.assertTrue(issubclass(CountryModel, pydantic.BaseModel)) + + country_model_keys = [i for i in CountryModel.__fields__.keys()] + self.assertEqual(country_model_keys, ["name"]) + + def test_cascaded_args(self): + """ + Make sure that arguments passed to ``create_pydantic_model`` are + cascaded to nested models. + """ + + class Country(Table): + name = Varchar(length=10) + + class Director(Table): + name = Varchar(length=10) + country = ForeignKey(Country) + + class Movie(Table): + name = Varchar(length=10) + director = ForeignKey(Director) + + MovieModel = create_pydantic_model( + table=Movie, nested=True, include_default_columns=True + ) + + ####################################################################### + + DirectorModel = MovieModel.__fields__["director"].type_ + + self.assertTrue(issubclass(DirectorModel, pydantic.BaseModel)) + + director_model_keys = [i for i in DirectorModel.__fields__.keys()] + self.assertEqual(director_model_keys, ["id", "name", "country"]) + + ####################################################################### + + CountryModel = DirectorModel.__fields__["country"].type_ + + self.assertTrue(issubclass(CountryModel, pydantic.BaseModel)) + + country_model_keys = [i for i in CountryModel.__fields__.keys()] + self.assertEqual(country_model_keys, ["id", "name"]) From 77b3d2792e62f7a9eaa383125f96b1152461c3d9 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 14 Sep 2021 09:19:48 +0100 Subject: [PATCH 061/727] bumped version --- CHANGES | 24 ++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 7d903dcf6..ace6157c9 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,30 @@ Changes ======= +0.46.0 +------ +Added the fixture app. This is used to dump data from a database to a JSON +file, and then reload it again. It's useful for seeding a database with +essential data, whether that's a colleague setting up their local environment, +or deploying to production. + +To create a fixture: + +.. code-block:: bash + + piccolo fixture dump --apps=blog > fixture.json + +To load a fixture: + +.. code-block:: bash + + piccolo fixture load fixture.json + +As part of this change, Piccolo's Pydantic support was brought into this +library (prior to this it only existed within the ``piccolo_api`` library). At +a later date, the ``piccolo_api`` library will be updated, so it's Pydantic +code just proxies to what's within the main ``piccolo`` library. + 0.45.1 ------ Improvements to ``piccolo schema generate``. It's now smarter about which diff --git a/piccolo/__init__.py b/piccolo/__init__.py index bb956ef59..21d535065 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.45.1" +__VERSION__ = "0.46.0" From 36784b60c9e5de43dc4508fde2c9650ba0d7039f Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 14 Sep 2021 10:14:02 +0100 Subject: [PATCH 062/727] added `pip install piccolo[all]` option (#230) * added `pip install piccolo[all]` option * tweak install docs * update README to mention piccolo[all] option --- README.md | 16 ++++++++++++---- .../getting_started/installing_piccolo.rst | 10 +++++++--- setup.py | 13 +++++++++---- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 3eb6732b4..a0114b39d 100644 --- a/README.md +++ b/README.md @@ -72,21 +72,27 @@ await b.remove().run() Installing with PostgreSQL driver: -``` +```bash pip install 'piccolo[postgres]' ``` Installing with SQLite driver: -``` +```bash pip install 'piccolo[sqlite]' ``` +Installing with all optional dependencies (easiest): + +```bash +pip install 'piccolo[all]' +``` + ## Building a web app? Let Piccolo scaffold you an ASGI web app, using Piccolo as the ORM: -``` +```bash piccolo asgi new ``` @@ -98,4 +104,6 @@ We have a handy page which shows the equivalent of [common Django queries in Pic ## Documentation -See [Read the docs](https://piccolo-orm.readthedocs.io/en/latest/piccolo/getting_started/index.html). +Our documentation is on [Read the docs](https://piccolo-orm.readthedocs.io/en/latest/piccolo/getting_started/index.html). + +We also have some great [tutorial videos on YouTube](https://www.youtube.com/channel/UCE7x5nm1Iy9KDfXPNrNQ5lA). diff --git a/docs/src/piccolo/getting_started/installing_piccolo.rst b/docs/src/piccolo/getting_started/installing_piccolo.rst index 0354a217b..c6e2fce23 100644 --- a/docs/src/piccolo/getting_started/installing_piccolo.rst +++ b/docs/src/piccolo/getting_started/installing_piccolo.rst @@ -30,7 +30,11 @@ Now install piccolo, ideally inside a `virtualenv List[str]: +def parse_requirement(req_path: str) -> t.List[str]: """ Parse requirement file. Example: @@ -31,7 +32,7 @@ def parse_requirement(req_path: str) -> List[str]: return [i.strip() for i in contents.strip().split("\n")] -def extras_require(): +def extras_require() -> t.Dict[str, t.List[str]]: """ Parse requirements in requirements/extras directory """ @@ -41,6 +42,10 @@ def extras_require(): os.path.join("extras", extra + ".txt") ) + extra_requirements["all"] = [ + i for i in itertools.chain.from_iterable(extra_requirements.values()) + ] + return extra_requirements From ccdabaa341c317180659b665cecb763aba3a8873 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 14 Sep 2021 10:23:11 +0100 Subject: [PATCH 063/727] bumped version --- CHANGES | 5 +++++ piccolo/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index ace6157c9..793216e72 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,11 @@ Changes ======= +0.47.0 +------ +You can now use ``pip install piccolo[all]``, which will install all optional +requirements. + 0.46.0 ------ Added the fixture app. This is used to dump data from a database to a JSON diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 21d535065..57e5dc78b 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.46.0" +__VERSION__ = "0.47.0" From 604e3f10d5b8433b142ea3b96e6f5fc50c9e5b62 Mon Sep 17 00:00:00 2001 From: ali sayyah Date: Wed, 15 Sep 2021 22:16:28 +0430 Subject: [PATCH 064/727] Create users with CLI arguments (#231) * added CLI arguments to 'user create' command * added unit test for creating users with cli arguments * fixed lint issues * update `piccolo user create` docs * ignore mypy error Co-authored-by: Daniel Townsend --- docs/src/piccolo/authentication/baseuser.rst | 13 +++++++- piccolo/apps/user/commands/create.py | 31 +++++++++++++------- tests/apps/user/commands/test_create.py | 23 +++++++++++++++ 3 files changed, 55 insertions(+), 12 deletions(-) diff --git a/docs/src/piccolo/authentication/baseuser.rst b/docs/src/piccolo/authentication/baseuser.rst index a3790382b..d6f07e145 100644 --- a/docs/src/piccolo/authentication/baseuser.rst +++ b/docs/src/piccolo/authentication/baseuser.rst @@ -26,12 +26,23 @@ The app comes with some useful commands. user create ~~~~~~~~~~~ -Create a new user. +Creates a new user. It presents an interactive prompt, asking for the username, +password etc. .. code-block:: bash piccolo user create +If you'd prefer to create a user without the interactive prompt (perhaps in a +script), you can pass all of the arguments in as follows: + +.. code-block:: bash + + piccolo user create --username=bob --password=bob123 --email=foo@bar.com --is_admin=t --is_superuser=t --is_active=t + +If you choose this approach then be careful, as the password will be in the +shell's history. + user change_password ~~~~~~~~~~~~~~~~~~~~ diff --git a/piccolo/apps/user/commands/create.py b/piccolo/apps/user/commands/create.py index 11f1e7bc7..d1fb5ed93 100644 --- a/piccolo/apps/user/commands/create.py +++ b/piccolo/apps/user/commands/create.py @@ -1,4 +1,5 @@ import sys +import typing as t from getpass import getpass, getuser from piccolo.apps.user.tables import BaseUser @@ -55,24 +56,32 @@ def get_is_active() -> bool: return active == "y" -def create(): +def create( + username: t.Optional[str] = None, + email: t.Optional[str] = None, + password: t.Optional[str] = None, + is_admin: t.Optional[bool] = None, + is_superuser: t.Optional[bool] = None, + is_active: t.Optional[bool] = None, +): """ Create a new user. """ - username = get_username() - email = get_email() - password = get_password() - confirmed_password = get_confirmed_password() + username = get_username() if username is None else username + email = get_email() if email is None else email + if password is None: + password = get_password() + confirmed_password = get_confirmed_password() - if not password == confirmed_password: - sys.exit("Passwords don't match!") + if not password == confirmed_password: + sys.exit("Passwords don't match!") if len(password) < 4: sys.exit("The password is too short") - is_admin = get_is_admin() - is_superuser = get_is_superuser() - is_active = get_is_active() + is_admin = get_is_admin() if is_admin is None else is_admin + is_superuser = get_is_superuser() if is_superuser is None else is_superuser + is_active = get_is_active() if is_active is None else is_active user = BaseUser( username=username, @@ -84,4 +93,4 @@ def create(): ) user.save().run_sync() - print(f"Created User {user.id}") + print(f"Created User {user.id}") # type: ignore diff --git a/tests/apps/user/commands/test_create.py b/tests/apps/user/commands/test_create.py index 5f78dab89..6e2854821 100644 --- a/tests/apps/user/commands/test_create.py +++ b/tests/apps/user/commands/test_create.py @@ -54,3 +54,26 @@ def test_create(self, *args, **kwargs): ) .run_sync() ) + + def test_create_with_arguments(self, *args, **kwargs): + arguments = { + "username": "bob123", + "email": "bob@test.com", + "password": "password123", + "is_admin": True, + "is_superuser": True, + "is_active": True, + } + create(**arguments) + + self.assertTrue( + BaseUser.exists() + .where( + (BaseUser.admin == True) # noqa: E712 + & (BaseUser.username == "bob123") + & (BaseUser.email == "bob@test.com") + & (BaseUser.superuser == True) + & (BaseUser.active == True) + ) + .run_sync() + ) From 9cd631ba49e617f203114d50be0c9fdcca84c891 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 15 Sep 2021 18:56:15 +0100 Subject: [PATCH 065/727] bumped version --- CHANGES | 10 ++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 793216e72..cc19063b9 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,16 @@ Changes ======= +0.48.0 +------ +The ``piccolo user create`` command can now be used without using the +interactive prompt, by passing in command line arguments instead (courtesy +@AliSayyah). + +For example ``piccolo user create --username=bob ...``. + +This is useful when you want to create users in a script. + 0.47.0 ------ You can now use ``pip install piccolo[all]``, which will install all optional diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 57e5dc78b..851aa6ec5 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.47.0" +__VERSION__ = "0.48.0" From 23a8c1f8cedee518288ecb702454f62521583c13 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 15 Sep 2021 19:06:10 +0100 Subject: [PATCH 066/727] Update CHANGES --- CHANGES | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index cc19063b9..327ca0f55 100644 --- a/CHANGES +++ b/CHANGES @@ -3,9 +3,8 @@ Changes 0.48.0 ------ -The ``piccolo user create`` command can now be used without using the -interactive prompt, by passing in command line arguments instead (courtesy -@AliSayyah). +The ``piccolo user create`` command can now be used by passing in command line +arguments, instead of using the interactive prompt (courtesy @AliSayyah). For example ``piccolo user create --username=bob ...``. From 3af0ac49d948904ee04958416b2d19f6ab657308 Mon Sep 17 00:00:00 2001 From: ali sayyah Date: Thu, 16 Sep 2021 16:32:24 +0430 Subject: [PATCH 067/727] Fix numeric bug (#236) * fixed the bug discussed in #232 . if precision and scale are not set for the Numeric Field, 'None' value will be returned. * added a unit test for testing the bug-fix * fixed lint issues * add type annotations for precision and scale Co-authored-by: Daniel Townsend --- piccolo/columns/column_types.py | 8 ++++---- tests/utils/test_pydantic.py | 16 ++++++++++++++-- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 782eda04c..f89e2cfad 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -890,18 +890,18 @@ def column_type(self): return "NUMERIC" @property - def precision(self): + def precision(self) -> t.Optional[int]: """ The total number of digits allowed. """ - return self.digits[0] + return self.digits[0] if self.digits is not None else None @property - def scale(self): + def scale(self) -> t.Optional[int]: """ The number of digits after the decimal point. """ - return self.digits[1] + return self.digits[1] if self.digits is not None else None def __init__( self, diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index 8c3f626be..5ab834453 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -45,6 +45,20 @@ class Movie(Table): pydantic_model(box_office=decimal.Decimal("1.0")) + def test_numeric_without_digits(self): + class Movie(Table): + box_office = Numeric() + + try: + create_pydantic_model(table=Movie) + except TypeError: + self.fail( + "Creating numeric field without" + " digits failed in pydantic model." + ) + else: + self.assertTrue(True) + class TestSecretColumn(TestCase): def test_secret_param(self): @@ -93,7 +107,6 @@ class TestColumnHelpText(TestCase): """ def test_help_text_present(self): - help_text = "In millions of US dollars." class Movie(Table): @@ -115,7 +128,6 @@ class TestTableHelpText(TestCase): """ def test_help_text_present(self): - help_text = "Movies which were released in cinemas." class Movie(Table, help_text=help_text): From 478578993c0b04a23a2642bf380b206eccd5d19f Mon Sep 17 00:00:00 2001 From: ali sayyah Date: Thu, 16 Sep 2021 18:31:34 +0430 Subject: [PATCH 068/727] create multiple tables at once (#235) * added 'create_tables' function for creating multiple tables at once * added an unittest for 'create_tables' function * added a unit test for 'create_tables' function * fix for 'CASCADE' problem in 'create_tables' test * moved sort_table_classes, _get_graph and create_tables to piccolo/table.py * add 'create_tables' to docs --- docs/src/piccolo/query_types/create_table.rst | 10 +++ piccolo/apps/fixture/commands/dump.py | 2 +- .../apps/migrations/auto/migration_manager.py | 73 +---------------- piccolo/apps/schema/commands/generate.py | 3 +- piccolo/table.py | 82 ++++++++++++++++++- .../migrations/auto/test_migration_manager.py | 7 +- tests/table/test_create_tables.py | 15 ++++ 7 files changed, 110 insertions(+), 82 deletions(-) create mode 100644 tests/table/test_create_tables.py diff --git a/docs/src/piccolo/query_types/create_table.rst b/docs/src/piccolo/query_types/create_table.rst index 726b90329..0144d90f3 100644 --- a/docs/src/piccolo/query_types/create_table.rst +++ b/docs/src/piccolo/query_types/create_table.rst @@ -19,3 +19,13 @@ To prevent an error from being raised if the table already exists: >>> Band.create_table(if_not_exists=True).run_sync() [] + +Also, you can create multiple tables at once. + +This function will automatically sort tables based on their foreign keys so they're created in the right order: + +.. code-block:: python + + >>> from piccolo.table import create_tables + >>> create_tables(Band, Manager, if_not_exists=True) + diff --git a/piccolo/apps/fixture/commands/dump.py b/piccolo/apps/fixture/commands/dump.py index 1f13ed199..7f410bd77 100644 --- a/piccolo/apps/fixture/commands/dump.py +++ b/piccolo/apps/fixture/commands/dump.py @@ -6,8 +6,8 @@ FixtureConfig, create_pydantic_fixture_model, ) -from piccolo.apps.migrations.auto.migration_manager import sort_table_classes from piccolo.conf.apps import Finder +from piccolo.table import sort_table_classes async def get_dump( diff --git a/piccolo/apps/migrations/auto/migration_manager.py b/piccolo/apps/migrations/auto/migration_manager.py index 0b2d1e3f5..4d3b19f90 100644 --- a/piccolo/apps/migrations/auto/migration_manager.py +++ b/piccolo/apps/migrations/auto/migration_manager.py @@ -14,8 +14,7 @@ from piccolo.apps.migrations.auto.serialisation import deserialise_params from piccolo.columns import Column, column_types from piccolo.engine import engine_finder -from piccolo.table import Table, create_table_class -from piccolo.utils.graphlib import TopologicalSorter +from piccolo.table import Table, create_table_class, sort_table_classes @dataclass @@ -118,76 +117,6 @@ def table_class_names(self) -> t.List[str]: return list(set([i.table_class_name for i in self.alter_columns])) -def _get_graph( - table_classes: t.List[t.Type[Table]], - iterations: int = 0, - max_iterations: int = 5, -) -> t.Dict[str, t.Set[str]]: - """ - Analyses the tables based on their foreign keys, and returns a data - structure like: - - .. code-block:: python - - {'band': {'manager'}, 'concert': {'band', 'venue'}, 'manager': set()} - - The keys are tablenames, and the values are tablenames directly connected - to it via a foreign key. - - """ - output: t.Dict[str, t.Set[str]] = {} - - if iterations >= max_iterations: - return output - - for table_class in table_classes: - dependents: t.Set[str] = set() - for fk in table_class._meta.foreign_key_columns: - dependents.add( - fk._foreign_key_meta.resolved_references._meta.tablename - ) - - # We also recursively check the related tables to get a fuller - # picture of the schema and relationships. - referenced_table = fk._foreign_key_meta.resolved_references - output.update( - _get_graph( - [referenced_table], - iterations=iterations + 1, - ) - ) - - output[table_class._meta.tablename] = dependents - - return output - - -def sort_table_classes( - table_classes: t.List[t.Type[Table]], -) -> t.List[t.Type[Table]]: - """ - Sort the table classes based on their foreign keys, so they can be created - in the correct order. - """ - table_class_dict = { - table_class._meta.tablename: table_class - for table_class in table_classes - } - - graph = _get_graph(table_classes) - - sorter = TopologicalSorter(graph) - ordered_tablenames = tuple(sorter.static_order()) - - output: t.List[t.Type[Table]] = [] - for tablename in ordered_tablenames: - table_class = table_class_dict.get(tablename, None) - if table_class is not None: - output.append(table_class) - - return output - - @dataclass class MigrationManager: """ diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index 8641568f8..cd1229712 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -6,7 +6,6 @@ import black from typing_extensions import Literal -from piccolo.apps.migrations.auto.migration_manager import sort_table_classes from piccolo.apps.migrations.auto.serialisation import serialise_params from piccolo.columns.base import Column from piccolo.columns.column_types import ( @@ -31,7 +30,7 @@ ) from piccolo.engine.finder import engine_finder from piccolo.engine.postgres import PostgresEngine -from piccolo.table import Table, create_table_class +from piccolo.table import Table, create_table_class, sort_table_classes from piccolo.utils.naming import _snake_to_camel if t.TYPE_CHECKING: # pragma: no cover diff --git a/piccolo/table.py b/piccolo/table.py index d20e4a1cc..6edf7728b 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -40,12 +40,12 @@ from piccolo.query.methods.indexes import Indexes from piccolo.querystring import QueryString, Unquoted from piccolo.utils import _camel_to_snake +from piccolo.utils.graphlib import TopologicalSorter from piccolo.utils.sql_values import convert_to_sql_value if t.TYPE_CHECKING: from piccolo.columns import Selectable - PROTECTED_TABLENAMES = ("user",) @@ -126,7 +126,6 @@ def __str__(cls): class Table(metaclass=TableMetaclass): - # These are just placeholder values, so type inference isn't confused - the # actual values are set in __init_subclass__. _meta = TableMeta() @@ -975,3 +974,82 @@ def create_table_class( kwds=class_kwargs, exec_body=lambda namespace: namespace.update(class_members), ) + + +def create_tables(*args: t.Type[Table], if_not_exists: bool = False) -> None: + """ + Creates multiple tables that passed to it. + """ + sorted_table_classes = sort_table_classes(list(args)) + for table in sorted_table_classes: + Create(table=table, if_not_exists=if_not_exists).run_sync() + + +def sort_table_classes( + table_classes: t.List[t.Type[Table]], +) -> t.List[t.Type[Table]]: + """ + Sort the table classes based on their foreign keys, so they can be created + in the correct order. + """ + table_class_dict = { + table_class._meta.tablename: table_class + for table_class in table_classes + } + + graph = _get_graph(table_classes) + + sorter = TopologicalSorter(graph) + ordered_tablenames = tuple(sorter.static_order()) + + output: t.List[t.Type[Table]] = [] + for tablename in ordered_tablenames: + table_class = table_class_dict.get(tablename, None) + if table_class is not None: + output.append(table_class) + + return output + + +def _get_graph( + table_classes: t.List[t.Type[Table]], + iterations: int = 0, + max_iterations: int = 5, +) -> t.Dict[str, t.Set[str]]: + """ + Analyses the tables based on their foreign keys, and returns a data + structure like: + + .. code-block:: python + + {'band': {'manager'}, 'concert': {'band', 'venue'}, 'manager': set()} + + The keys are tablenames, and the values are tablenames directly connected + to it via a foreign key. + + """ + output: t.Dict[str, t.Set[str]] = {} + + if iterations >= max_iterations: + return output + + for table_class in table_classes: + dependents: t.Set[str] = set() + for fk in table_class._meta.foreign_key_columns: + dependents.add( + fk._foreign_key_meta.resolved_references._meta.tablename + ) + + # We also recursively check the related tables to get a fuller + # picture of the schema and relationships. + referenced_table = fk._foreign_key_meta.resolved_references + output.update( + _get_graph( + [referenced_table], + iterations=iterations + 1, + ) + ) + + output[table_class._meta.tablename] = dependents + + return output diff --git a/tests/apps/migrations/auto/test_migration_manager.py b/tests/apps/migrations/auto/test_migration_manager.py index 124ed1ba4..af3e5625c 100644 --- a/tests/apps/migrations/auto/test_migration_manager.py +++ b/tests/apps/migrations/auto/test_migration_manager.py @@ -2,16 +2,13 @@ from unittest import TestCase from unittest.mock import MagicMock, patch -from piccolo.apps.migrations.auto.migration_manager import ( - MigrationManager, - sort_table_classes, -) +from piccolo.apps.migrations.auto.migration_manager import MigrationManager from piccolo.apps.migrations.commands.base import BaseMigrationManager from piccolo.columns import Text, Varchar from piccolo.columns.base import OnDelete, OnUpdate from piccolo.columns.column_types import ForeignKey from piccolo.conf.apps import AppConfig -from piccolo.table import Table +from piccolo.table import Table, sort_table_classes from piccolo.utils.lazy_loader import LazyLoader from tests.base import DBTestCase, postgres_only, set_mock_return_value from tests.example_apps.music.tables import Band, Concert, Manager, Venue diff --git a/tests/table/test_create_tables.py b/tests/table/test_create_tables.py new file mode 100644 index 000000000..0e3c94237 --- /dev/null +++ b/tests/table/test_create_tables.py @@ -0,0 +1,15 @@ +from unittest import TestCase + +from piccolo.table import create_tables +from tests.example_apps.music.tables import Band, Manager + + +class TestCreateTables(TestCase): + def tearDown(self) -> None: + Band.alter().drop_table(if_exists=True).run_sync() + Manager.alter().drop_table(if_exists=True).run_sync() + + def test_create_tables(self): + create_tables(Manager, Band, if_not_exists=False) + self.assertTrue(Manager.table_exists().run_sync()) + self.assertTrue(Band.table_exists().run_sync()) From 461a7786d7f4f8926d7c74332dcdc5c1fcbb8ffd Mon Sep 17 00:00:00 2001 From: Jaroslaw Zabiello Date: Thu, 16 Sep 2021 20:10:41 +0100 Subject: [PATCH 069/727] Fix `fixtures` app name typos (#234) * Update CHANGES Fixed wrong command * make sure `fixtures` is used everywhere Co-authored-by: Daniel Townsend --- CHANGES | 6 +++--- docs/src/piccolo/projects_and_apps/included_apps.rst | 8 ++++---- piccolo/apps/{fixture => fixtures}/__init__.py | 0 piccolo/apps/{fixture => fixtures}/commands/__init__.py | 0 piccolo/apps/{fixture => fixtures}/commands/dump.py | 2 +- piccolo/apps/{fixture => fixtures}/commands/load.py | 2 +- piccolo/apps/{fixture => fixtures}/commands/shared.py | 0 piccolo/apps/{fixture => fixtures}/piccolo_app.py | 0 piccolo/main.py | 4 ++-- tests/apps/{fixture => fixtures}/__init__.py | 0 tests/apps/{fixture => fixtures}/commands/__init__.py | 0 .../apps/{fixture => fixtures}/commands/test_dump_load.py | 4 ++-- tests/apps/{fixture => fixtures}/commands/test_shared.py | 2 +- 13 files changed, 14 insertions(+), 14 deletions(-) rename piccolo/apps/{fixture => fixtures}/__init__.py (100%) rename piccolo/apps/{fixture => fixtures}/commands/__init__.py (100%) rename piccolo/apps/{fixture => fixtures}/commands/dump.py (98%) rename piccolo/apps/{fixture => fixtures}/commands/load.py (97%) rename piccolo/apps/{fixture => fixtures}/commands/shared.py (100%) rename piccolo/apps/{fixture => fixtures}/piccolo_app.py (100%) rename tests/apps/{fixture => fixtures}/__init__.py (100%) rename tests/apps/{fixture => fixtures}/commands/__init__.py (100%) rename tests/apps/{fixture => fixtures}/commands/test_dump_load.py (97%) rename tests/apps/{fixture => fixtures}/commands/test_shared.py (97%) diff --git a/CHANGES b/CHANGES index 327ca0f55..9e2421b1e 100644 --- a/CHANGES +++ b/CHANGES @@ -17,7 +17,7 @@ requirements. 0.46.0 ------ -Added the fixture app. This is used to dump data from a database to a JSON +Added the fixtures app. This is used to dump data from a database to a JSON file, and then reload it again. It's useful for seeding a database with essential data, whether that's a colleague setting up their local environment, or deploying to production. @@ -26,13 +26,13 @@ To create a fixture: .. code-block:: bash - piccolo fixture dump --apps=blog > fixture.json + piccolo fixtures dump --apps=blog > fixture.json To load a fixture: .. code-block:: bash - piccolo fixture load fixture.json + piccolo fixtures load fixture.json As part of this change, Piccolo's Pydantic support was brought into this library (prior to this it only existed within the ``piccolo_api`` library). At diff --git a/docs/src/piccolo/projects_and_apps/included_apps.rst b/docs/src/piccolo/projects_and_apps/included_apps.rst index 89431692d..aad322526 100644 --- a/docs/src/piccolo/projects_and_apps/included_apps.rst +++ b/docs/src/piccolo/projects_and_apps/included_apps.rst @@ -38,8 +38,8 @@ Lets you scaffold an ASGI web app. See :ref:`ASGICommand`. ------------------------------------------------------------------------------- -fixture -~~~~~~~ +fixtures +~~~~~~~~ Fixtures are used when you want to seed your database with essential data (for example, country names). @@ -66,7 +66,7 @@ a subset of apps and tables instead, for example: .. code-block:: bash - piccolo fixture dump --apps=blog --tables=Post > fixtures.json + piccolo fixtures dump --apps=blog --tables=Post > fixtures.json # Or for multiple apps / tables piccolo fixtures dump --apps=blog,shop --tables=Post,Product > fixtures.json @@ -75,7 +75,7 @@ To load the fixture: .. code-block:: bash - piccolo fixture load fixtures.json + piccolo fixtures load fixtures.json ------------------------------------------------------------------------------- diff --git a/piccolo/apps/fixture/__init__.py b/piccolo/apps/fixtures/__init__.py similarity index 100% rename from piccolo/apps/fixture/__init__.py rename to piccolo/apps/fixtures/__init__.py diff --git a/piccolo/apps/fixture/commands/__init__.py b/piccolo/apps/fixtures/commands/__init__.py similarity index 100% rename from piccolo/apps/fixture/commands/__init__.py rename to piccolo/apps/fixtures/commands/__init__.py diff --git a/piccolo/apps/fixture/commands/dump.py b/piccolo/apps/fixtures/commands/dump.py similarity index 98% rename from piccolo/apps/fixture/commands/dump.py rename to piccolo/apps/fixtures/commands/dump.py index 7f410bd77..512622575 100644 --- a/piccolo/apps/fixture/commands/dump.py +++ b/piccolo/apps/fixtures/commands/dump.py @@ -2,7 +2,7 @@ import typing as t -from piccolo.apps.fixture.commands.shared import ( +from piccolo.apps.fixtures.commands.shared import ( FixtureConfig, create_pydantic_fixture_model, ) diff --git a/piccolo/apps/fixture/commands/load.py b/piccolo/apps/fixtures/commands/load.py similarity index 97% rename from piccolo/apps/fixture/commands/load.py rename to piccolo/apps/fixtures/commands/load.py index 64cfc2ba0..22ab195ab 100644 --- a/piccolo/apps/fixture/commands/load.py +++ b/piccolo/apps/fixtures/commands/load.py @@ -1,6 +1,6 @@ from __future__ import annotations -from piccolo.apps.fixture.commands.shared import ( +from piccolo.apps.fixtures.commands.shared import ( FixtureConfig, create_pydantic_fixture_model, ) diff --git a/piccolo/apps/fixture/commands/shared.py b/piccolo/apps/fixtures/commands/shared.py similarity index 100% rename from piccolo/apps/fixture/commands/shared.py rename to piccolo/apps/fixtures/commands/shared.py diff --git a/piccolo/apps/fixture/piccolo_app.py b/piccolo/apps/fixtures/piccolo_app.py similarity index 100% rename from piccolo/apps/fixture/piccolo_app.py rename to piccolo/apps/fixtures/piccolo_app.py diff --git a/piccolo/main.py b/piccolo/main.py index f0bc5f9bb..33f9a11db 100644 --- a/piccolo/main.py +++ b/piccolo/main.py @@ -12,7 +12,7 @@ from piccolo.apps.app.piccolo_app import APP_CONFIG as app_config from piccolo.apps.asgi.piccolo_app import APP_CONFIG as asgi_config -from piccolo.apps.fixture.piccolo_app import APP_CONFIG as fixture_config +from piccolo.apps.fixtures.piccolo_app import APP_CONFIG as fixtures_config from piccolo.apps.meta.piccolo_app import APP_CONFIG as meta_config from piccolo.apps.migrations.commands.check import CheckMigrationManager from piccolo.apps.migrations.piccolo_app import APP_CONFIG as migrations_config @@ -61,7 +61,7 @@ def main(): for _app_config in [ app_config, asgi_config, - fixture_config, + fixtures_config, meta_config, migrations_config, playground_config, diff --git a/tests/apps/fixture/__init__.py b/tests/apps/fixtures/__init__.py similarity index 100% rename from tests/apps/fixture/__init__.py rename to tests/apps/fixtures/__init__.py diff --git a/tests/apps/fixture/commands/__init__.py b/tests/apps/fixtures/commands/__init__.py similarity index 100% rename from tests/apps/fixture/commands/__init__.py rename to tests/apps/fixtures/commands/__init__.py diff --git a/tests/apps/fixture/commands/test_dump_load.py b/tests/apps/fixtures/commands/test_dump_load.py similarity index 97% rename from tests/apps/fixture/commands/test_dump_load.py rename to tests/apps/fixtures/commands/test_dump_load.py index 9f4b6693f..0a8be8ba0 100644 --- a/tests/apps/fixture/commands/test_dump_load.py +++ b/tests/apps/fixtures/commands/test_dump_load.py @@ -3,11 +3,11 @@ import uuid from unittest import TestCase -from piccolo.apps.fixture.commands.dump import ( +from piccolo.apps.fixtures.commands.dump import ( FixtureConfig, dump_to_json_string, ) -from piccolo.apps.fixture.commands.load import load_json_string +from piccolo.apps.fixtures.commands.load import load_json_string from piccolo.utils.sync import run_sync from tests.example_apps.mega.tables import MegaTable, SmallTable diff --git a/tests/apps/fixture/commands/test_shared.py b/tests/apps/fixtures/commands/test_shared.py similarity index 97% rename from tests/apps/fixture/commands/test_shared.py rename to tests/apps/fixtures/commands/test_shared.py index 7b56dabd1..b2246aa71 100644 --- a/tests/apps/fixture/commands/test_shared.py +++ b/tests/apps/fixtures/commands/test_shared.py @@ -3,7 +3,7 @@ import uuid from unittest import TestCase -from piccolo.apps.fixture.commands.shared import ( +from piccolo.apps.fixtures.commands.shared import ( FixtureConfig, create_pydantic_fixture_model, ) From 7f062ad9d8e3d5b96652f439093d82274f510e24 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 16 Sep 2021 20:23:30 +0100 Subject: [PATCH 070/727] bumped version --- CHANGES | 25 +++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 9e2421b1e..8692ebf69 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,31 @@ Changes ======= +0.49.0 +------ +Fixed a bug with ``create_pydantic_model`` when used with a ``Decimal`` / +``Numeric`` column when no ``digits`` arguments was set (courtesy @AliSayyah). + +Added the ``create_tables`` function, which accepts a sequence of ``Table`` +subclasses, then sorts them based on their ``ForeignKey`` columns, and creates +them. This is really useful for people who aren't using migrations (for +example, when using Piccolo in a simple data science script). Courtesy +@AliSayyah. + +.. code-block:: python + + from piccolo.tables import create_tables + + create_tables(Band, Manager, if_not_exists=True) + + # Equivalent to: + Manager.create_table(if_not_exists=True).run_sync() + Band.create_table(if_not_exists=True).run_sync() + +Fixed typos with the new fixtures app - sometimes it was referred to as +``fixture`` and other times ``fixtures``. It's now standardised as +``fixtures`` (courtesy @hipertracker). + 0.48.0 ------ The ``piccolo user create`` command can now be used by passing in command line diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 851aa6ec5..197168277 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.48.0" +__VERSION__ = "0.49.0" From 52051255a91c73f3219689932b6077924870dcd1 Mon Sep 17 00:00:00 2001 From: ali sayyah Date: Sun, 19 Sep 2021 15:38:24 +0430 Subject: [PATCH 071/727] multiple arguments in WHERE clause (#244) * WHERE clause now will support multiple arguments(and empty clause) based on discussion #237 * updated Docs --- docs/src/piccolo/query_clauses/where.rst | 15 +++++++++++++++ piccolo/query/methods/count.py | 4 ++-- piccolo/query/methods/delete.py | 4 ++-- piccolo/query/methods/exists.py | 4 ++-- piccolo/query/methods/objects.py | 4 ++-- piccolo/query/methods/select.py | 4 ++-- piccolo/query/methods/update.py | 4 ++-- piccolo/query/mixins.py | 11 ++++++----- tests/table/test_select.py | 13 ++++++++++++- 9 files changed, 45 insertions(+), 18 deletions(-) diff --git a/docs/src/piccolo/query_clauses/where.rst b/docs/src/piccolo/query_clauses/where.rst index 6878e2297..85f583f96 100644 --- a/docs/src/piccolo/query_clauses/where.rst +++ b/docs/src/piccolo/query_clauses/where.rst @@ -191,6 +191,21 @@ Using multiple ``where`` clauses is equivalent to an AND. b.popularity < 1000 ).run_sync() +Also, multiple arguments inside ``where`` clause is equivalent to an AND. + +.. code-block:: python + + b = Band + + # These are equivalent: + b.select().where( + (b.popularity >= 100) & (b.popularity < 1000) + ).run_sync() + + b.select().where( + b.popularity >= 100, b.popularity < 1000 + ).run_sync() + Using And / Or directly ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/piccolo/query/methods/count.py b/piccolo/query/methods/count.py index 100bf43a0..7292793f2 100644 --- a/piccolo/query/methods/count.py +++ b/piccolo/query/methods/count.py @@ -20,8 +20,8 @@ def __init__(self, table: t.Type[Table], **kwargs): super().__init__(table, **kwargs) self.where_delegate = WhereDelegate() - def where(self, where: Combinable) -> Count: - self.where_delegate.where(where) + def where(self, *where: Combinable) -> Count: + self.where_delegate.where(*where) return self async def response_handler(self, response) -> bool: diff --git a/piccolo/query/methods/delete.py b/piccolo/query/methods/delete.py index be95a0c01..f9e816e64 100644 --- a/piccolo/query/methods/delete.py +++ b/piccolo/query/methods/delete.py @@ -24,8 +24,8 @@ def __init__(self, table: t.Type[Table], force: bool = False, **kwargs): self.force = force self.where_delegate = WhereDelegate() - def where(self, where: Combinable) -> Delete: - self.where_delegate.where(where) + def where(self, *where: Combinable) -> Delete: + self.where_delegate.where(*where) return self def _validate(self): diff --git a/piccolo/query/methods/exists.py b/piccolo/query/methods/exists.py index 09fc40071..7b5b4b3a7 100644 --- a/piccolo/query/methods/exists.py +++ b/piccolo/query/methods/exists.py @@ -21,8 +21,8 @@ def __init__(self, table: t.Type[Table], **kwargs): super().__init__(table, **kwargs) self.where_delegate = WhereDelegate() - def where(self, where: Combinable) -> Exists: - self.where_delegate.where(where) + def where(self, *where: Combinable) -> Exists: + self.where_delegate.where(*where) return self async def response_handler(self, response) -> bool: diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index b5ef82a9d..11da12193 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -155,8 +155,8 @@ def order_by(self, *columns: Column, ascending=True) -> Objects: self.order_by_delegate.order_by(*columns, ascending=ascending) return self - def where(self, where: Combinable) -> Objects: - self.where_delegate.where(where) + def where(self, *where: Combinable) -> Objects: + self.where_delegate.where(*where) return self async def batch( diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 92ad42cbb..c5be9c902 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -246,8 +246,8 @@ def output( ) return self - def where(self, where: Combinable) -> Select: - self.where_delegate.where(where) + def where(self, *where: Combinable) -> Select: + self.where_delegate.where(*where) return self async def batch( diff --git a/piccolo/query/methods/update.py b/piccolo/query/methods/update.py index 013c9d53e..94997c0bc 100644 --- a/piccolo/query/methods/update.py +++ b/piccolo/query/methods/update.py @@ -30,8 +30,8 @@ def values( self.values_delegate.values(values) return self - def where(self, where: Combinable) -> Update: - self.where_delegate.where(where) + def where(self, *where: Combinable) -> Update: + self.where_delegate.where(*where) return self def validate(self): diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index 178434821..8f75abbc9 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -113,11 +113,12 @@ def _extract_columns(self, combinable: Combinable): self._extract_columns(combinable.first) self._extract_columns(combinable.second) - def where(self, where: Combinable): - if self._where: - self._where = And(self._where, where) - else: - self._where = where + def where(self, *where: Combinable): + for arg in where: + if self._where: + self._where = And(self._where, arg) + else: + self._where = arg @dataclass diff --git a/tests/table/test_select.py b/tests/table/test_select.py index f2d9f0ace..92ac15ab2 100644 --- a/tests/table/test_select.py +++ b/tests/table/test_select.py @@ -63,6 +63,18 @@ def test_where_equals(self): ) self.assertEqual(response, [{"name": "Pythonistas"}]) + # check multiple arguments inside WHERE clause + response = ( + Band.select(Band.name) + .where(Band.manager.id == 1, Band.popularity == 500) + .run_sync() + ) + self.assertEqual(response, []) + + # check empty WHERE clause + response = Band.select(Band.name).where().run_sync() + self.assertEqual(response, [{"name": "Pythonistas"}]) + @postgres_only def test_where_like_postgres(self): """ @@ -148,7 +160,6 @@ def test_where_ilike_sqlite(self): "%ist%", "%IST%", ): - self.assertEqual( Band.select(Band.name) .where(Band.name.ilike(ilike_query)) From d861bb5671dbbd4906ea83b290fe1cbfbc8759a4 Mon Sep 17 00:00:00 2001 From: ali sayyah Date: Sun, 19 Sep 2021 15:47:55 +0430 Subject: [PATCH 072/727] added create method (#245) * added create method #238 * removed unnecessary comment * updated Docs --- docs/src/piccolo/query_types/objects.rst | 6 +++++ piccolo/query/methods/objects.py | 33 ++++++++++++++++++++++++ tests/table/instance/test_create.py | 20 ++++++++++++++ 3 files changed, 59 insertions(+) create mode 100644 tests/table/instance/test_create.py diff --git a/docs/src/piccolo/query_types/objects.rst b/docs/src/piccolo/query_types/objects.rst index 80ba9c4e7..39b75a0c4 100644 --- a/docs/src/piccolo/query_types/objects.rst +++ b/docs/src/piccolo/query_types/objects.rst @@ -57,6 +57,12 @@ Creating objects >>> band = Band(name="C-Sharps", popularity=100) >>> band.save().run_sync() +This can also be done like this: + +.. code-block:: python + + >>> band.objects().create(name="C-Sharps", popularity=100).run_sync() + ------------------------------------------------------------------------------- Updating objects diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index 11da12193..1e0ec450b 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -82,6 +82,36 @@ def prefetch(self, *fk_columns) -> GetOrCreate: return self +@dataclass +class Create: + query: Objects + columns: t.Dict[str, t.Any] + + async def run(self): + instance = self.query.table() + + for column, value in self.columns.items(): + if isinstance(column, str): + column = instance._meta.get_column_by_name(column) + setattr(instance, column._meta.name, value) + + await instance.save().run() + + instance._was_created = True + + return instance + + def __await__(self): + """ + If the user doesn't explicity call .run(), proxy to it as a + convenience. + """ + return self.run().__await__() + + def run_sync(self): + return run_sync(self.run()) + + @dataclass class Objects(Query): """ @@ -151,6 +181,9 @@ def get_or_create( ): return GetOrCreate(query=self, where=where, defaults=defaults) + def create(self, **columns: t.Any): + return Create(query=self, columns=columns) + def order_by(self, *columns: Column, ascending=True) -> Objects: self.order_by_delegate.order_by(*columns, ascending=ascending) return self diff --git a/tests/table/instance/test_create.py b/tests/table/instance/test_create.py new file mode 100644 index 000000000..73d028b23 --- /dev/null +++ b/tests/table/instance/test_create.py @@ -0,0 +1,20 @@ +from unittest import TestCase + +from tests.example_apps.music.tables import Manager + + +class TestCreate(TestCase): + def setUp(self): + Manager.create_table().run_sync() + + def tearDown(self): + Manager.alter().drop_table().run_sync() + + def test_create_new(self): + """ + Make sure that creating a new instance works. + """ + Manager.objects().create(name="Maz").run_sync() + + names = [i["name"] for i in Manager.select(Manager.name).run_sync()] + self.assertTrue("Maz" in names) From 8d1bcf25e1b800d8181b42844eac7c3dcf4fc796 Mon Sep 17 00:00:00 2001 From: ali sayyah Date: Sun, 19 Sep 2021 17:36:53 +0430 Subject: [PATCH 073/727] #242 bug-fix (#246) * This should fix #242 * add test for unknown column types Co-authored-by: Daniel Townsend --- piccolo/apps/schema/commands/generate.py | 80 ++++++++++----------- tests/apps/schema/commands/test_generate.py | 29 +++++++- 2 files changed, 68 insertions(+), 41 deletions(-) diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index cd1229712..1fea0fd0f 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -309,52 +309,52 @@ class Schema(Table, db=engine): data_type = pg_row_meta.data_type column_type = COLUMN_TYPE_MAP.get(data_type, None) column_name = pg_row_meta.column_name - - if column_type: - kwargs: t.Dict[str, t.Any] = { - "null": pg_row_meta.is_nullable == "YES", - "unique": constraints.is_unique(column_name=column_name), - } - - if constraints.is_primary_key(column_name=column_name): - kwargs["primary_key"] = True - if column_type == Integer: - column_type = Serial - - if constraints.is_foreign_key(column_name=column_name): - fk_constraint_name = ( - constraints.get_foreign_key_constraint_name( - column_name=column_name - ) - ) - column_type = ForeignKey - referenced_tablename = await get_foreign_key_reference( - table_class=Schema, constraint_name=fk_constraint_name + if not column_type: + warnings.append(f"{tablename}.{column_name} ['{data_type}']") + column_type = Column + + kwargs: t.Dict[str, t.Any] = { + "null": pg_row_meta.is_nullable == "YES", + "unique": constraints.is_unique(column_name=column_name), + } + + if constraints.is_primary_key(column_name=column_name): + kwargs["primary_key"] = True + if column_type == Integer: + column_type = Serial + + if constraints.is_foreign_key(column_name=column_name): + fk_constraint_name = ( + constraints.get_foreign_key_constraint_name( + column_name=column_name ) - if referenced_tablename: - kwargs["references"] = create_table_class( - _snake_to_camel(referenced_tablename) - ) - else: - kwargs["references"] = ForeignKeyPlaceholder - - imports.add( - "from piccolo.columns.column_types import " - + column_type.__name__ ) + column_type = ForeignKey + referenced_tablename = await get_foreign_key_reference( + table_class=Schema, constraint_name=fk_constraint_name + ) + if referenced_tablename: + kwargs["references"] = create_table_class( + _snake_to_camel(referenced_tablename) + ) + else: + kwargs["references"] = ForeignKeyPlaceholder + + imports.add( + "from piccolo.columns.column_types import " + + column_type.__name__ + ) - if column_type is Varchar: - kwargs["length"] = pg_row_meta.character_maximum_length + if column_type is Varchar: + kwargs["length"] = pg_row_meta.character_maximum_length - column = column_type(**kwargs) + column = column_type(**kwargs) - serialised_params = serialise_params(column._meta.params) - for extra_import in serialised_params.extra_imports: - imports.add(extra_import.__repr__()) + serialised_params = serialise_params(column._meta.params) + for extra_import in serialised_params.extra_imports: + imports.add(extra_import.__repr__()) - columns[column_name] = column - else: - warnings.append(f"{tablename}.{column_name} ['{data_type}']") + columns[column_name] = column table = create_table_class( class_name=_snake_to_camel(tablename), diff --git a/tests/apps/schema/commands/test_generate.py b/tests/apps/schema/commands/test_generate.py index afdfb5256..37f4c2b2a 100644 --- a/tests/apps/schema/commands/test_generate.py +++ b/tests/apps/schema/commands/test_generate.py @@ -10,6 +10,7 @@ generate, get_output_schema, ) +from piccolo.columns.base import Column from piccolo.columns.column_types import Varchar from piccolo.table import Table from piccolo.utils.sync import run_sync @@ -71,7 +72,7 @@ def test_get_output_schema(self): self._compare_table_columns(SmallTable, SmallTable_) @patch("piccolo.apps.schema.commands.generate.print") - def test_generate(self, print_: MagicMock): + def test_generate_command(self, print_: MagicMock): """ Test the main generate command runs without errors. """ @@ -81,3 +82,29 @@ def test_generate(self, print_: MagicMock): # Make sure the output is valid Python code (will raise a SyntaxError # exception otherwise). ast.parse(file_contents) + + def test_unknown_column_type(self): + """ + Make sure unknown column types are handled gracefully. + """ + + class Box(Column): + """ + A column type which isn't supported by Piccolo officially yet. + """ + + pass + + MegaTable.alter().add_column("box", Box()).run_sync() + + output_schema: OutputSchema = run_sync(get_output_schema()) + + # Make sure there's a warning. + self.assertEqual(output_schema.warnings, ["mega_table.box ['box']"]) + + # Make sure the column type of the generated table is just ``Column``. + for table in output_schema.tables: + if table.__name__ == "MegaTable": + self.assertEqual( + output_schema.tables[1].box.__class__.__name__, "Column" + ) From ac195c463daf907373121e1c9fc03b919d6435a4 Mon Sep 17 00:00:00 2001 From: Jaroslaw Zabiello Date: Mon, 20 Sep 2021 21:20:06 +0100 Subject: [PATCH 074/727] Update piccolo_apps.rst (#254) * Update piccolo_apps.rst Add missing information about the --trace flag to the command line call * add extra backticks Otherwise RST seems to do something weird where the two dashes look like they're merged into one. Co-authored-by: Daniel Townsend --- docs/src/piccolo/projects_and_apps/piccolo_apps.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/src/piccolo/projects_and_apps/piccolo_apps.rst b/docs/src/piccolo/projects_and_apps/piccolo_apps.rst index b83cc1df5..19a1b3347 100644 --- a/docs/src/piccolo/projects_and_apps/piccolo_apps.rst +++ b/docs/src/piccolo/projects_and_apps/piccolo_apps.rst @@ -182,6 +182,13 @@ And from the command line: >>> piccolo my_app say_hello bob bob +If the code contains an error to see more details in the output add a ``--trace`` flag to the command line. + +.. code-block:: bash + + >>> piccolo my_app say_hello bob --trace + + By convention, store the command definitions in a `commands` folder in your app. From 81a6574585555288893c746924fc82700408a46e Mon Sep 17 00:00:00 2001 From: ali sayyah Date: Tue, 21 Sep 2021 01:19:35 +0430 Subject: [PATCH 075/727] added `Double Precision` column type (#255) * added `Double Precision` column type * Update column_types.rst * tweak docs * add migration file to `mega` app This isn't directly related to this new feature - I just realised that it was missing. * add tests for migrations using `Real` or `DoublePrecision` column types * add one more test for `DoublePrecision` column type Co-authored-by: Daniel Townsend --- docs/src/piccolo/schema/column_types.rst | 7 + piccolo/apps/schema/commands/generate.py | 2 + piccolo/columns/__init__.py | 1 + piccolo/columns/column_types.py | 10 + .../apps/fixtures/commands/test_dump_load.py | 2 + .../auto/integration/test_migrations.py | 32 ++ tests/columns/test_defaults.py | 9 + tests/columns/test_double_precision.py | 24 + .../2021-09-20T21-23-25-698988.py | 448 ++++++++++++++++++ tests/example_apps/mega/tables.py | 2 + 10 files changed, 537 insertions(+) create mode 100644 tests/columns/test_double_precision.py create mode 100644 tests/example_apps/mega/piccolo_migrations/2021-09-20T21-23-25-698988.py diff --git a/docs/src/piccolo/schema/column_types.rst b/docs/src/piccolo/schema/column_types.rst index ce3f72424..191331c6b 100644 --- a/docs/src/piccolo/schema/column_types.rst +++ b/docs/src/piccolo/schema/column_types.rst @@ -78,6 +78,13 @@ Real .. hint:: There is also a ``Float`` column type, which is an alias for ``Real``. + +==== +Double Precision +==== + +.. autoclass:: DoublePrecision + ======== SmallInt ======== diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index 1fea0fd0f..ac01e0796 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -16,6 +16,7 @@ Boolean, Bytea, Date, + DoublePrecision, ForeignKey, Integer, Interval, @@ -156,6 +157,7 @@ def get_table_with_name(self, name: str) -> t.Optional[t.Type[Table]]: "jsonb": JSONB, "numeric": Numeric, "real": Real, + "double precision": DoublePrecision, "smallint": SmallInt, "text": Text, "timestamp with time zone": Timestamptz, diff --git a/piccolo/columns/__init__.py b/piccolo/columns/__init__.py index eb6fc9a5f..83c0289ec 100644 --- a/piccolo/columns/__init__.py +++ b/piccolo/columns/__init__.py @@ -9,6 +9,7 @@ Bytea, Date, Decimal, + DoublePrecision, Float, ForeignKey, Integer, diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index f89e2cfad..f26cc1f13 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -980,6 +980,16 @@ class Float(Real): pass +class DoublePrecision(Real): + """ + The same as ``Real``, except the numbers are stored with greater precision. + """ + + @property + def column_type(self): + return "DOUBLE PRECISION" + + ############################################################################### diff --git a/tests/apps/fixtures/commands/test_dump_load.py b/tests/apps/fixtures/commands/test_dump_load.py index 0a8be8ba0..98996ca4b 100644 --- a/tests/apps/fixtures/commands/test_dump_load.py +++ b/tests/apps/fixtures/commands/test_dump_load.py @@ -42,6 +42,7 @@ def insert_row(self): jsonb_col={"a": 1}, numeric_col=decimal.Decimal("1.1"), real_col=1.1, + double_precision_col=1.344, smallint_col=1, text_col="hello", timestamp_col=datetime.datetime(year=2021, month=1, day=1), @@ -116,6 +117,7 @@ def test_dump_load(self): "jsonb_col": '{"a":1}', "numeric_col": decimal.Decimal("1.1"), "real_col": 1.1, + "double_precision_col": 1.344, "smallint_col": 1, "text_col": "hello", "timestamp_col": datetime.datetime(2021, 1, 1, 0, 0), diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index 9682e888d..a9f0ff013 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -23,8 +23,10 @@ BigInt, Boolean, Date, + DoublePrecision, Integer, Interval, + Real, SmallInt, Text, Time, @@ -182,6 +184,36 @@ def test_integer_column(self): ] ) + def test_real_column(self): + self._test_migrations( + table_classes=[ + self.table(column) + for column in [ + Real(), + Real(default=1.1), + Real(null=True), + Real(null=False), + Real(index=True), + Real(index=False), + ] + ] + ) + + def test_double_precision_column(self): + self._test_migrations( + table_classes=[ + self.table(column) + for column in [ + DoublePrecision(), + DoublePrecision(default=1.1), + DoublePrecision(null=True), + DoublePrecision(null=False), + DoublePrecision(index=True), + DoublePrecision(index=False), + ] + ] + ) + def test_smallint_column(self): self._test_migrations( table_classes=[ diff --git a/tests/columns/test_defaults.py b/tests/columns/test_defaults.py index b8198d3c6..1e2bb48bc 100644 --- a/tests/columns/test_defaults.py +++ b/tests/columns/test_defaults.py @@ -9,6 +9,7 @@ BigInt, Date, DateNow, + DoublePrecision, ForeignKey, Integer, Numeric, @@ -55,6 +56,14 @@ def test_real(self): with self.assertRaises(ValueError): Real(default=None, null=False) + def test_double_precision(self): + DoublePrecision(default=0.0) + DoublePrecision(default=None, null=True) + with self.assertRaises(ValueError): + DoublePrecision(default="hello world") + with self.assertRaises(ValueError): + DoublePrecision(default=None, null=False) + def test_numeric(self): Numeric(default=decimal.Decimal(1.0)) Numeric(default=None, null=True) diff --git a/tests/columns/test_double_precision.py b/tests/columns/test_double_precision.py new file mode 100644 index 000000000..0b008bc73 --- /dev/null +++ b/tests/columns/test_double_precision.py @@ -0,0 +1,24 @@ +from unittest import TestCase + +from piccolo.columns.column_types import DoublePrecision +from piccolo.table import Table + + +class MyTable(Table): + column_a = DoublePrecision() + + +class TestDoublePrecision(TestCase): + def setUp(self): + MyTable.create_table().run_sync() + + def tearDown(self): + MyTable.alter().drop_table().run_sync() + + def test_creation(self): + row = MyTable(column_a=1.23) + row.save().run_sync() + + _row = MyTable.objects().first().run_sync() + self.assertTrue(type(_row.column_a) == float) + self.assertAlmostEqual(_row.column_a, 1.23) diff --git a/tests/example_apps/mega/piccolo_migrations/2021-09-20T21-23-25-698988.py b/tests/example_apps/mega/piccolo_migrations/2021-09-20T21-23-25-698988.py new file mode 100644 index 000000000..83b013c8a --- /dev/null +++ b/tests/example_apps/mega/piccolo_migrations/2021-09-20T21-23-25-698988.py @@ -0,0 +1,448 @@ +from piccolo.apps.migrations.auto import MigrationManager +from decimal import Decimal +from piccolo.columns.base import OnDelete +from piccolo.columns.base import OnUpdate +from piccolo.columns.column_types import BigInt +from piccolo.columns.column_types import Boolean +from piccolo.columns.column_types import Bytea +from piccolo.columns.column_types import Date +from piccolo.columns.column_types import DoublePrecision +from piccolo.columns.column_types import ForeignKey +from piccolo.columns.column_types import Integer +from piccolo.columns.column_types import Interval +from piccolo.columns.column_types import JSON +from piccolo.columns.column_types import JSONB +from piccolo.columns.column_types import Numeric +from piccolo.columns.column_types import Real +from piccolo.columns.column_types import Serial +from piccolo.columns.column_types import SmallInt +from piccolo.columns.column_types import Text +from piccolo.columns.column_types import Timestamp +from piccolo.columns.column_types import Timestamptz +from piccolo.columns.column_types import UUID +from piccolo.columns.column_types import Varchar +from piccolo.columns.defaults.date import DateNow +from piccolo.columns.defaults.interval import IntervalCustom +from piccolo.columns.defaults.timestamp import TimestampNow +from piccolo.columns.defaults.timestamptz import TimestamptzNow +from piccolo.columns.defaults.uuid import UUID4 +from piccolo.columns.indexes import IndexMethod +from piccolo.table import Table + + +class SmallTable(Table, tablename="small_table"): + id = Serial( + null=False, + primary_key=True, + unique=False, + index=False, + index_method=IndexMethod.btree, + choices=None, + ) + + +ID = "2021-09-20T21:23:25:698988" +VERSION = "0.49.0" +DESCRIPTION = "" + + +async def forwards(): + manager = MigrationManager( + migration_id=ID, app_name="mega", description=DESCRIPTION + ) + + manager.add_table("MegaTable", tablename="mega_table") + + manager.add_table("SmallTable", tablename="small_table") + + manager.add_column( + table_class_name="MegaTable", + tablename="mega_table", + column_name="bigint_col", + column_class_name="BigInt", + column_class=BigInt, + params={ + "default": 0, + "null": False, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + }, + ) + + manager.add_column( + table_class_name="MegaTable", + tablename="mega_table", + column_name="boolean_col", + column_class_name="Boolean", + column_class=Boolean, + params={ + "default": False, + "null": False, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + }, + ) + + manager.add_column( + table_class_name="MegaTable", + tablename="mega_table", + column_name="bytea_col", + column_class_name="Bytea", + column_class=Bytea, + params={ + "default": b"", + "null": False, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + }, + ) + + manager.add_column( + table_class_name="MegaTable", + tablename="mega_table", + column_name="date_col", + column_class_name="Date", + column_class=Date, + params={ + "default": DateNow(), + "null": False, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + }, + ) + + manager.add_column( + table_class_name="MegaTable", + tablename="mega_table", + column_name="foreignkey_col", + column_class_name="ForeignKey", + column_class=ForeignKey, + params={ + "references": SmallTable, + "on_delete": OnDelete.cascade, + "on_update": OnUpdate.cascade, + "null": True, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + }, + ) + + manager.add_column( + table_class_name="MegaTable", + tablename="mega_table", + column_name="integer_col", + column_class_name="Integer", + column_class=Integer, + params={ + "default": 0, + "null": False, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + }, + ) + + manager.add_column( + table_class_name="MegaTable", + tablename="mega_table", + column_name="interval_col", + column_class_name="Interval", + column_class=Interval, + params={ + "default": IntervalCustom( + weeks=0, + days=0, + hours=0, + minutes=0, + seconds=0, + milliseconds=0, + microseconds=0, + ), + "null": False, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + }, + ) + + manager.add_column( + table_class_name="MegaTable", + tablename="mega_table", + column_name="json_col", + column_class_name="JSON", + column_class=JSON, + params={ + "default": "{}", + "null": False, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + }, + ) + + manager.add_column( + table_class_name="MegaTable", + tablename="mega_table", + column_name="jsonb_col", + column_class_name="JSONB", + column_class=JSONB, + params={ + "default": "{}", + "null": False, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + }, + ) + + manager.add_column( + table_class_name="MegaTable", + tablename="mega_table", + column_name="numeric_col", + column_class_name="Numeric", + column_class=Numeric, + params={ + "default": Decimal("0"), + "digits": (5, 2), + "null": False, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + }, + ) + + manager.add_column( + table_class_name="MegaTable", + tablename="mega_table", + column_name="real_col", + column_class_name="Real", + column_class=Real, + params={ + "default": 0.0, + "null": False, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + }, + ) + + manager.add_column( + table_class_name="MegaTable", + tablename="mega_table", + column_name="double_precision_col", + column_class_name="DoublePrecision", + column_class=DoublePrecision, + params={ + "default": 0.0, + "null": False, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + }, + ) + + manager.add_column( + table_class_name="MegaTable", + tablename="mega_table", + column_name="smallint_col", + column_class_name="SmallInt", + column_class=SmallInt, + params={ + "default": 0, + "null": False, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + }, + ) + + manager.add_column( + table_class_name="MegaTable", + tablename="mega_table", + column_name="text_col", + column_class_name="Text", + column_class=Text, + params={ + "default": "", + "null": False, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + }, + ) + + manager.add_column( + table_class_name="MegaTable", + tablename="mega_table", + column_name="timestamp_col", + column_class_name="Timestamp", + column_class=Timestamp, + params={ + "default": TimestampNow(), + "null": False, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + }, + ) + + manager.add_column( + table_class_name="MegaTable", + tablename="mega_table", + column_name="timestamptz_col", + column_class_name="Timestamptz", + column_class=Timestamptz, + params={ + "default": TimestamptzNow(), + "null": False, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + }, + ) + + manager.add_column( + table_class_name="MegaTable", + tablename="mega_table", + column_name="uuid_col", + column_class_name="UUID", + column_class=UUID, + params={ + "default": UUID4(), + "null": False, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + }, + ) + + manager.add_column( + table_class_name="MegaTable", + tablename="mega_table", + column_name="varchar_col", + column_class_name="Varchar", + column_class=Varchar, + params={ + "length": 255, + "default": "", + "null": False, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + }, + ) + + manager.add_column( + table_class_name="MegaTable", + tablename="mega_table", + column_name="unique_col", + column_class_name="Varchar", + column_class=Varchar, + params={ + "length": 255, + "default": "", + "null": False, + "primary_key": False, + "unique": True, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + }, + ) + + manager.add_column( + table_class_name="MegaTable", + tablename="mega_table", + column_name="null_col", + column_class_name="Varchar", + column_class=Varchar, + params={ + "length": 255, + "default": "", + "null": True, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + }, + ) + + manager.add_column( + table_class_name="MegaTable", + tablename="mega_table", + column_name="not_null_col", + column_class_name="Varchar", + column_class=Varchar, + params={ + "length": 255, + "default": "", + "null": False, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + }, + ) + + manager.add_column( + table_class_name="SmallTable", + tablename="small_table", + column_name="varchar_col", + column_class_name="Varchar", + column_class=Varchar, + params={ + "length": 255, + "default": "", + "null": False, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + }, + ) + + return manager diff --git a/tests/example_apps/mega/tables.py b/tests/example_apps/mega/tables.py index 6ecb17d94..cf35853be 100644 --- a/tests/example_apps/mega/tables.py +++ b/tests/example_apps/mega/tables.py @@ -10,6 +10,7 @@ Boolean, Bytea, Date, + DoublePrecision, ForeignKey, Integer, Interval, @@ -44,6 +45,7 @@ class MegaTable(Table): jsonb_col = JSONB() numeric_col = Numeric(digits=(5, 2)) real_col = Real() + double_precision_col = DoublePrecision() smallint_col = SmallInt() text_col = Text() timestamp_col = Timestamp() From c78cd3bac5bf7921d72bc9c7a19dd5af0f613207 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20Eren=20=C3=96zt=C3=BCrk?= Date: Tue, 21 Sep 2021 00:03:36 +0300 Subject: [PATCH 076/727] AppRegistry provides a warning if an app doesn't end piccolo_app (#253) * AppRegistry provides a warning if an app doesn't end piccolo_app * minor tweaks to docstring Co-authored-by: Daniel Townsend --- piccolo/conf/apps.py | 18 ++++++++++++++++-- tests/conf/test_apps.py | 23 +++++++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/piccolo/conf/apps.py b/piccolo/conf/apps.py index 3464156a2..d0482470d 100644 --- a/piccolo/conf/apps.py +++ b/piccolo/conf/apps.py @@ -182,8 +182,22 @@ def __post_init__(self): app_names = [] for app in self.apps: - app_conf_module = import_module(app) - app_config: AppConfig = getattr(app_conf_module, "APP_CONFIG") + try: + app_conf_module = import_module(app) + app_config: AppConfig = getattr(app_conf_module, "APP_CONFIG") + except (ImportError, AttributeError) as e: + if not app.endswith(".piccolo_app"): + app += ".piccolo_app" + app_conf_module = import_module(app) + app_config: AppConfig = getattr( + app_conf_module, "APP_CONFIG" + ) + colored_warning( + f"App {app[:-12]} should end with `.piccolo_app`", + level=Level.medium, + ) + else: + raise e self.app_configs[app_config.app_name] = app_config app_names.append(app_config.app_name) diff --git a/tests/conf/test_apps.py b/tests/conf/test_apps.py index f613a453c..fdf09d793 100644 --- a/tests/conf/test_apps.py +++ b/tests/conf/test_apps.py @@ -31,6 +31,29 @@ def test_duplicate_app_names(self): ] ) + def test_app_names_not_ending_piccolo_app(self): + """ + Should automatically add `.piccolo_app` to end. + """ + AppRegistry( + apps=[ + "piccolo.apps.user", + ] + ) + + def test_duplicate_app_names_with_auto_changed(self): + """ + Make sure duplicate app names are still detected when `piccolo_app` + is omitted from the end. + """ + with self.assertRaises(ValueError): + AppRegistry( + apps=[ + "piccolo.apps.user.piccolo_app", + "piccolo.apps.user", + ] + ) + def test_get_table_with_name(self): app_registry = AppRegistry(apps=["piccolo.apps.user.piccolo_app"]) table = app_registry.get_table_with_name( From e85fdae10324ae0719d1042aeabbc0a464ac0e2d Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 20 Sep 2021 22:16:02 +0100 Subject: [PATCH 077/727] fix typo in docs for `create` --- docs/src/piccolo/query_types/objects.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/piccolo/query_types/objects.rst b/docs/src/piccolo/query_types/objects.rst index 39b75a0c4..03946d700 100644 --- a/docs/src/piccolo/query_types/objects.rst +++ b/docs/src/piccolo/query_types/objects.rst @@ -61,7 +61,7 @@ This can also be done like this: .. code-block:: python - >>> band.objects().create(name="C-Sharps", popularity=100).run_sync() + >>> band = Band.objects().create(name="C-Sharps", popularity=100).run_sync() ------------------------------------------------------------------------------- From 0f28e07233ea3aa490bbeb0c1b30ada08e4fb2b8 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 20 Sep 2021 22:30:42 +0100 Subject: [PATCH 078/727] bumped version --- CHANGES | 51 +++++++++++++++++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 8692ebf69..f33937f6d 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,57 @@ Changes ======= +0.50.0 +------ +The ``where`` clause can now accept multiple arguments (courtesy @AliSayyah): + +.. code-block:: python + + Concert.select().where( + Concert.venue.name == 'Royal Albert Hall', + Concert.band_1.name == 'Pythonistas' + ).run_sync() + +It's another way of expressing `AND`. It's equivalent to both of these: + +.. code-block:: python + + Concert.select().where( + Concert.venue.name == 'Royal Albert Hall' + ).where( + Concert.band_1.name == 'Pythonistas' + ).run_sync() + + Concert.select().where( + (Concert.venue.name == 'Royal Albert Hall') & (Concert.band_1.name == 'Pythonistas') + ).run_sync() + +Added a ``create`` method, which is an easier way of creating objects (courtesy +@AliSayyah). + +.. code-block:: python + + # This still works: + band = Band(name="C-Sharps", popularity=100) + band.save().run_sync() + + # But now we can do it in a single line using `create`: + band = Band.objects().create(name="C-Sharps", popularity=100).run_sync() + +Fixed a bug with ``piccolo schema generate`` where columns with unrecognised +column types were omitted from the output (courtesy @AliSayyah). + +Added docs for the ``--trace`` argument, which can be used with Piccolo +commands to get a traceback if the command fails (courtesy @hipertracker). + +Added ``DoublePrecision`` column type, which is similar to ``Real`` in that +it stores ``float`` values. However, those values are stored at greater +precision (courtesy @AliSayyah). + +Improved ``AppRegistry``, so if a user only adds the app name (e.g. ``blog``), +instead of ``blog.piccolo_app``, it will now emit a warning, and will try to +import ``blog.piccolo_app`` (courtesy @aliereno). + 0.49.0 ------ Fixed a bug with ``create_pydantic_model`` when used with a ``Decimal`` / diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 197168277..7526dda8d 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.49.0" +__VERSION__ = "0.50.0" From eb9a1f549fc636339a7982c529b589be03e40757 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Tue, 21 Sep 2021 23:04:37 +0200 Subject: [PATCH 079/727] format json (#261) * format json * adding test for format json --- piccolo/utils/pydantic.py | 2 ++ tests/utils/test_pydantic.py | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/piccolo/utils/pydantic.py b/piccolo/utils/pydantic.py index 889c7cf8f..fd7698340 100644 --- a/piccolo/utils/pydantic.py +++ b/piccolo/utils/pydantic.py @@ -172,6 +172,8 @@ def create_pydantic_model( columns[f"{column_name}_readable"] = (str, None) elif isinstance(column, Text): field = pydantic.Field(format="text-area", extra=extra, **params) + elif isinstance(column, (JSON, JSONB)): + field = pydantic.Field(format="json", extra=extra, **params) elif isinstance(column, Secret): field = pydantic.Field(extra={"secret": True, **extra}) else: diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index 5ab834453..9b1cc8ded 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -185,6 +185,17 @@ class Movie(Table): with self.assertRaises(pydantic.ValidationError): pydantic_model(meta=json_string, meta_b=json_string) + def test_json_format(self): + class Movie(Table): + features = JSON() + + pydantic_model = create_pydantic_model(table=Movie) + + self.assertEqual( + pydantic_model.schema()["properties"]["features"]["format"], + "json", + ) + class TestExcludeColumn(TestCase): def test_all(self): From 3d50170ab2250d11cc9c50aa7e9ed2519aa402c8 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 21 Sep 2021 22:56:00 +0100 Subject: [PATCH 080/727] make sure the fixtures load command is registered (#262) --- piccolo/apps/fixtures/piccolo_app.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/piccolo/apps/fixtures/piccolo_app.py b/piccolo/apps/fixtures/piccolo_app.py index 79e7665cc..7468dad89 100644 --- a/piccolo/apps/fixtures/piccolo_app.py +++ b/piccolo/apps/fixtures/piccolo_app.py @@ -1,11 +1,12 @@ from piccolo.conf.apps import AppConfig from .commands.dump import dump +from .commands.load import load APP_CONFIG = AppConfig( app_name="fixtures", migrations_folder_path="", table_classes=[], migration_dependencies=[], - commands=[dump], + commands=[dump, load], ) From 8ec9d10313c9ac44916f01e27c9ef58eb22ae0d9 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 21 Sep 2021 23:01:20 +0100 Subject: [PATCH 081/727] bumped version --- CHANGES | 9 +++++++++ docs/src/piccolo/schema/column_types.rst | 4 ++-- piccolo/__init__.py | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index f33937f6d..3ddee81f1 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,15 @@ Changes ======= +0.51.0 +------ +Modified ``create_pydantic_model``, so ``JSON`` and ``JSONB`` columns have a +``format`` attribute of ``'json'``. This will be used by Piccolo Admin for +improved JSON support. Courtesy @sinisaos. + +Fixing a bug where the ``piccolo fixtures load`` command wasn't registered +with the Piccolo CLI. + 0.50.0 ------ The ``where`` clause can now accept multiple arguments (courtesy @AliSayyah): diff --git a/docs/src/piccolo/schema/column_types.rst b/docs/src/piccolo/schema/column_types.rst index 191331c6b..c1b13cf29 100644 --- a/docs/src/piccolo/schema/column_types.rst +++ b/docs/src/piccolo/schema/column_types.rst @@ -79,9 +79,9 @@ Real ``Real``. -==== +================ Double Precision -==== +================ .. autoclass:: DoublePrecision diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 7526dda8d..c9ffd6c07 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.50.0" +__VERSION__ = "0.51.0" From 0f78151b42aaa9318f1525318d463793f11f7b5e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 25 Sep 2021 17:51:59 +0100 Subject: [PATCH 082/727] Foreign key meta bug (#273) * fix bug with `on_delete` and `on_update` * add test to make sure `on_update` and `on_delete` get assigned properly * add additional test to make sure `ON UPDATE` and `ON DELETE` are set properly in the database * reformatted files --- piccolo/columns/column_types.py | 6 +- tests/columns/test_foreignkey.py | 69 +++++++++++++++++++ .../2021-09-20T21-23-25-698988.py | 46 +++++++------ tests/utils/test_pydantic.py | 2 +- 4 files changed, 96 insertions(+), 27 deletions(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index f26cc1f13..283550d3c 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -1217,11 +1217,9 @@ def __init__( super().__init__(**kwargs) - # This is here just for type inference - the actual value is set by - # the Table metaclass. We can't set the actual value here, as - # only the metaclass has access to the table. + # The Table metaclass sets the actual value for `table`: self._foreign_key_meta = ForeignKeyMeta( - Table, OnDelete.cascade, OnUpdate.cascade + references=Table, on_delete=on_delete, on_update=on_update ) def copy(self) -> ForeignKey: diff --git a/tests/columns/test_foreignkey.py b/tests/columns/test_foreignkey.py index c1708becc..e3e5510a2 100644 --- a/tests/columns/test_foreignkey.py +++ b/tests/columns/test_foreignkey.py @@ -2,7 +2,9 @@ from unittest import TestCase from piccolo.columns import Column, ForeignKey, LazyTableReference, Varchar +from piccolo.columns.base import OnDelete, OnUpdate from piccolo.table import Table +from tests.base import postgres_only from tests.example_apps.music.tables import Band, Concert, Manager, Ticket @@ -32,6 +34,73 @@ class Band4(Table, tablename="band"): manager = ForeignKey(references="tests.columns.test_foreignkey.Manager1") +class Band5(Table): + """ + Contains a ForeignKey with non-default `on_delete` and `on_update` values. + """ + + manager = ForeignKey( + references=Manager, + on_delete=OnDelete.set_null, + on_update=OnUpdate.set_null, + ) + + +class TestForeignKeyMeta(TestCase): + """ + Make sure that `ForeignKeyMeta` is setup correctly. + """ + + def test_foreignkeymeta(self): + self.assertTrue( + Band5.manager._foreign_key_meta.on_update == OnUpdate.set_null + ) + self.assertTrue( + Band5.manager._foreign_key_meta.on_delete == OnDelete.set_null + ) + self.assertTrue(Band.manager._foreign_key_meta.references == Manager) + + +@postgres_only +class TestOnDeleteOnUpdate(TestCase): + """ + Make sure that on_delete, and on_update are correctly applied in the + database. + """ + + def setUp(self): + for table_class in (Manager, Band5): + table_class.create_table().run_sync() + + def tearDown(self): + for table_class in (Band5, Manager): + table_class.alter().drop_table(if_exists=True).run_sync() + + def test_on_delete_on_update(self): + response = Band5.raw( + """ + SELECT + rc.update_rule AS on_update, + rc.delete_rule AS on_delete + FROM information_schema.table_constraints tc + LEFT JOIN information_schema.key_column_usage kcu + ON tc.constraint_catalog = kcu.constraint_catalog + AND tc.constraint_schema = kcu.constraint_schema + AND tc.constraint_name = kcu.constraint_name + LEFT JOIN information_schema.referential_constraints rc + ON tc.constraint_catalog = rc.constraint_catalog + AND tc.constraint_schema = rc.constraint_schema + AND tc.constraint_name = rc.constraint_name + WHERE + lower(tc.constraint_type) in ('foreign key') + AND tc.table_name = 'band5' + AND kcu.column_name = 'manager'; + """ + ).run_sync() + self.assertTrue(response[0]["on_update"] == "SET NULL") + self.assertTrue(response[0]["on_delete"] == "SET NULL") + + class TestForeignKeySelf(TestCase): """ Test that ForeignKey columns can be created with references to the parent diff --git a/tests/example_apps/mega/piccolo_migrations/2021-09-20T21-23-25-698988.py b/tests/example_apps/mega/piccolo_migrations/2021-09-20T21-23-25-698988.py index 83b013c8a..f204dbb7c 100644 --- a/tests/example_apps/mega/piccolo_migrations/2021-09-20T21-23-25-698988.py +++ b/tests/example_apps/mega/piccolo_migrations/2021-09-20T21-23-25-698988.py @@ -1,26 +1,28 @@ -from piccolo.apps.migrations.auto import MigrationManager from decimal import Decimal -from piccolo.columns.base import OnDelete -from piccolo.columns.base import OnUpdate -from piccolo.columns.column_types import BigInt -from piccolo.columns.column_types import Boolean -from piccolo.columns.column_types import Bytea -from piccolo.columns.column_types import Date -from piccolo.columns.column_types import DoublePrecision -from piccolo.columns.column_types import ForeignKey -from piccolo.columns.column_types import Integer -from piccolo.columns.column_types import Interval -from piccolo.columns.column_types import JSON -from piccolo.columns.column_types import JSONB -from piccolo.columns.column_types import Numeric -from piccolo.columns.column_types import Real -from piccolo.columns.column_types import Serial -from piccolo.columns.column_types import SmallInt -from piccolo.columns.column_types import Text -from piccolo.columns.column_types import Timestamp -from piccolo.columns.column_types import Timestamptz -from piccolo.columns.column_types import UUID -from piccolo.columns.column_types import Varchar + +from piccolo.apps.migrations.auto import MigrationManager +from piccolo.columns.base import OnDelete, OnUpdate +from piccolo.columns.column_types import ( + JSON, + JSONB, + UUID, + BigInt, + Boolean, + Bytea, + Date, + DoublePrecision, + ForeignKey, + Integer, + Interval, + Numeric, + Real, + Serial, + SmallInt, + Text, + Timestamp, + Timestamptz, + Varchar, +) from piccolo.columns.defaults.date import DateNow from piccolo.columns.defaults.interval import IntervalCustom from piccolo.columns.defaults.timestamp import TimestampNow diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index 9b1cc8ded..a0339aa39 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -190,7 +190,7 @@ class Movie(Table): features = JSON() pydantic_model = create_pydantic_model(table=Movie) - + self.assertEqual( pydantic_model.schema()["properties"]["features"]["format"], "json", From 1cf5aa7dc5ea19bd3361e0506d8fc769362e5ecd Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 25 Sep 2021 17:54:06 +0100 Subject: [PATCH 083/727] bumped version --- CHANGES | 5 +++++ piccolo/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 3ddee81f1..de50ed46d 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,11 @@ Changes ======= +0.51.1 +------ +Fixing a bug with ``on_delete`` and ``on_update`` not being set correctly. +Thanks to @wmshort for discovering this. + 0.51.0 ------ Modified ``create_pydantic_model``, so ``JSON`` and ``JSONB`` columns have a diff --git a/piccolo/__init__.py b/piccolo/__init__.py index c9ffd6c07..65127698c 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.51.0" +__VERSION__ = "0.51.1" From 5874e5b6f512e210700c1a0a85db87fd76358495 Mon Sep 17 00:00:00 2001 From: ali sayyah Date: Sat, 25 Sep 2021 21:33:42 +0330 Subject: [PATCH 084/727] Better schema generation (#260) * schema generation now will recursivly add refrenced tables even from other schemas. * lint fix * cleanup * added unit test * changes based on discussion * docstring fixes * args to kwargs * added `tablenames` and `exclude` as optional parameters to `get_output_schema` * added docstring * performance improvements to piccolo schema generate * small refactors * fixed minor spelling error * `create_table_class_from_db` now returns `OutputSchema` instead of `Table` * fixed incompatibility issue * ignored lint issue in python 3.7 * docstring tweaks * fix typo in comment Co-authored-by: Daniel Townsend --- piccolo/apps/schema/commands/generate.py | 274 +++++++++++++------- tests/apps/schema/commands/test_generate.py | 80 +++++- 2 files changed, 260 insertions(+), 94 deletions(-) diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index ac01e0796..8495ce321 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import dataclasses import typing as t @@ -42,6 +43,12 @@ class ForeignKeyPlaceholder(Table): pass +@dataclasses.dataclass +class ConstraintTable: + name: str = "" + schema: str = "" + + @dataclasses.dataclass class RowMeta: column_default: str @@ -60,6 +67,7 @@ def get_column_name_str(cls) -> str: class Constraint: constraint_type: Literal["PRIMARY KEY", "UNIQUE", "FOREIGN KEY", "CHECK"] constraint_name: str + constraint_schema: t.Optional[str] = None column_name: t.Optional[str] = None @@ -107,10 +115,12 @@ def is_foreign_key(self, column_name: str) -> bool: return True return False - def get_foreign_key_constraint_name(self, column_name) -> str: + def get_foreign_key_constraint_name(self, column_name) -> ConstraintTable: for i in self.foreign_key_constraints: if i.column_name == column_name: - return i.constraint_name + return ConstraintTable( + name=i.constraint_name, schema=i.constraint_schema + ) raise ValueError("No matching constraint found") @@ -119,31 +129,44 @@ def get_foreign_key_constraint_name(self, column_name) -> str: class OutputSchema: """ Represents the schema which will be printed out. - :param imports: e.g. ["from piccolo.table import Table"] :param warnings: e.g. ["some_table.some_column unrecognised_type"] :param tables: e.g. ["class MyTable(Table): ..."] - """ - imports: t.List[str] - warnings: t.List[str] - tables: t.List[t.Type[Table]] + imports: t.List[str] = dataclasses.field(default_factory=list) + warnings: t.List[str] = dataclasses.field(default_factory=list) + tables: t.List[t.Type[Table]] = dataclasses.field(default_factory=list) - def get_table_with_name(self, name: str) -> t.Optional[t.Type[Table]]: + def get_table_with_name(self, tablename: str) -> t.Optional[t.Type[Table]]: """ - Just used by unit tests. + Used to search for a table by name. """ + tablename = _snake_to_camel(tablename) try: return next( - table for table in self.tables if table.__name__ == name + table for table in self.tables if table.__name__ == tablename ) except StopIteration: return None + def __radd__(self, value: OutputSchema) -> OutputSchema: + if isinstance(value, int): + return self + value.imports.extend(self.imports) + value.warnings.extend(self.warnings) + value.tables.extend(self.tables) + return value + + def __add__(self, value: OutputSchema) -> OutputSchema: + self.imports.extend(value.imports) + self.warnings.extend(value.warnings) + self.tables.extend(value.tables) + return self + COLUMN_TYPE_MAP = { "bigint": BigInt, @@ -166,7 +189,7 @@ def get_table_with_name(self, name: str) -> t.Optional[t.Type[Table]]: } -async def get_contraints( +async def get_constraints( table_class: t.Type[Table], tablename: str, schema_name: str = "public" ) -> TableConstraints: """ @@ -174,11 +197,16 @@ async def get_contraints( :param table_class: Any Table subclass - just used to execute raw queries on the database. + :param tablename: + Name of the table. + :param schema_name: + Name of the schema. + """ constraints = await table_class.raw( ( - "SELECT tc.constraint_name, tc.constraint_type, kcu.column_name " # noqa: E501 + "SELECT tc.constraint_name, tc.constraint_type, kcu.column_name, tc.constraint_schema " # noqa: E501 "FROM information_schema.table_constraints tc " "LEFT JOIN information_schema.key_column_usage kcu " "ON tc.constraint_name = kcu.constraint_name " @@ -202,6 +230,8 @@ async def get_tablenames( :param table_class: Any Table subclass - just used to execute raw queries on the database. + :param schema_name: + Name of the schema. :returns: A list of tablenames for the given schema. @@ -251,28 +281,142 @@ async def get_table_schema( async def get_foreign_key_reference( - table_class: t.Type[Table], constraint_name: str -) -> t.Optional[str]: + table_class: t.Type[Table], constraint_name: str, constraint_schema: str +) -> ConstraintTable: """ Retrieve the name of the table that a foreign key is referencing. """ response = await table_class.raw( ( - "SELECT table_name " + "SELECT table_name, table_schema " "FROM information_schema.constraint_column_usage " - "WHERE constraint_name = {};" + "WHERE constraint_name = {} AND constraint_schema = {};" ), constraint_name, + constraint_schema, ) if len(response) > 0: - return response[0]["table_name"] + return ConstraintTable( + name=response[0]["table_name"], schema=response[0]["table_schema"] + ) else: - return None + return ConstraintTable() + + +def get_table_name(name: str, schema: str) -> str: + if schema == "public": + return name + return f"{schema}.{name}" + + +async def create_table_class_from_db( + table_class: t.Type[Table], tablename: str, schema_name: str +) -> OutputSchema: + constraints = await get_constraints( + table_class=table_class, tablename=tablename, schema_name=schema_name + ) + table_schema = await get_table_schema( + table_class=table_class, tablename=tablename, schema_name=schema_name + ) + output_schema = OutputSchema() + columns: t.Dict[str, Column] = {} + + for pg_row_meta in table_schema: + data_type = pg_row_meta.data_type + column_type = COLUMN_TYPE_MAP.get(data_type, None) + column_name = pg_row_meta.column_name + if not column_type: + output_schema.warnings.append( + f"{tablename}.{column_name} ['{data_type}']" + ) + column_type = Column + + kwargs: t.Dict[str, t.Any] = { + "null": pg_row_meta.is_nullable == "YES", + "unique": constraints.is_unique(column_name=column_name), + } + + if constraints.is_primary_key(column_name=column_name): + kwargs["primary_key"] = True + if column_type == Integer: + column_type = Serial + + if constraints.is_foreign_key(column_name=column_name): + fk_constraint_table = constraints.get_foreign_key_constraint_name( + column_name=column_name + ) + column_type = ForeignKey + constraint_table = await get_foreign_key_reference( + table_class=table_class, + constraint_name=fk_constraint_table.name, + constraint_schema=fk_constraint_table.schema, + ) + if constraint_table.name: + referenced_output_schema = await create_table_class_from_db( + table_class=table_class, + tablename=constraint_table.name, + schema_name=constraint_table.schema, + ) + referenced_table = ( + referenced_output_schema.get_table_with_name( + tablename=constraint_table.name + ) + ) + kwargs["references"] = ( + referenced_table + if referenced_table is not None + else ForeignKeyPlaceholder + ) + output_schema = sum( # type: ignore + [output_schema, referenced_output_schema] # type: ignore + ) # type: ignore + else: + kwargs["references"] = ForeignKeyPlaceholder + + output_schema.imports.append( + "from piccolo.columns.column_types import " + column_type.__name__ + ) + if column_type is Varchar: + kwargs["length"] = pg_row_meta.character_maximum_length -async def get_output_schema(schema_name: str = "public") -> OutputSchema: + column = column_type(**kwargs) + + serialised_params = serialise_params(column._meta.params) + for extra_import in serialised_params.extra_imports: + output_schema.imports.append(extra_import.__repr__()) + + columns[column_name] = column + + table = create_table_class( + class_name=_snake_to_camel(tablename), + class_kwargs={"tablename": get_table_name(tablename, schema_name)}, + class_members=columns, + ) + output_schema.tables.append(table) + return output_schema + + +async def get_output_schema( + schema_name: str = "public", + tablenames: t.Optional[t.List[str]] = None, + exclude: t.Optional[t.List[str]] = None, +) -> OutputSchema: + """ + :param schema_name: + Name of the schema. + :param tablenames: + Optional list of table names. Only creates the specifed tables. + :param exclude: + Optional list of table names. excludes the specified tables. + :returns: + OutputSchema + """ engine: t.Optional[Engine] = engine_finder() + if exclude is None: + exclude = [] + if engine is None: raise ValueError( "Unable to find the engine - make sure piccolo_conf is on the " @@ -291,91 +435,35 @@ class Schema(Table, db=engine): pass - tablenames = await get_tablenames(Schema, schema_name=schema_name) - - tables: t.List[t.Type[Table]] = [] - imports: t.Set[str] = {"from piccolo.table import Table"} - warnings: t.List[str] = [] + if not tablenames: + tablenames = await get_tablenames(Schema, schema_name=schema_name) - for tablename in tablenames: - constraints = await get_contraints( + table_coroutines = ( + create_table_class_from_db( table_class=Schema, tablename=tablename, schema_name=schema_name ) - table_schema = await get_table_schema( - table_class=Schema, tablename=tablename, schema_name=schema_name - ) - - columns: t.Dict[str, Column] = {} - - for pg_row_meta in table_schema: - data_type = pg_row_meta.data_type - column_type = COLUMN_TYPE_MAP.get(data_type, None) - column_name = pg_row_meta.column_name - if not column_type: - warnings.append(f"{tablename}.{column_name} ['{data_type}']") - column_type = Column - - kwargs: t.Dict[str, t.Any] = { - "null": pg_row_meta.is_nullable == "YES", - "unique": constraints.is_unique(column_name=column_name), - } - - if constraints.is_primary_key(column_name=column_name): - kwargs["primary_key"] = True - if column_type == Integer: - column_type = Serial - - if constraints.is_foreign_key(column_name=column_name): - fk_constraint_name = ( - constraints.get_foreign_key_constraint_name( - column_name=column_name - ) - ) - column_type = ForeignKey - referenced_tablename = await get_foreign_key_reference( - table_class=Schema, constraint_name=fk_constraint_name - ) - if referenced_tablename: - kwargs["references"] = create_table_class( - _snake_to_camel(referenced_tablename) - ) - else: - kwargs["references"] = ForeignKeyPlaceholder - - imports.add( - "from piccolo.columns.column_types import " - + column_type.__name__ - ) - - if column_type is Varchar: - kwargs["length"] = pg_row_meta.character_maximum_length - - column = column_type(**kwargs) - - serialised_params = serialise_params(column._meta.params) - for extra_import in serialised_params.extra_imports: - imports.add(extra_import.__repr__()) - - columns[column_name] = column + for tablename in tablenames + if tablename not in exclude + ) + output_schemas = await asyncio.gather(*table_coroutines) - table = create_table_class( - class_name=_snake_to_camel(tablename), - class_kwargs={"tablename": tablename}, - class_members=columns, - ) - tables.append(table) + # Merge all the output schemas to a single OutputSchema object + output_schema: OutputSchema = sum(output_schemas) # type: ignore # Sort the tables based on their ForeignKeys. - tables = sort_table_classes(tables) + output_schema.tables = sort_table_classes( + sorted(output_schema.tables, key=lambda x: x._meta.tablename) + ) + output_schema.imports = sorted(list(set(output_schema.imports))) # We currently don't show the index argument for columns in the output, # so we don't need this import for now: - imports.remove("from piccolo.columns.indexes import IndexMethod") - - return OutputSchema( - imports=sorted(list(imports)), warnings=warnings, tables=tables + output_schema.imports.remove( + "from piccolo.columns.indexes import IndexMethod" ) + return output_schema + # This is currently a beta version, and can be improved. However, having # something working is still useful for people migrating large schemas to diff --git a/tests/apps/schema/commands/test_generate.py b/tests/apps/schema/commands/test_generate.py index 37f4c2b2a..f64497ee0 100644 --- a/tests/apps/schema/commands/test_generate.py +++ b/tests/apps/schema/commands/test_generate.py @@ -11,13 +11,29 @@ get_output_schema, ) from piccolo.columns.base import Column -from piccolo.columns.column_types import Varchar +from piccolo.columns.column_types import ForeignKey, Integer, Varchar +from piccolo.engine import Engine, engine_finder from piccolo.table import Table from piccolo.utils.sync import run_sync from tests.base import postgres_only from tests.example_apps.mega.tables import MegaTable, SmallTable +class Publication(Table, tablename="schema2.publication"): + name = Varchar(length=50) + + +class Writer(Table, tablename="schema1.writer"): + name = Varchar(length=50) + publication = ForeignKey(Publication, null=True) + + +class Book(Table): + name = Varchar(length=50) + writer = ForeignKey(Writer, null=True) + popularity = Integer(default=0) + + @postgres_only class TestGenerate(TestCase): def setUp(self): @@ -108,3 +124,65 @@ class Box(Column): self.assertEqual( output_schema.tables[1].box.__class__.__name__, "Column" ) + + def test_generate_required_tables(self): + """ + Make sure only tables passed to `tablenames` are created + """ + output_schema: OutputSchema = run_sync( + get_output_schema(tablenames=[SmallTable._meta.tablename]) + ) + self.assertEqual(len(output_schema.tables), 1) + SmallTable_ = output_schema.get_table_with_name("SmallTable") + self._compare_table_columns(SmallTable, SmallTable_) + + def test_exclude_table(self): + """ + make sure exclude works + """ + output_schema: OutputSchema = run_sync( + get_output_schema(exclude=[MegaTable._meta.tablename]) + ) + self.assertEqual(len(output_schema.tables), 1) + SmallTable_ = output_schema.get_table_with_name("SmallTable") + self._compare_table_columns(SmallTable, SmallTable_) + + +@postgres_only +class TestGenerateWithSchema(TestCase): + def setUp(self) -> None: + engine: t.Optional[Engine] = engine_finder() + + class Schema(Table, db=engine): + """ + Only for raw query execution + """ + + pass + + Schema.raw("CREATE SCHEMA IF NOT EXISTS schema1").run_sync() + Schema.raw("CREATE SCHEMA IF NOT EXISTS schema2").run_sync() + Publication.create_table().run_sync() + Writer.create_table().run_sync() + Book.create_table().run_sync() + + def tearDown(self) -> None: + Book.alter().drop_table().run_sync() + Writer.alter().drop_table().run_sync() + Publication.alter().drop_table().run_sync() + + def test_reference_to_another_schema(self): + output_schema: OutputSchema = run_sync(get_output_schema()) + self.assertEqual(len(output_schema.tables), 3) + publication = output_schema.tables[0] + writer = output_schema.tables[1] + book = output_schema.tables[2] + # Make sure referenced tables have been created + self.assertEqual( + Publication._meta.tablename, publication._meta.tablename + ) + self.assertEqual(Writer._meta.tablename, writer._meta.tablename) + + # Make sure foreign key values are correct. + self.assertEqual(writer.publication, publication) + self.assertEqual(book.writer, writer) From 9445fe14dca7ec61452a46304061a6e0eef9d53f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20Eren=20=C3=96zt=C3=BCrk?= Date: Sat, 25 Sep 2021 21:49:34 +0300 Subject: [PATCH 085/727] Add `bigserial` type (#271) * add bigserial type * tweaked tests and docs Co-authored-by: Daniel Townsend --- docs/src/piccolo/schema/column_types.rst | 21 ++++++++++--- piccolo/columns/__init__.py | 1 + piccolo/columns/column_types.py | 15 +++++++++ tests/columns/test_primary_key.py | 39 +++++++++++++++++++++--- 4 files changed, 66 insertions(+), 10 deletions(-) diff --git a/docs/src/piccolo/schema/column_types.rst b/docs/src/piccolo/schema/column_types.rst index c1b13cf29..4b86b01ec 100644 --- a/docs/src/piccolo/schema/column_types.rst +++ b/docs/src/piccolo/schema/column_types.rst @@ -54,6 +54,18 @@ BigInt .. autoclass:: BigInt +========= +BigSerial +========= + +.. autoclass:: BigSerial + +================ +Double Precision +================ + +.. autoclass:: DoublePrecision + ======= Integer ======= @@ -78,12 +90,11 @@ Real .. hint:: There is also a ``Float`` column type, which is an alias for ``Real``. +====== +Serial +====== -================ -Double Precision -================ - -.. autoclass:: DoublePrecision +.. autoclass:: Serial ======== SmallInt diff --git a/piccolo/columns/__init__.py b/piccolo/columns/__init__.py index 83c0289ec..644b685ac 100644 --- a/piccolo/columns/__init__.py +++ b/piccolo/columns/__init__.py @@ -5,6 +5,7 @@ UUID, Array, BigInt, + BigSerial, Boolean, Bytea, Date, diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 283550d3c..17340e1ed 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -522,6 +522,21 @@ def default(self): raise Exception("Unrecognized engine type") +class BigSerial(Serial): + """ + An alias to a large autoincrementing integer column in Postgres. + """ + + @property + def column_type(self): + engine_type = self._meta.table._meta.db.engine_type + if engine_type == "postgres": + return "BIGSERIAL" + elif engine_type == "sqlite": + return "INTEGER" + raise Exception("Unrecognized engine type") + + class PrimaryKey(Serial): def __init__(self, **kwargs) -> None: # Set the index to False, as a database should automatically create diff --git a/tests/columns/test_primary_key.py b/tests/columns/test_primary_key.py index 42629a495..30ba19ccc 100644 --- a/tests/columns/test_primary_key.py +++ b/tests/columns/test_primary_key.py @@ -1,7 +1,13 @@ import uuid from unittest import TestCase -from piccolo.columns.column_types import UUID, ForeignKey, Serial, Varchar +from piccolo.columns.column_types import ( + UUID, + BigSerial, + ForeignKey, + Serial, + Varchar, +) from piccolo.table import Table @@ -14,8 +20,13 @@ class MyTablePrimaryKeySerial(Table): name = Varchar() +class MyTablePrimaryKeyBigSerial(Table): + pk = BigSerial(null=False, primary_key=True) + name = Varchar() + + class MyTablePrimaryKeyUUID(Table): - id = UUID(null=False, primary_key=True) + pk = UUID(null=False, primary_key=True) name = Varchar() @@ -31,6 +42,7 @@ def test_return_type(self): row.save().run_sync() self.assertIsInstance(row._meta.primary_key, Serial) + self.assertIsInstance(row["id"], int) class TestPrimaryKeyInteger(TestCase): @@ -42,9 +54,25 @@ def tearDown(self): def test_return_type(self): row = MyTablePrimaryKeySerial() - result = row.save().run_sync()[0] + row.save().run_sync() + + self.assertIsInstance(row._meta.primary_key, Serial) + self.assertIsInstance(row["pk"], int) + + +class TestPrimaryKeyBigSerial(TestCase): + def setUp(self): + MyTablePrimaryKeyBigSerial.create_table().run_sync() + + def tearDown(self): + MyTablePrimaryKeyBigSerial.alter().drop_table().run_sync() + + def test_return_type(self): + row = MyTablePrimaryKeyBigSerial() + row.save().run_sync() - self.assertIsInstance(result["pk"], int) + self.assertIsInstance(row._meta.primary_key, BigSerial) + self.assertIsInstance(row["pk"], int) class TestPrimaryKeyUUID(TestCase): @@ -58,7 +86,8 @@ def test_return_type(self): row = MyTablePrimaryKeyUUID() row.save().run_sync() - self.assertIsInstance(row.id, uuid.UUID) + self.assertIsInstance(row._meta.primary_key, UUID) + self.assertIsInstance(row["pk"], uuid.UUID) class Manager(Table): From aa3a70be58e9855f98a4dce27303c50e4b1078c1 Mon Sep 17 00:00:00 2001 From: Abhijith Ganesh <67182544+AbhijithGanesh@users.noreply.github.com> Date: Sun, 26 Sep 2021 00:58:22 +0530 Subject: [PATCH 086/727] Addition of templates (#267) * Addition of templates * Correction for labels * Correction for labels * simplify templates for v1 Co-authored-by: Daniel Townsend --- .github/ISSUE_TEMPLATES/features.yml | 22 ++++++++++++++++++++++ .github/ISSUE_TEMPLATES/issues.yml | 22 ++++++++++++++++++++++ .github/ISSUE_TEMPLATES/support.yml | 16 ++++++++++++++++ 3 files changed, 60 insertions(+) create mode 100644 .github/ISSUE_TEMPLATES/features.yml create mode 100644 .github/ISSUE_TEMPLATES/issues.yml create mode 100644 .github/ISSUE_TEMPLATES/support.yml diff --git a/.github/ISSUE_TEMPLATES/features.yml b/.github/ISSUE_TEMPLATES/features.yml new file mode 100644 index 000000000..143c336c3 --- /dev/null +++ b/.github/ISSUE_TEMPLATES/features.yml @@ -0,0 +1,22 @@ +name: Features 💡 +description: Suggest a new feature +title: "[Feature]: Describe your feature idea here" +labels: ["goal:addition"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this suggestion form! + - type: input + attributes: + label: Suggestion + description: Can you please elaborate on your suggestion? + placeholder: I would like to see... + validations: + required: true + - type: textarea + id: multimedia + attributes: + label: Relevant media + description: You can add any media which helps explain your idea. + placeholder: Please upload your multimedia (screenshots and video) here. diff --git a/.github/ISSUE_TEMPLATES/issues.yml b/.github/ISSUE_TEMPLATES/issues.yml new file mode 100644 index 000000000..81e8145b8 --- /dev/null +++ b/.github/ISSUE_TEMPLATES/issues.yml @@ -0,0 +1,22 @@ +name: Bug 🐛 +description: Report a bug +title: "[Issue]: Describe the bug here" +labels: ["goal:fix, priority:medium"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report! + - type: input + attributes: + label: Bug description + description: Can you please describe the bug in more detail? + placeholder: Tell us what you found. + validations: + required: true + - type: textarea + id: multimedia + attributes: + label: Relevant log output + description: Please paste any relevant log output. + placeholder: Please upload your multimedia (screenshots and video) here. diff --git a/.github/ISSUE_TEMPLATES/support.yml b/.github/ISSUE_TEMPLATES/support.yml new file mode 100644 index 000000000..4cecf39b5 --- /dev/null +++ b/.github/ISSUE_TEMPLATES/support.yml @@ -0,0 +1,16 @@ +name: Support 💁‍♂️ +description: Raise a support ticket +title: "[Support]: Describe your issue / support requirement here" +labels: [support] +body: + - type: markdown + attributes: + value: | + Thanks for reaching out! + - type: input + attributes: + label: Support ticket information + description: Please elaborate on your support requirement. + placeholder: I would like to see... + validations: + required: true From dab4f9d641642b1dbc49bd5029400ccc6995a945 Mon Sep 17 00:00:00 2001 From: William Michael Short <36488354+wmshort@users.noreply.github.com> Date: Sat, 25 Sep 2021 22:35:57 +0100 Subject: [PATCH 087/727] Add defaults in automatic schema generation (#263) * initial prototype of defaults generation * update example tables for testing defaults generation * clean up imports * fix date default parsing * add double precision * linting * fix BigInt and SmallInt parsing * add interval parsing * linting * code clean up * code clean up * type fix * oops * clean up * Update generate.py Co-authored-by: William Michael Short --- piccolo/apps/schema/commands/generate.py | 140 ++++++++++++++++++++++- tests/example_apps/music/tables.py | 24 +++- 2 files changed, 161 insertions(+), 3 deletions(-) diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index 8495ce321..7a5b3429c 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -2,12 +2,17 @@ import asyncio import dataclasses +import json +import re import typing as t +import uuid +from datetime import date, datetime import black from typing_extensions import Literal from piccolo.apps.migrations.auto.serialisation import serialise_params +from piccolo.columns import defaults from piccolo.columns.base import Column from piccolo.columns.column_types import ( JSON, @@ -30,6 +35,7 @@ Timestamptz, Varchar, ) +from piccolo.columns.defaults.interval import IntervalCustom from piccolo.engine.finder import engine_finder from piccolo.engine.postgres import PostgresEngine from piccolo.table import Table, create_table_class, sort_table_classes @@ -168,7 +174,7 @@ def __add__(self, value: OutputSchema) -> OutputSchema: return self -COLUMN_TYPE_MAP = { +COLUMN_TYPE_MAP: t.Dict[str, t.Type[Column]] = { "bigint": BigInt, "boolean": Boolean, "bytea": Bytea, @@ -188,6 +194,132 @@ def __add__(self, value: OutputSchema) -> OutputSchema: "uuid": UUID, } +COLUMN_DEFAULT_PARSER = { + BigInt: re.compile(r"^'?(?P-?[0-9]\d*)'?(?:::bigint)?$"), + Boolean: re.compile(r"^(?Ptrue|false)$"), + Bytea: re.compile(r"'(?P.*)'::bytea$"), + DoublePrecision: re.compile(r"(?P[+-]?(?:[0-9]*[.])?[0-9]+)"), + Varchar: re.compile(r"^'(?P.*)'::character varying$"), + Date: re.compile(r"^(?P(?:\d{4}-\d{2}-\d{2})|CURRENT_DATE)$"), + Integer: re.compile(r"^(?P-?\d+)$"), + Interval: re.compile( + r"""^ + (?:')? + (?: + (?:(?P\d+)[ ]y(?:ear(?:s)?)?\b) | + (?:(?P\d+)[ ]m(?:onth(?:s)?)?\b) | + (?:(?P\d+)[ ]w(?:eek(?:s)?)?\b) | + (?:(?P\d+)[ ]d(?:ay(?:s)?)?\b) | + (?: + (?: + (?:(?P\d+)[ ]h(?:our(?:s)?)?\b) | + (?:(?P\d+)[ ]m(?:inute(?:s)?)?\b) | + (?:(?P\d+)[ ]s(?:econd(?:s)?)?\b) + ) | + (?: + (?P-?\d{2}:\d{2}:\d{2}))?\b) + ) + +(?Pago)? + (?:'::interval)? + $""", + re.X, + ), + JSON: re.compile(r"^'(?P.*)'::json$"), + JSONB: re.compile(r"^'(?P.*)'::jsonb$"), + Numeric: re.compile(r"(?P\d+)"), + Real: re.compile(r"^(?P-?[0-9]\d*(?:\.\d+)?)$"), + SmallInt: re.compile(r"^'?(?P-?[0-9]\d*)'?(?:::integer)?$"), + Text: re.compile(r"^'(?P.*)'::text$"), + Timestamp: re.compile( + r"""^ + (?P + (?:\d{4}-\d{2}-\d{2}[ ]\d{2}:\d{2}:\d{2}) | + CURRENT_TIMESTAMP + ) + $""", + re.VERBOSE, + ), + Timestamptz: re.compile( + r"""^ + (?P + (?:\d{4}-\d{2}-\d{2}[ ]\d{2}:\d{2}:\d{2}(?:\.\d+)?-\d{2}) | + CURRENT_TIMESTAMP + ) + $""", + re.X, + ), + UUID: None, + Serial: None, + ForeignKey: None, +} + + +def get_column_default( + column_type: t.Type[Column], column_default: str +) -> t.Any: + pat = COLUMN_DEFAULT_PARSER[column_type] + + if pat is not None: + match = re.match(pat, column_default) + if match is not None: + value = match.groupdict() + + if column_type is Boolean: + return True if value["value"] == "true" else False + elif column_type is Interval: + kwargs = {} + for period in [ + "years", + "months", + "weeks", + "days", + "hours", + "minutes", + "seconds", + ]: + period_match = value.get(period, 0) + if period_match: + kwargs[period] = int(period_match) + digits = value["digits"] + if digits: + kwargs.update( + dict( + zip( + ["hours", "minutes", "seconds"], + [int(v) for v in value["digits"].split(":")], + ) + ) + ) + return IntervalCustom(**kwargs) + elif column_type is JSON or column_type is JSONB: + return json.loads(value["value"]) + elif column_type is UUID: + return uuid.uuid4 + elif column_type is Date: + return ( + date.today + if value["value"] == "CURRENT_DATE" + else defaults.date.DateCustom( + *[int(v) for v in value["value"].split("-")] + ) + ) + elif column_type is Bytea: + return value["value"].encode("utf8") + elif column_type is Timestamp: + return ( + datetime.now + if value["value"] == "CURRENT_TIMESTAMP" + else datetime.fromtimestamp(float(value["value"])) + ) + elif column_type is Timestamptz: + return ( + datetime.now + if value["value"] == "CURRENT_TIMESTAMP" + else datetime.fromtimestamp(float(value["value"])) + ) + else: + return column_type.value_type(value["value"]) + async def get_constraints( table_class: t.Type[Table], tablename: str, schema_name: str = "public" @@ -325,6 +457,7 @@ async def create_table_class_from_db( data_type = pg_row_meta.data_type column_type = COLUMN_TYPE_MAP.get(data_type, None) column_name = pg_row_meta.column_name + column_default = pg_row_meta.column_default if not column_type: output_schema.warnings.append( f"{tablename}.{column_name} ['{data_type}']" @@ -380,6 +513,11 @@ async def create_table_class_from_db( if column_type is Varchar: kwargs["length"] = pg_row_meta.character_maximum_length + if column_default: + default_value = get_column_default(column_type, column_default) + if default_value: + kwargs["default"] = default_value + column = column_type(**kwargs) serialised_params = serialise_params(column._meta.params) diff --git a/tests/example_apps/music/tables.py b/tests/example_apps/music/tables.py index 331fab081..49488fae2 100644 --- a/tests/example_apps/music/tables.py +++ b/tests/example_apps/music/tables.py @@ -1,13 +1,23 @@ +from datetime import timedelta from enum import Enum from piccolo.columns import ( JSON, JSONB, + UUID, + BigInt, + Boolean, + Bytea, + Date, ForeignKey, Integer, + Interval, Numeric, + SmallInt, Text, - Varchar, + Timestamp, + Timestamptz, + Varchar ) from piccolo.columns.readable import Readable from piccolo.table import Table @@ -18,6 +28,7 @@ class Manager(Table): name = Varchar(length=50) + touring = Boolean(default=False) @classmethod def get_readable(cls) -> Readable: @@ -25,6 +36,8 @@ def get_readable(cls) -> Readable: class Band(Table): + label_id = UUID() + date_signed = Date() name = Varchar(length=50) manager = ForeignKey(Manager, null=True) popularity = Integer(default=0) @@ -44,10 +57,15 @@ class Concert(Table): band_2 = ForeignKey(Band) venue = ForeignKey(Venue) + duration = Interval(default=timedelta(weeks=5, days=3)) + net_profit = SmallInt(default=-32768) + class Ticket(Table): concert = ForeignKey(Concert) price = Numeric(digits=(5, 2)) + purchase_time = Timestamp() + purchase_time_tz = Timestamptz() class Poster(Table, tags=["special"]): @@ -55,6 +73,7 @@ class Poster(Table, tags=["special"]): Has tags for tests which need it. """ + image = Bytea(default=b"\xbd\x78\xd8") content = Text() @@ -76,5 +95,6 @@ class RecordingStudio(Table): Used for testing JSON and JSONB columns. """ - facilities = JSON() + facilities = JSON(default={"amplifier": False, "microphone": True}) facilities_b = JSONB() + records = BigInt(default=9223372036854775807) From c86170721976976e5477e48734986d8129e9d5c4 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 25 Sep 2021 22:40:17 +0100 Subject: [PATCH 088/727] split out new tables for now --- tests/example_apps/music/tables.py | 24 +---- tests/example_apps/music/tables_detailed.py | 104 ++++++++++++++++++++ 2 files changed, 106 insertions(+), 22 deletions(-) create mode 100644 tests/example_apps/music/tables_detailed.py diff --git a/tests/example_apps/music/tables.py b/tests/example_apps/music/tables.py index 49488fae2..331fab081 100644 --- a/tests/example_apps/music/tables.py +++ b/tests/example_apps/music/tables.py @@ -1,23 +1,13 @@ -from datetime import timedelta from enum import Enum from piccolo.columns import ( JSON, JSONB, - UUID, - BigInt, - Boolean, - Bytea, - Date, ForeignKey, Integer, - Interval, Numeric, - SmallInt, Text, - Timestamp, - Timestamptz, - Varchar + Varchar, ) from piccolo.columns.readable import Readable from piccolo.table import Table @@ -28,7 +18,6 @@ class Manager(Table): name = Varchar(length=50) - touring = Boolean(default=False) @classmethod def get_readable(cls) -> Readable: @@ -36,8 +25,6 @@ def get_readable(cls) -> Readable: class Band(Table): - label_id = UUID() - date_signed = Date() name = Varchar(length=50) manager = ForeignKey(Manager, null=True) popularity = Integer(default=0) @@ -57,15 +44,10 @@ class Concert(Table): band_2 = ForeignKey(Band) venue = ForeignKey(Venue) - duration = Interval(default=timedelta(weeks=5, days=3)) - net_profit = SmallInt(default=-32768) - class Ticket(Table): concert = ForeignKey(Concert) price = Numeric(digits=(5, 2)) - purchase_time = Timestamp() - purchase_time_tz = Timestamptz() class Poster(Table, tags=["special"]): @@ -73,7 +55,6 @@ class Poster(Table, tags=["special"]): Has tags for tests which need it. """ - image = Bytea(default=b"\xbd\x78\xd8") content = Text() @@ -95,6 +76,5 @@ class RecordingStudio(Table): Used for testing JSON and JSONB columns. """ - facilities = JSON(default={"amplifier": False, "microphone": True}) + facilities = JSON() facilities_b = JSONB() - records = BigInt(default=9223372036854775807) diff --git a/tests/example_apps/music/tables_detailed.py b/tests/example_apps/music/tables_detailed.py new file mode 100644 index 000000000..f28692396 --- /dev/null +++ b/tests/example_apps/music/tables_detailed.py @@ -0,0 +1,104 @@ +# TODO - these are much better example tables than in tables.py, but many +# tests will break if we change them. In the future migrate this file to +# tables.py and fix the tests. + +from datetime import timedelta +from enum import Enum + +from piccolo.columns import ( + JSON, + JSONB, + UUID, + BigInt, + Boolean, + Bytea, + Date, + ForeignKey, + Integer, + Interval, + Numeric, + SmallInt, + Text, + Timestamp, + Timestamptz, + Varchar, +) +from piccolo.columns.readable import Readable +from piccolo.table import Table + +############################################################################### +# Simple example + + +class Manager(Table): + name = Varchar(length=50) + touring = Boolean(default=False) + + @classmethod + def get_readable(cls) -> Readable: + return Readable(template="%s", columns=[cls.name]) + + +class Band(Table): + label_id = UUID() + date_signed = Date() + name = Varchar(length=50) + manager = ForeignKey(Manager, null=True) + popularity = Integer(default=0) + + +############################################################################### +# More complex + + +class Venue(Table): + name = Varchar(length=100) + capacity = Integer(default=0) + + +class Concert(Table): + band_1 = ForeignKey(Band) + band_2 = ForeignKey(Band) + venue = ForeignKey(Venue) + + duration = Interval(default=timedelta(weeks=5, days=3)) + net_profit = SmallInt(default=-32768) + + +class Ticket(Table): + concert = ForeignKey(Concert) + price = Numeric(digits=(5, 2)) + purchase_time = Timestamp() + purchase_time_tz = Timestamptz() + + +class Poster(Table, tags=["special"]): + """ + Has tags for tests which need it. + """ + + image = Bytea(default=b"\xbd\x78\xd8") + content = Text() + + +class Shirt(Table): + """ + Used for testing columns with a choices attribute. + """ + + class Size(str, Enum): + small = "s" + medium = "m" + large = "l" + + size = Varchar(length=1, choices=Size, default=Size.large) + + +class RecordingStudio(Table): + """ + Used for testing JSON and JSONB columns. + """ + + facilities = JSON(default={"amplifier": False, "microphone": True}) + facilities_b = JSONB() + records = BigInt(default=9223372036854775807) From 88bdf94868411769c16ef602f028fb1df419775b Mon Sep 17 00:00:00 2001 From: William Michael Short Date: Sat, 25 Sep 2021 22:20:05 +0000 Subject: [PATCH 089/727] capture decimal precision from schema --- piccolo/apps/schema/commands/generate.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index 7a5b3429c..8d20747a4 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -63,7 +63,10 @@ class RowMeta: table_name: str character_maximum_length: t.Optional[int] data_type: str - + numeric_precision: t.Optional[t.Union[int, str]] + numeric_scale: t.Optional[t.Union[int, str]] + numeric_precision_radix: t.Optional[Literal[2, 10]] + @classmethod def get_column_name_str(cls) -> str: return ", ".join([i.name for i in dataclasses.fields(cls)]) @@ -512,6 +515,11 @@ async def create_table_class_from_db( if column_type is Varchar: kwargs["length"] = pg_row_meta.character_maximum_length + elif isinstance(column_type, Numeric): + radix = pg_row_meta.numeric_precision_radix + precision = int(str(pg_row_meta.numeric_precision), radix) + scale = int(str(pg_row_meta.numeric_scale), radix) + kwargs["digits"] = (precision, scale) if column_default: default_value = get_column_default(column_type, column_default) From 33754ed6ea1abfd3c0fda71a3d4576cf4f52bc08 Mon Sep 17 00:00:00 2001 From: William Michael Short Date: Sat, 25 Sep 2021 22:35:41 +0000 Subject: [PATCH 090/727] reflect ON DELETE/UPDATE triggers in schema --- piccolo/apps/schema/commands/generate.py | 107 ++++++++++++++++++++++- 1 file changed, 106 insertions(+), 1 deletion(-) diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index 7a5b3429c..496bb251f 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -13,7 +13,7 @@ from piccolo.apps.migrations.auto.serialisation import serialise_params from piccolo.columns import defaults -from piccolo.columns.base import Column +from piccolo.columns.base import Column, OnDelete, OnUpdate from piccolo.columns.column_types import ( JSON, JSONB, @@ -131,6 +131,39 @@ def get_foreign_key_constraint_name(self, column_name) -> ConstraintTable: raise ValueError("No matching constraint found") +@dataclasses.dataclass +class Trigger: + constraint_name: str + constraint_type: str + table_name: str + column_name: str + on_update: str + on_delete: Literal["NO ACTION", "RESTRICT", "CASCADE", "SET NULL", "SET_DEFAULT"] + references_table: str + references_column: str + +@dataclasses.dataclass +class TableTriggers: + """ + All of the triggers for a certain table in the database. + """ + + tablename: str + triggers: t.List[Trigger] + + def get_column_triggers(self, column_name) -> List[Trigger]: + triggers = [] + for i in self.triggers: + if i.column_name == column_name: + triggers.append(i) + return triggers + + def get_column_ref_trigger(self, column_name, references_table) -> Trigger: + for i in self.triggers: + if i.column_name == column_name and i.references_table == references_table: + return i + + @dataclasses.dataclass class OutputSchema: """ @@ -321,6 +354,69 @@ def get_column_default( return column_type.value_type(value["value"]) +async def get_fk_triggers( + table_class: t.Type[Table], tablename: str, schema_name: str = "public" +) -> TableTriggers: + """ + Get all of the constraints for a table. + + :param table_class: + Any Table subclass - just used to execute raw queries on the database. + + """ + triggers = await table_class.raw( + ( + "SELECT tc.constraint_name, " + " tc.constraint_type, " + " tc.table_name, " + " kcu.column_name, " + " rc.update_rule AS on_update, " + " rc.delete_rule AS on_delete, " + " ccu.table_name AS references_table, " + " ccu.column_name AS references_column " + "FROM information_schema.table_constraints tc " + "LEFT JOIN information_schema.key_column_usage kcu " + " ON tc.constraint_catalog = kcu.constraint_catalog " + " AND tc.constraint_schema = kcu.constraint_schema " + " AND tc.constraint_name = kcu.constraint_name " + "LEFT JOIN information_schema.referential_constraints rc " + " ON tc.constraint_catalog = rc.constraint_catalog " + " AND tc.constraint_schema = rc.constraint_schema " + " AND tc.constraint_name = rc.constraint_name " + "LEFT JOIN information_schema.constraint_column_usage ccu " + " ON rc.unique_constraint_catalog = ccu.constraint_catalog " + " AND rc.unique_constraint_schema = ccu.constraint_schema " + " AND rc.unique_constraint_name = ccu.constraint_name " + "WHERE lower(tc.constraint_type) in ('foreign key')" + "AND tc.table_schema = {} " + "AND tc.table_name = {}; " + ), + schema_name, + tablename, + ) + return TableTriggers( + tablename=tablename, + triggers=[Trigger(**i) for i in triggers], + ) + + +ONDELETE_MAP = { + "NO ACTION": OnDelete.no_action, + "RESTRICT": OnDelete.restrict, + "CASCADE": OnDelete.cascade, + "SET NULL": OnDelete.set_null, + "SET DEFAULT": OnDelete.set_default +} + +ONUPDATE_MAP = { + "NO ACTION": OnUpdate.no_action, + "RESTRICT": OnUpdate.restrict, + "CASCADE": OnUpdate.cascade, + "SET NULL": OnUpdate.set_null, + "SET DEFAULT": OnUpdate.set_default +} + + async def get_constraints( table_class: t.Type[Table], tablename: str, schema_name: str = "public" ) -> TableConstraints: @@ -447,6 +543,9 @@ async def create_table_class_from_db( constraints = await get_constraints( table_class=table_class, tablename=tablename, schema_name=schema_name ) + triggers = await get_fk_triggers( + table_class=table_class, tablename=tablename, schema_name=schema_name + ) table_schema = await get_table_schema( table_class=table_class, tablename=tablename, schema_name=schema_name ) @@ -500,6 +599,12 @@ async def create_table_class_from_db( if referenced_table is not None else ForeignKeyPlaceholder ) + + trigger = triggers.get_column_ref_trigger(column_name, referenced_tablename) + if trigger: + kwargs["on_update"] = ONUPDATE_MAP[trigger.on_update] + kwargs["on_delete"] = ONDELETE_MAP[trigger.on_delete] + output_schema = sum( # type: ignore [output_schema, referenced_output_schema] # type: ignore ) # type: ignore From 8fcc8a8dd59f27df442fea692fae36351a7928fb Mon Sep 17 00:00:00 2001 From: William Michael Short Date: Sun, 26 Sep 2021 08:13:23 +0000 Subject: [PATCH 091/727] linting fixes --- piccolo/apps/schema/commands/generate.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index 496bb251f..0ff6a2cd5 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -151,7 +151,7 @@ class TableTriggers: tablename: str triggers: t.List[Trigger] - def get_column_triggers(self, column_name) -> List[Trigger]: + def get_column_triggers(self, column_name) -> t.List[Trigger]: triggers = [] for i in self.triggers: if i.column_name == column_name: @@ -163,6 +163,8 @@ def get_column_ref_trigger(self, column_name, references_table) -> Trigger: if i.column_name == column_name and i.references_table == references_table: return i + raise ValueError("No matching trigger found") + @dataclasses.dataclass class OutputSchema: @@ -600,7 +602,7 @@ async def create_table_class_from_db( else ForeignKeyPlaceholder ) - trigger = triggers.get_column_ref_trigger(column_name, referenced_tablename) + trigger = triggers.get_column_ref_trigger(column_name, constraint_table.name) if trigger: kwargs["on_update"] = ONUPDATE_MAP[trigger.on_update] kwargs["on_delete"] = ONDELETE_MAP[trigger.on_delete] From cb2a6c8fd14f2f45c742dd2dfa55aa54b9063267 Mon Sep 17 00:00:00 2001 From: William Michael Short <36488354+wmshort@users.noreply.github.com> Date: Sun, 26 Sep 2021 22:11:45 +0100 Subject: [PATCH 092/727] Update generate.py --- piccolo/apps/schema/commands/generate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index 0ff6a2cd5..c780c0a72 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -151,14 +151,14 @@ class TableTriggers: tablename: str triggers: t.List[Trigger] - def get_column_triggers(self, column_name) -> t.List[Trigger]: + def get_column_triggers(self, column_name: str) -> t.List[Trigger]: triggers = [] for i in self.triggers: if i.column_name == column_name: triggers.append(i) return triggers - def get_column_ref_trigger(self, column_name, references_table) -> Trigger: + def get_column_ref_trigger(self, column_name: str, references_table: str) -> Trigger: for i in self.triggers: if i.column_name == column_name and i.references_table == references_table: return i From 6025b0d10d4a4e19dc794e7f128d7f2db30b76d7 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 26 Sep 2021 22:45:07 +0100 Subject: [PATCH 093/727] bumped version --- CHANGES | 18 ++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index de50ed46d..d6587563a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,24 @@ Changes ======= +0.52.0 +------ +Lots of improvements to ``piccolo schema generate``: + + * Dramatically improved performance, by executing more queries in parallel + (courtesy @AliSayyah). + * If a table in the database has a foreign key to a table in another + schema, this will now work (courtesy @AliSayyah). + * The column defaults are now extracted from the database (courtesy @wmshort). + * The ``scale`` and ``precision`` values for ``Numeric`` / ``Decimal`` column + types are extracted from the datbase (courtesy @wmshort). + * The ``ON DELETE`` and ``ON UPDATE`` values for ``ForeignKey`` columns are + now extracted from the database (courtesy @wmshort). + +Added ``BigSerial`` column type (courtesy @aliereno). + +Added GitHub issue templates (courtesy @AbhijithGanesh). + 0.51.1 ------ Fixing a bug with ``on_delete`` and ``on_update`` not being set correctly. diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 65127698c..a05d6fb4e 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.51.1" +__VERSION__ = "0.52.0" From 2bd60cfbac22a9091285ed7bde9eb4d9e11f46ba Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 26 Sep 2021 22:47:46 +0100 Subject: [PATCH 094/727] fix typo --- CHANGES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index d6587563a..a502ceb6a 100644 --- a/CHANGES +++ b/CHANGES @@ -11,7 +11,7 @@ Lots of improvements to ``piccolo schema generate``: schema, this will now work (courtesy @AliSayyah). * The column defaults are now extracted from the database (courtesy @wmshort). * The ``scale`` and ``precision`` values for ``Numeric`` / ``Decimal`` column - types are extracted from the datbase (courtesy @wmshort). + types are extracted from the database (courtesy @wmshort). * The ``ON DELETE`` and ``ON UPDATE`` values for ``ForeignKey`` columns are now extracted from the database (courtesy @wmshort). From 935c491ed069c9b336b51e9413030d0428b70c7a Mon Sep 17 00:00:00 2001 From: Yasser Tahiri Date: Mon, 27 Sep 2021 17:58:10 +0100 Subject: [PATCH 095/727] =?UTF-8?q?Inline=20Variables=20that=20immediately?= =?UTF-8?q?=20returned=20&=20Refactor=20Multipart=20of=20Codes=20=E2=9C=A8?= =?UTF-8?q?=20(#279)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐍: Refactor app Folder and Fix Code Issue * ⚡️: Fix Engine Code * 🌪: Fix Issue relate to Return & Refactor the Code * ❄️: Refactor piccolo configuration * ☀️: Refactor Utils & Fix Code Issues * 🪶: Fix Queries Code * 🐍: Refactor Multipart of Code * 🐛: Extract Some Methods and Variables * 🪄: Revert `test_apps` & `test_insert` * 📦: Use `any()` instead of for loop --- piccolo/apps/asgi/commands/new.py | 4 +- piccolo/apps/fixtures/commands/dump.py | 10 +-- .../apps/migrations/auto/diffable_table.py | 9 +- .../apps/migrations/auto/schema_snapshot.py | 15 ++-- piccolo/apps/migrations/auto/serialisation.py | 7 +- .../migrations/auto/serialisation_legacy.py | 15 ++-- piccolo/apps/migrations/commands/backwards.py | 21 +++-- piccolo/apps/migrations/commands/check.py | 2 +- piccolo/apps/migrations/commands/clean.py | 3 +- piccolo/apps/migrations/commands/new.py | 15 ++-- piccolo/apps/playground/commands/run.py | 7 +- piccolo/apps/schema/commands/generate.py | 62 +++++++------- piccolo/apps/shell/commands/run.py | 4 +- piccolo/apps/user/commands/change_password.py | 2 +- piccolo/apps/user/commands/create.py | 2 +- piccolo/apps/user/tables.py | 5 +- piccolo/columns/base.py | 84 +++++++++---------- piccolo/columns/combination.py | 2 +- piccolo/columns/readable.py | 2 +- piccolo/conf/apps.py | 25 +++--- piccolo/engine/postgres.py | 3 +- piccolo/engine/sqlite.py | 41 ++++----- piccolo/query/methods/alter.py | 40 +++++---- piccolo/query/methods/create.py | 2 +- piccolo/query/methods/insert.py | 8 +- piccolo/query/methods/objects.py | 14 ++-- piccolo/query/methods/select.py | 26 +++--- piccolo/query/methods/update.py | 22 +++-- piccolo/query/mixins.py | 13 ++- piccolo/querystring.py | 22 ++--- piccolo/table.py | 41 +++++---- piccolo/utils/graphlib/_graphlib.py | 2 +- piccolo/utils/printing.py | 9 +- piccolo/utils/repr.py | 3 +- piccolo/utils/sync.py | 16 ++-- 35 files changed, 251 insertions(+), 307 deletions(-) diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index 7fdd20da0..3b7a2482d 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -18,9 +18,7 @@ def print_instruction(message: str): def get_options_string(options: t.List[str]): - return ", ".join( - [f"{name} [{index}]" for index, name in enumerate(options)] - ) + return ", ".join(f"{name} [{index}]" for index, name in enumerate(options)) def get_routing_framework() -> str: diff --git a/piccolo/apps/fixtures/commands/dump.py b/piccolo/apps/fixtures/commands/dump.py index 512622575..59c563162 100644 --- a/piccolo/apps/fixtures/commands/dump.py +++ b/piccolo/apps/fixtures/commands/dump.py @@ -62,8 +62,7 @@ async def dump_to_json_string( pydantic_model = create_pydantic_fixture_model( fixture_configs=fixture_configs ) - json_output = pydantic_model(**dump).json() - return json_output + return pydantic_model(**dump).json() def parse_args(apps: str, tables: str) -> t.List[FixtureConfig]: @@ -84,12 +83,7 @@ def parse_args(apps: str, tables: str) -> t.List[FixtureConfig]: table_class_names: t.Optional[t.List[str]] = None if tables != "all": - if "," in tables: - table_class_names = tables.split(",") - else: - # Must be a single table class name - table_class_names = [tables] - + table_class_names = tables.split(",") if "," in tables else [tables] output: t.List[FixtureConfig] = [] for app_name in app_names: diff --git a/piccolo/apps/migrations/auto/diffable_table.py b/piccolo/apps/migrations/auto/diffable_table.py index 3e3ed03b9..95775a9ca 100644 --- a/piccolo/apps/migrations/auto/diffable_table.py +++ b/piccolo/apps/migrations/auto/diffable_table.py @@ -36,11 +36,12 @@ def compare_dicts(dict_1, dict_2) -> t.Dict[str, t.Any]: for key, value in dict_1.items(): dict_2_value = dict_2.get(key, ...) - if dict_2_value is ...: + if ( + dict_2_value is not ... + and dict_2_value != value + or dict_2_value is ... + ): output[key] = value - else: - if dict_2_value != value: - output[key] = value return output diff --git a/piccolo/apps/migrations/auto/schema_snapshot.py b/piccolo/apps/migrations/auto/schema_snapshot.py index 8e1a06fb9..1b51ccd91 100644 --- a/piccolo/apps/migrations/auto/schema_snapshot.py +++ b/piccolo/apps/migrations/auto/schema_snapshot.py @@ -22,7 +22,7 @@ class SchemaSnapshot: def get_table_from_snapshot(self, table_class_name: str) -> DiffableTable: snapshot = self.get_snapshot() filtered = [i for i in snapshot if i.class_name == table_class_name] - if len(filtered) == 0: + if not filtered: raise ValueError(f"No match was found for {table_class_name}") return filtered[0] @@ -85,13 +85,12 @@ def get_snapshot(self) -> t.List[DiffableTable]: if ( alter_column.column_class != alter_column.old_column_class - ): - if alter_column.column_class is not None: - new_column = alter_column.column_class( - **column._meta.params - ) - new_column._meta = column._meta - table.columns[index] = new_column + ) and alter_column.column_class is not None: + new_column = alter_column.column_class( + **column._meta.params + ) + new_column._meta = column._meta + table.columns[index] = new_column ############################################################### diff --git a/piccolo/apps/migrations/auto/serialisation.py b/piccolo/apps/migrations/auto/serialisation.py index ecca2d79b..087b64a30 100644 --- a/piccolo/apps/migrations/auto/serialisation.py +++ b/piccolo/apps/migrations/auto/serialisation.py @@ -69,11 +69,10 @@ def __eq__(self, other): def __repr__(self): args = ", ".join( - [ - f"{key}={self.serialised_params.params.get(key).__repr__()}" # noqa: E501 - for key in self.instance._meta.params.keys() - ] + f"{key}={self.serialised_params.params.get(key).__repr__()}" + for key in self.instance._meta.params.keys() ) + return f"{self.instance.__class__.__name__}({args})" diff --git a/piccolo/apps/migrations/auto/serialisation_legacy.py b/piccolo/apps/migrations/auto/serialisation_legacy.py index 3bec6ce2d..b1e776054 100644 --- a/piccolo/apps/migrations/auto/serialisation_legacy.py +++ b/piccolo/apps/migrations/auto/serialisation_legacy.py @@ -41,7 +41,7 @@ def deserialise_legacy_params(name: str, value: str) -> t.Any: ########################################################################### - if name == "on_update": + elif name == "on_update": enum_name, item_name = value.split(".") if enum_name == "OnUpdate": return getattr(OnUpdate, item_name) @@ -49,14 +49,13 @@ def deserialise_legacy_params(name: str, value: str) -> t.Any: ########################################################################### if name == "default": - if value in ("TimestampDefault.now", "DatetimeDefault.now"): + if value in {"TimestampDefault.now", "DatetimeDefault.now"}: return TimestampNow() + try: + _value = datetime.datetime.fromisoformat(value) + except ValueError: + pass else: - try: - _value = datetime.datetime.fromisoformat(value) - except ValueError: - pass - else: - return _value + return _value return value diff --git a/piccolo/apps/migrations/commands/backwards.py b/piccolo/apps/migrations/commands/backwards.py index acec79b3e..1925d4ef6 100644 --- a/piccolo/apps/migrations/commands/backwards.py +++ b/piccolo/apps/migrations/commands/backwards.py @@ -134,18 +134,17 @@ async def run_backwards( "Enter y to continue.\n" ) ) - if _continue == "y": - for _app_name in sorted_app_names: - print(f"Undoing {_app_name}") - manager = BackwardsMigrationManager( - app_name=_app_name, - migration_id="all", - auto_agree=auto_agree, - ) - await manager.run() - return MigrationResult(success=True) - else: + if _continue != "y": return MigrationResult(success=False, message="User cancelled") + for _app_name in sorted_app_names: + print(f"Undoing {_app_name}") + manager = BackwardsMigrationManager( + app_name=_app_name, + migration_id="all", + auto_agree=auto_agree, + ) + await manager.run() + return MigrationResult(success=True) else: manager = BackwardsMigrationManager( app_name=app_name, diff --git a/piccolo/apps/migrations/commands/check.py b/piccolo/apps/migrations/commands/check.py index c2377c002..fd2b49c3d 100644 --- a/piccolo/apps/migrations/commands/check.py +++ b/piccolo/apps/migrations/commands/check.py @@ -32,7 +32,7 @@ async def get_migration_statuses(self) -> t.List[MigrationStatus]: app_name = app_config.app_name - if (self.app_name != "all") and (self.app_name != app_name): + if self.app_name not in ["all", app_name]: continue migration_modules = self.get_migration_modules( diff --git a/piccolo/apps/migrations/commands/clean.py b/piccolo/apps/migrations/commands/clean.py index e2fa7ca6b..08c075672 100644 --- a/piccolo/apps/migrations/commands/clean.py +++ b/piccolo/apps/migrations/commands/clean.py @@ -37,8 +37,7 @@ def get_migration_ids_to_remove(self) -> t.List[str]: if len(migration_ids) > 0: query = query.where(Migration.name.not_in(migration_ids)) - migration_ids_to_remove = query.run_sync() - return migration_ids_to_remove + return query.run_sync() async def run(self): print("Checking the migration table ...") diff --git a/piccolo/apps/migrations/commands/new.py b/piccolo/apps/migrations/commands/new.py index f4d3fdb1a..6179059bc 100644 --- a/piccolo/apps/migrations/commands/new.py +++ b/piccolo/apps/migrations/commands/new.py @@ -47,11 +47,10 @@ def _create_migrations_folder(migrations_path: str) -> bool: """ if os.path.exists(migrations_path): return False - else: - os.mkdir(migrations_path) - with open(os.path.join(migrations_path, "__init__.py"), "w"): - pass - return True + os.mkdir(migrations_path) + with open(os.path.join(migrations_path, "__init__.py"), "w"): + pass + return True @dataclass @@ -112,7 +111,7 @@ async def _create_new_migration( list(set(chain(*[i.extra_definitions for i in alter_statements]))), ) - if sum([len(i.statements) for i in alter_statements]) == 0: + if sum(len(i.statements) for i in alter_statements) == 0: raise NoChanges() file_contents = render_template( @@ -171,9 +170,7 @@ async def get_alter_statements( differ = SchemaDiffer( schema=current_diffable_tables, schema_snapshot=snapshot ) - alter_statements = differ.get_alter_statements() - - return alter_statements + return differ.get_alter_statements() ############################################################################### diff --git a/piccolo/apps/playground/commands/run.py b/piccolo/apps/playground/commands/run.py index e8fe044a3..31a28d631 100644 --- a/piccolo/apps/playground/commands/run.py +++ b/piccolo/apps/playground/commands/run.py @@ -159,13 +159,10 @@ def run( "port": port, } ) - for _table in TABLES: - _table._meta._db = db else: db = SQLiteEngine() - for _table in TABLES: - _table._meta._db = db - + for _table in TABLES: + _table._meta._db = db print("Tables:\n") for _table in TABLES: diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index b70c5f267..5c9cd60d2 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -66,10 +66,10 @@ class RowMeta: numeric_precision: t.Optional[t.Union[int, str]] numeric_scale: t.Optional[t.Union[int, str]] numeric_precision_radix: t.Optional[Literal[2, 10]] - + @classmethod def get_column_name_str(cls) -> str: - return ", ".join([i.name for i in dataclasses.fields(cls)]) + return ", ".join(i.name for i in dataclasses.fields(cls)) @dataclasses.dataclass @@ -107,22 +107,19 @@ def __post_init__(self): self.primary_key_constraints = primary_key_constraints def is_primary_key(self, column_name: str) -> bool: - for i in self.primary_key_constraints: - if i.column_name == column_name: - return True - return False + return any( + i.column_name == column_name for i in self.primary_key_constraints + ) def is_unique(self, column_name: str) -> bool: - for i in self.unique_constraints: - if i.column_name == column_name: - return True - return False + return any( + i.column_name == column_name for i in self.unique_constraints + ) def is_foreign_key(self, column_name: str) -> bool: - for i in self.foreign_key_constraints: - if i.column_name == column_name: - return True - return False + return any( + i.column_name == column_name for i in self.foreign_key_constraints + ) def get_foreign_key_constraint_name(self, column_name) -> ConstraintTable: for i in self.foreign_key_constraints: @@ -141,10 +138,13 @@ class Trigger: table_name: str column_name: str on_update: str - on_delete: Literal["NO ACTION", "RESTRICT", "CASCADE", "SET NULL", "SET_DEFAULT"] + on_delete: Literal[ + "NO ACTION", "RESTRICT", "CASCADE", "SET NULL", "SET_DEFAULT" + ] references_table: str references_column: str + @dataclasses.dataclass class TableTriggers: """ @@ -155,15 +155,16 @@ class TableTriggers: triggers: t.List[Trigger] def get_column_triggers(self, column_name: str) -> t.List[Trigger]: - triggers = [] - for i in self.triggers: - if i.column_name == column_name: - triggers.append(i) - return triggers - - def get_column_ref_trigger(self, column_name: str, references_table: str) -> Trigger: + return [i for i in self.triggers if i.column_name == column_name] + + def get_column_ref_trigger( + self, column_name: str, references_table: str + ) -> Trigger: for i in self.triggers: - if i.column_name == column_name and i.references_table == references_table: + if ( + i.column_name == column_name + and i.references_table == references_table + ): return i raise ValueError("No matching trigger found") @@ -303,7 +304,7 @@ def get_column_default( value = match.groupdict() if column_type is Boolean: - return True if value["value"] == "true" else False + return value["value"] == "true" elif column_type is Interval: kwargs = {} for period in [ @@ -324,10 +325,11 @@ def get_column_default( dict( zip( ["hours", "minutes", "seconds"], - [int(v) for v in value["digits"].split(":")], + [int(v) for v in digits.split(":")], ) ) ) + return IntervalCustom(**kwargs) elif column_type is JSON or column_type is JSONB: return json.loads(value["value"]) @@ -410,7 +412,7 @@ async def get_fk_triggers( "RESTRICT": OnDelete.restrict, "CASCADE": OnDelete.cascade, "SET NULL": OnDelete.set_null, - "SET DEFAULT": OnDelete.set_default + "SET DEFAULT": OnDelete.set_default, } ONUPDATE_MAP = { @@ -418,7 +420,7 @@ async def get_fk_triggers( "RESTRICT": OnUpdate.restrict, "CASCADE": OnUpdate.cascade, "SET NULL": OnUpdate.set_null, - "SET DEFAULT": OnUpdate.set_default + "SET DEFAULT": OnUpdate.set_default, } @@ -605,7 +607,9 @@ async def create_table_class_from_db( else ForeignKeyPlaceholder ) - trigger = triggers.get_column_ref_trigger(column_name, constraint_table.name) + trigger = triggers.get_column_ref_trigger( + column_name, constraint_table.name + ) if trigger: kwargs["on_update"] = ONUPDATE_MAP[trigger.on_update] kwargs["on_delete"] = ONDELETE_MAP[trigger.on_delete] @@ -632,7 +636,7 @@ async def create_table_class_from_db( default_value = get_column_default(column_type, column_default) if default_value: kwargs["default"] = default_value - + column = column_type(**kwargs) serialised_params = serialise_params(column._meta.params) diff --git a/piccolo/apps/shell/commands/run.py b/piccolo/apps/shell/commands/run.py index 225025f93..566c36c29 100644 --- a/piccolo/apps/shell/commands/run.py +++ b/piccolo/apps/shell/commands/run.py @@ -37,9 +37,9 @@ def run(): app_registry: AppRegistry = Finder().get_app_registry() tables = {} - spacer = "-------" - if app_registry.app_configs: + spacer = "-------" + print(spacer) for app_name, app_config in app_registry.app_configs.items(): diff --git a/piccolo/apps/user/commands/change_password.py b/piccolo/apps/user/commands/change_password.py index b4106333f..22c65e6de 100644 --- a/piccolo/apps/user/commands/change_password.py +++ b/piccolo/apps/user/commands/change_password.py @@ -16,7 +16,7 @@ def change_password(): password = get_password() confirmed_password = get_confirmed_password() - if not password == confirmed_password: + if password != confirmed_password: sys.exit("Passwords don't match!") BaseUser.update_password_sync(user=username, password=password) diff --git a/piccolo/apps/user/commands/create.py b/piccolo/apps/user/commands/create.py index d1fb5ed93..dabb0652a 100644 --- a/piccolo/apps/user/commands/create.py +++ b/piccolo/apps/user/commands/create.py @@ -73,7 +73,7 @@ def create( password = get_password() confirmed_password = get_confirmed_password() - if not password == confirmed_password: + if password != confirmed_password: sys.exit("Passwords don't match!") if len(password) < 4: diff --git a/piccolo/apps/user/tables.py b/piccolo/apps/user/tables.py index 01da9c124..879e14f8c 100644 --- a/piccolo/apps/user/tables.py +++ b/piccolo/apps/user/tables.py @@ -107,9 +107,8 @@ def __setattr__(self, name: str, value: t.Any): """ Make sure that if the password is set, it's stored in a hashed form. """ - if name == "password": - if not value.startswith("pbkdf2_sha256"): - value = self.__class__.hash_password(value) + if name == "password" and not value.startswith("pbkdf2_sha256"): + value = self.__class__.hash_password(value) super().__setattr__(name, value) diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index b6992513c..56e671f72 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -174,22 +174,21 @@ def get_choices_dict(self) -> t.Optional[t.Dict[str, t.Any]]: """ if self.choices is None: return None - else: - output = {} - for element in self.choices: - if isinstance(element.value, Choice): - display_name = element.value.display_name - value = element.value.value - else: - display_name = element.name.replace("_", " ").title() - value = element.value - - output[element.name] = { - "display_name": display_name, - "value": value, - } - - return output + output = {} + for element in self.choices: + if isinstance(element.value, Choice): + display_name = element.value.display_name + value = element.value.value + else: + display_name = element.name.replace("_", " ").title() + value = element.value + + output[element.name] = { + "display_name": display_name, + "value": value, + } + + return output def get_full_name(self, just_alias=False) -> str: """ @@ -201,9 +200,9 @@ def get_full_name(self, just_alias=False) -> str: return f"{self.table._meta.tablename}.{column_name}" column_name = ( - "$".join([i._meta.name for i in self.call_chain]) - + f"${column_name}" + "$".join(i._meta.name for i in self.call_chain) + f"${column_name}" ) + alias = f"{self.call_chain[-1]._meta.table_alias}.{self.name}" if just_alias: return alias @@ -521,8 +520,7 @@ def get_default_value(self) -> t.Any: if default is not ...: default = default.value if isinstance(default, Enum) else default is_callable = hasattr(default, "__call__") - value = default() if is_callable else default - return value + return default() if is_callable else default return None def get_select_string(self, engine_type: str, just_alias=False) -> str: @@ -531,9 +529,8 @@ def get_select_string(self, engine_type: str, just_alias=False) -> str: """ if self.alias is None: return self._meta.get_full_name(just_alias=just_alias) - else: - original_name = self._meta.get_full_name(just_alias=True) - return f"{original_name} AS {self.alias}" + original_name = self._meta.get_full_name(just_alias=True) + return f"{original_name} AS {self.alias}" def get_where_string(self, engine_type: str) -> str: return self.get_select_string(engine_type=engine_type, just_alias=True) @@ -553,46 +550,41 @@ def get_sql_value(self, value: t.Any) -> t.Any: """ if isinstance(value, Default): - output = getattr(value, self._meta.engine_type) + return getattr(value, self._meta.engine_type) elif value is None: - output = "null" + return "null" elif isinstance(value, (float, decimal.Decimal)): - output = str(value) + return str(value) elif isinstance(value, str): - output = f"'{value}'" + return f"'{value}'" elif isinstance(value, bool): - output = str(value).lower() + return str(value).lower() elif isinstance(value, datetime.datetime): - output = f"'{value.isoformat().replace('T', ' ')}'" + return f"'{value.isoformat().replace('T', ' ')}'" elif isinstance(value, datetime.date): - output = f"'{value.isoformat()}'" + return f"'{value.isoformat()}'" elif isinstance(value, datetime.time): - output = f"'{value.isoformat()}'" + return f"'{value.isoformat()}'" elif isinstance(value, datetime.timedelta): interval = IntervalCustom.from_timedelta(value) - output = getattr(interval, self._meta.engine_type) + return getattr(interval, self._meta.engine_type) elif isinstance(value, bytes): - output = f"'{value.hex()}'" + return f"'{value.hex()}'" elif isinstance(value, uuid.UUID): - output = f"'{value}'" + return f"'{value}'" elif isinstance(value, list): # Convert to the array syntax. - output = ( + return ( "'{" + ", ".join( - [ - f'"{i}"' - if isinstance(i, str) - else str(self.get_sql_value(i)) - for i in value - ] + f'"{i}"' + if isinstance(i, str) + else str(self.get_sql_value(i)) + for i in value ) - + "}'" - ) + ) + "}'" else: - output = value - - return output + return value @property def column_type(self): diff --git a/piccolo/columns/combination.py b/piccolo/columns/combination.py index d254ba359..e03862c00 100644 --- a/piccolo/columns/combination.py +++ b/piccolo/columns/combination.py @@ -163,7 +163,7 @@ def values_querystring(self) -> QueryString: if isinstance(values, Undefined): raise ValueError("values is undefined") - template = ", ".join(["{}" for _ in values]) + template = ", ".join("{}" for _ in values) return QueryString(template, *values) @property diff --git a/piccolo/columns/readable.py b/piccolo/columns/readable.py index bf007e939..7780ac05b 100644 --- a/piccolo/columns/readable.py +++ b/piccolo/columns/readable.py @@ -24,7 +24,7 @@ class Readable(Selectable): @property def _columns_string(self) -> str: return ", ".join( - [i._meta.get_full_name(just_alias=True) for i in self.columns] + i._meta.get_full_name(just_alias=True) for i in self.columns ) def _get_string(self, operator: str) -> str: diff --git a/piccolo/conf/apps.py b/piccolo/conf/apps.py index d0482470d..133104d22 100644 --- a/piccolo/conf/apps.py +++ b/piccolo/conf/apps.py @@ -158,7 +158,7 @@ def get_table_with_name(self, table_class_name: str) -> t.Type[Table]: for table_class in self.table_classes if table_class.__name__ == table_class_name ] - if len(filtered) == 0: + if not filtered: raise ValueError( f"No table with class name {table_class_name} exists." ) @@ -186,18 +186,16 @@ def __post_init__(self): app_conf_module = import_module(app) app_config: AppConfig = getattr(app_conf_module, "APP_CONFIG") except (ImportError, AttributeError) as e: - if not app.endswith(".piccolo_app"): - app += ".piccolo_app" - app_conf_module = import_module(app) - app_config: AppConfig = getattr( - app_conf_module, "APP_CONFIG" - ) - colored_warning( - f"App {app[:-12]} should end with `.piccolo_app`", - level=Level.medium, - ) - else: + if app.endswith(".piccolo_app"): raise e + app += ".piccolo_app" + app_conf_module = import_module(app) + app_config: AppConfig = getattr(app_conf_module, "APP_CONFIG") + colored_warning( + f"App {app[:-12]} should end with `.piccolo_app`", + level=Level.medium, + ) + self.app_configs[app_config.app_name] = app_config app_names.append(app_config.app_name) @@ -365,8 +363,7 @@ def get_app_registry(self) -> AppRegistry: Returns the AppRegistry instance within piccolo_conf. """ piccolo_conf_module = self.get_piccolo_conf_module() - app_registry = getattr(piccolo_conf_module, "APP_REGISTRY") - return app_registry + return getattr(piccolo_conf_module, "APP_REGISTRY") def get_engine( self, module_name: t.Optional[str] = None diff --git a/piccolo/engine/postgres.py b/piccolo/engine/postgres.py index a823c4b84..9ba9302ee 100644 --- a/piccolo/engine/postgres.py +++ b/piccolo/engine/postgres.py @@ -260,8 +260,7 @@ def _parse_raw_version_string(version_string: str) -> float: """ version_segment = version_string.split(" ")[0] major, minor = version_segment.split(".")[:2] - version = float(f"{major}.{minor}") - return version + return float(f"{major}.{minor}") async def get_version(self) -> float: """ diff --git a/piccolo/engine/sqlite.py b/piccolo/engine/sqlite.py index 538440077..cca29600e 100644 --- a/piccolo/engine/sqlite.py +++ b/piccolo/engine/sqlite.py @@ -84,9 +84,8 @@ def convert_array_in(value: list): """ Converts a list value into a string. """ - if len(value) > 0: - if type(value[0]) not in [str, int, float]: - raise ValueError("Can only serialise str, int and float.") + if value and type(value[0]) not in [str, int, float]: + raise ValueError("Can only serialise str, int and float.") return dump_json(value) @@ -332,10 +331,7 @@ async def __aexit__(self, exception_type, exception, traceback): def dict_factory(cursor, row) -> t.Dict: - d = {} - for idx, col in enumerate(cursor.description): - d[col[0]] = row[idx] - return d + return {col[0]: row[idx] for idx, col in enumerate(cursor.description)} class SQLiteEngine(Engine): @@ -407,21 +403,16 @@ def create_db(self, migrate=False): Create the database file, with the option to run migrations. Useful for testing purposes. """ - if not os.path.exists(self.path): - with open(self.path, "w"): - pass - else: + if os.path.exists(self.path): raise Exception(f"Database at {self.path} already exists") - if migrate: + with open(self.path, "w"): + pass # Commented out for now, as migrations for SQLite aren't as # well supported as Postgres. # from piccolo.commands.migration.forwards import ( # ForwardsMigrationManager, # ) - # ForwardsMigrationManager().run() - pass - ########################################################################### async def batch(self, query: Query, batch_size: int = 100) -> AsyncBatch: @@ -468,13 +459,13 @@ async def _run_in_new_connection( async with connection.execute(query, args) as cursor: await connection.commit() - if query_type == "insert": - assert table is not None - pk = await self._get_inserted_pk(cursor, table) - return [{table._meta.primary_key._meta.name: pk}] - else: + if query_type != "insert": return await cursor.fetchall() + assert table is not None + pk = await self._get_inserted_pk(cursor, table) + return [{table._meta.primary_key._meta.name: pk}] + async def _run_in_existing_connection( self, connection, @@ -492,13 +483,13 @@ async def _run_in_existing_connection( async with connection.execute(query, args) as cursor: response = await cursor.fetchall() - if query_type == "insert": - assert table is not None - pk = await self._get_inserted_pk(cursor, table) - return [{table._meta.primary_key._meta.name: pk}] - else: + if query_type != "insert": return response + assert table is not None + pk = await self._get_inserted_pk(cursor, table) + return [{table._meta.primary_key._meta.name: pk}] + async def run_querystring( self, querystring: QueryString, in_pool: bool = False ): diff --git a/piccolo/query/methods/alter.py b/piccolo/query/methods/alter.py index 9a5cac5ff..442f11587 100644 --- a/piccolo/query/methods/alter.py +++ b/piccolo/query/methods/alter.py @@ -141,16 +141,15 @@ class SetUnique(AlterColumnStatement): def ddl(self) -> str: if self.boolean: return f"ADD UNIQUE ({self.column_name})" - else: - if isinstance(self.column, str): - raise ValueError( - "Removing a unique constraint requires a Column instance " - "to be passed as the column arg instead of a string." - ) - tablename = self.column._meta.table._meta.tablename - column_name = self.column_name - key = f"{tablename}_{column_name}_key" - return f'DROP CONSTRAINT "{key}"' + if isinstance(self.column, str): + raise ValueError( + "Removing a unique constraint requires a Column instance " + "to be passed as the column arg instead of a string." + ) + tablename = self.column._meta.table._meta.tablename + column_name = self.column_name + key = f"{tablename}_{column_name}_key" + return f'DROP CONSTRAINT "{key}"' @dataclass @@ -231,16 +230,16 @@ class SetDigits(AlterColumnStatement): @property def ddl(self) -> str: - if self.digits is not None: - precision = self.digits[0] - scale = self.digits[1] - return ( - f"ALTER COLUMN {self.column_name} TYPE " - f"{self.column_type}({precision}, {scale})" - ) - else: + if self.digits is None: return f"ALTER COLUMN {self.column_name} TYPE {self.column_type}" + precision = self.digits[0] + scale = self.digits[1] + return ( + f"ALTER COLUMN {self.column_name} TYPE " + f"{self.column_type}({precision}, {scale})" + ) + @dataclass class DropTable: @@ -429,8 +428,7 @@ def set_length(self, column: t.Union[str, Varchar], length: int) -> Alter: def _get_constraint_name(self, column: t.Union[str, ForeignKey]) -> str: column_name = AlterColumnStatement(column=column).column_name tablename = self.table._meta.tablename - constraint_name = f"{tablename}_{column_name}_fk" - return constraint_name + return f"{tablename}_{column_name}_fk" def drop_constraint(self, constraint_name: str) -> Alter: self._drop_contraint.append( @@ -527,6 +525,6 @@ def default_ddl(self) -> t.Sequence[str]: return [f"{query} {i}" for i in alterations] # Postgres can perform them all at once: - query += ",".join([f" {i}" for i in alterations]) + query += ",".join(f" {i}" for i in alterations) return [query] diff --git a/piccolo/query/methods/create.py b/piccolo/query/methods/create.py index e84d1465d..94292de51 100644 --- a/piccolo/query/methods/create.py +++ b/piccolo/query/methods/create.py @@ -39,7 +39,7 @@ def default_ddl(self) -> t.Sequence[str]: columns = self.table._meta.columns base = f"{prefix} {self.table._meta.tablename}" - columns_sql = ", ".join([i.ddl for i in columns]) + columns_sql = ", ".join(i.ddl for i in columns) create_table_ddl = f"{base} ({columns_sql})" create_indexes: t.List[str] = [] diff --git a/piccolo/query/methods/insert.py b/piccolo/query/methods/insert.py index 63d6e0fe5..0dc3cdae8 100644 --- a/piccolo/query/methods/insert.py +++ b/piccolo/query/methods/insert.py @@ -40,8 +40,8 @@ def run_callback(self, results): @property def sqlite_querystrings(self) -> t.Sequence[QueryString]: base = f"INSERT INTO {self.table._meta.tablename}" - columns = ",".join([i._meta.name for i in self.table._meta.columns]) - values = ",".join(["{}" for _ in self.add_delegate._add]) + columns = ",".join(i._meta.name for i in self.table._meta.columns) + values = ",".join("{}" for _ in self.add_delegate._add) query = f"{base} ({columns}) VALUES {values}" return [ QueryString( @@ -56,9 +56,9 @@ def sqlite_querystrings(self) -> t.Sequence[QueryString]: def postgres_querystrings(self) -> t.Sequence[QueryString]: base = f"INSERT INTO {self.table._meta.tablename}" columns = ",".join( - [f'"{i._meta.name}"' for i in self.table._meta.columns] + f'"{i._meta.name}"' for i in self.table._meta.columns ) - values = ",".join(["{}" for i in self.add_delegate._add]) + values = ",".join("{}" for i in self.add_delegate._add) primary_key_name = self.table._meta.primary_key._meta.name query = ( f"{base} ({columns}) VALUES {values} RETURNING {primary_key_name}" diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index 1e0ec450b..db9469c39 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -203,16 +203,14 @@ async def response_handler(self, response): if self.limit_delegate._first: if len(response) == 0: return None - else: - if self.output_delegate._output.nested: - return make_nested(response[0]) - else: - return response[0] - else: if self.output_delegate._output.nested: - return [make_nested(i) for i in response] + return make_nested(response[0]) else: - return response + return response[0] + elif self.output_delegate._output.nested: + return [make_nested(i) for i in response] + else: + return response @property def default_querystrings(self) -> t.Sequence[QueryString]: diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index c5be9c902..6260ad917 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -216,11 +216,10 @@ async def response_handler(self, response): return make_nested(response[0]) else: return response[0] + elif self.output_delegate._output.nested and not was_select_star: + return [make_nested(i) for i in response] else: - if self.output_delegate._output.nested and not was_select_star: - return [make_nested(i) for i in response] - else: - return response + return response def order_by(self, *columns: Column, ascending=True) -> Select: _columns: t.List[Column] = [ @@ -281,11 +280,10 @@ def _get_joins(self, columns: t.Sequence[Selectable]) -> t.List[str]: _joins: t.List[str] = [] for index, key in enumerate(column._meta.call_chain, 0): table_alias = "$".join( - [ - f"{_key._meta.table._meta.tablename}${_key._meta.name}" - for _key in column._meta.call_chain[: index + 1] - ] + f"{_key._meta.table._meta.tablename}${_key._meta.name}" + for _key in column._meta.call_chain[: index + 1] ) + key._meta.table_alias = table_alias if index > 0: @@ -314,15 +312,13 @@ def _check_valid_call_chain(self, keys: t.Sequence[Selectable]) -> bool: for column in keys: if not isinstance(column, Column): continue - if column._meta.call_chain: + if column._meta.call_chain and len(column._meta.call_chain) > 10: # Make sure the call_chain isn't too large to discourage # very inefficient queries. - - if len(column._meta.call_chain) > 10: - raise Exception( - "Joining more than 10 tables isn't supported - " - "please restructure your query." - ) + raise Exception( + "Joining more than 10 tables isn't supported - " + "please restructure your query." + ) return True @property diff --git a/piccolo/query/methods/update.py b/piccolo/query/methods/update.py index 94997c0bc..d90f61295 100644 --- a/piccolo/query/methods/update.py +++ b/piccolo/query/methods/update.py @@ -51,10 +51,8 @@ def default_querystrings(self) -> t.Sequence[QueryString]: self.validate() columns_str = ", ".join( - [ - f"{col._meta.name} = {{}}" - for col, _ in self.values_delegate._values.items() - ] + f"{col._meta.name} = {{}}" + for col, _ in self.values_delegate._values.items() ) query = f"UPDATE {self.table._meta.tablename} SET " + columns_str @@ -63,12 +61,12 @@ def default_querystrings(self) -> t.Sequence[QueryString]: query, *self.values_delegate.get_sql_values() ) - if self.where_delegate._where: - where_querystring = QueryString( - "{} WHERE {}", - querystring, - self.where_delegate._where.querystring, - ) - return [where_querystring] - else: + if not self.where_delegate._where: return [querystring] + + where_querystring = QueryString( + "{} WHERE {}", + querystring, + self.where_delegate._where.querystring, + ) + return [where_querystring] diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index 8f75abbc9..ced0b0e77 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -64,8 +64,9 @@ class OrderBy: def querystring(self) -> QueryString: order = "ASC" if self.ascending else "DESC" columns_names = ", ".join( - [i._meta.get_full_name(just_alias=True) for i in self.columns] + i._meta.get_full_name(just_alias=True) for i in self.columns ) + return QueryString(f" ORDER BY {columns_names} {order}") def __str__(self): @@ -109,16 +110,13 @@ def get_where_columns(self): def _extract_columns(self, combinable: Combinable): if isinstance(combinable, Where): self._where_columns.append(combinable.column) - elif isinstance(combinable, And) or isinstance(combinable, Or): + elif isinstance(combinable, (And, Or)): self._extract_columns(combinable.first) self._extract_columns(combinable.second) def where(self, *where: Combinable): for arg in where: - if self._where: - self._where = And(self._where, arg) - else: - self._where = arg + self._where = And(self._where, arg) if self._where else arg @dataclass @@ -370,8 +368,9 @@ class GroupBy: @property def querystring(self) -> QueryString: columns_names = ", ".join( - [i._meta.get_full_name(just_alias=True) for i in self.columns] + i._meta.get_full_name(just_alias=True) for i in self.columns ) + return QueryString(f" GROUP BY {columns_names}") def __str__(self): diff --git a/piccolo/querystring.py b/piccolo/querystring.py index 3c6b25c57..c3727fc21 100644 --- a/piccolo/querystring.py +++ b/piccolo/querystring.py @@ -74,10 +74,8 @@ def __str__(self): start_index=1, bundled=[], combined_args=[] ) template = "".join( - [ - fragment.prefix + ("" if fragment.no_arg else "{}") - for fragment in bundled - ] + fragment.prefix + ("" if fragment.no_arg else "{}") + for fragment in bundled ) # Do some basic type conversion here. @@ -150,19 +148,17 @@ def compile_string( ) if engine_type == "postgres": string = "".join( - [ - fragment.prefix - + ("" if fragment.no_arg else f"${fragment.index}") - for fragment in bundled - ] + fragment.prefix + + ("" if fragment.no_arg else f"${fragment.index}") + for fragment in bundled ) + elif engine_type == "sqlite": string = "".join( - [ - fragment.prefix + ("" if fragment.no_arg else "?") - for fragment in bundled - ] + fragment.prefix + ("" if fragment.no_arg else "?") + for fragment in bundled ) + else: raise Exception("Engine type not recognised") diff --git a/piccolo/table.py b/piccolo/table.py index 6edf7728b..fc8c581f3 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -156,7 +156,7 @@ def __init_subclass__( Admin for tooltips. """ - tablename = tablename if tablename else _camel_to_snake(cls.__name__) + tablename = tablename or _camel_to_snake(cls.__name__) if tablename in PROTECTED_TABLENAMES: raise ValueError( @@ -332,24 +332,24 @@ def save(self) -> t.Union[Insert, Update]: """ cls = self.__class__ - if self._exists_in_db: - # pre-existing row - kwargs: t.Dict[Column, t.Any] = { - i: getattr(self, i._meta.name, None) - for i in cls._meta.columns - if i._meta.name != self._meta.primary_key._meta.name - } - return ( - cls.update() - .values(kwargs) # type: ignore - .where( - cls._meta.primary_key - == getattr(self, self._meta.primary_key._meta.name) - ) - ) - else: + if not self._exists_in_db: return cls.insert().add(self) + # pre-existing row + kwargs: t.Dict[Column, t.Any] = { + i: getattr(self, i._meta.name, None) + for i in cls._meta.columns + if i._meta.name != self._meta.primary_key._meta.name + } + return ( + cls.update() + .values(kwargs) # type: ignore + .where( + cls._meta.primary_key + == getattr(self, self._meta.primary_key._meta.name) + ) + ) + def remove(self) -> Delete: """ A proxy to a delete query. @@ -480,12 +480,11 @@ def _get_related_readable(cls, column: ForeignKey) -> Readable: output_name = f"{column._meta.name}_readable" - new_readable = Readable( + return Readable( template=readable.template, columns=columns, output_name=output_name, ) - return new_readable @classmethod def get_readable(cls) -> Readable: @@ -528,7 +527,7 @@ def __str__(self) -> str: return self.querystring.__str__() def __repr__(self) -> str: - _pk = self._meta.primary_key if self._meta.primary_key else None + _pk = self._meta.primary_key or None return f"<{self.__class__.__name__}: {_pk}>" ########################################################################### @@ -1004,7 +1003,7 @@ def sort_table_classes( output: t.List[t.Type[Table]] = [] for tablename in ordered_tablenames: - table_class = table_class_dict.get(tablename, None) + table_class = table_class_dict.get(tablename) if table_class is not None: output.append(table_class) diff --git a/piccolo/utils/graphlib/_graphlib.py b/piccolo/utils/graphlib/_graphlib.py index 1cc64d51d..73f0c0499 100644 --- a/piccolo/utils/graphlib/_graphlib.py +++ b/piccolo/utils/graphlib/_graphlib.py @@ -109,7 +109,7 @@ def prepare(self): # nodes as possible before cycles block more progress cycle = self._find_cycle() if cycle: - raise CycleError(f"nodes are in a cycle", cycle) + raise CycleError("nodes are in a cycle", cycle) def get_ready(self): """Return a tuple of all the nodes that are ready. diff --git a/piccolo/utils/printing.py b/piccolo/utils/printing.py index 5de6626ff..84c8926d0 100644 --- a/piccolo/utils/printing.py +++ b/piccolo/utils/printing.py @@ -3,9 +3,6 @@ def get_fixed_length_string(string: str, length=20) -> str: Add spacing to the end of the string so it's a fixed length. """ if len(string) > length: - fixed_length_string = string[: length - 3] + "..." - else: - spacing = "".join([" " for i in range(length - len(string))]) - fixed_length_string = f"{string}{spacing}" - - return fixed_length_string + return string[: length - 3] + "..." + spacing = "".join(" " for i in range(length - len(string))) + return f"{string}{spacing}" diff --git a/piccolo/utils/repr.py b/piccolo/utils/repr.py index 2e8ed8c52..d3750c104 100644 --- a/piccolo/utils/repr.py +++ b/piccolo/utils/repr.py @@ -20,6 +20,7 @@ def repr_class_instance(class_instance: object) -> str: args_dict[arg_name] = value args_str = ", ".join( - [f"{key}={value.__repr__()}" for key, value in args_dict.items()] + f"{key}={value.__repr__()}" for key, value in args_dict.items() ) + return f"{class_instance.__class__.__name__}({args_str})" diff --git a/piccolo/utils/sync.py b/piccolo/utils/sync.py index dbed89e14..42317af01 100644 --- a/piccolo/utils/sync.py +++ b/piccolo/utils/sync.py @@ -21,13 +21,11 @@ def run_sync(coroutine: t.Coroutine): except RuntimeError: return asyncio.run(coroutine) else: - if loop.is_running(): - new_loop = asyncio.new_event_loop() - - with ThreadPoolExecutor(max_workers=1) as executor: - future = executor.submit( - new_loop.run_until_complete, coroutine - ) - return future.result() - else: + if not loop.is_running(): return loop.run_until_complete(coroutine) + + new_loop = asyncio.new_event_loop() + + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(new_loop.run_until_complete, coroutine) + return future.result() From 2b9848521f6769c532cc067789aebf8ce015fc9f Mon Sep 17 00:00:00 2001 From: William Michael Short <36488354+wmshort@users.noreply.github.com> Date: Wed, 29 Sep 2021 21:34:53 +0100 Subject: [PATCH 096/727] sample basic migrations ui updates (#280) * sample basic ui updates * fix tests * emoji updates for improved display * remove unused import * minor tweaks Co-authored-by: William Michael Short Co-authored-by: Daniel Townsend --- .../apps/migrations/auto/migration_manager.py | 4 +- piccolo/apps/migrations/commands/backwards.py | 49 +++++++++---------- piccolo/apps/migrations/commands/forwards.py | 45 +++++++++-------- .../commands/test_forwards_backwards.py | 6 ++- 4 files changed, 55 insertions(+), 49 deletions(-) diff --git a/piccolo/apps/migrations/auto/migration_manager.py b/piccolo/apps/migrations/auto/migration_manager.py index 4d3b19f90..a95ecf33e 100644 --- a/piccolo/apps/migrations/auto/migration_manager.py +++ b/piccolo/apps/migrations/auto/migration_manager.py @@ -640,7 +640,7 @@ async def _run_add_columns(self, backwards=False): await _Table.create_index([add_column.column]).run() async def run(self): - print("Running MigrationManager ...") + print(f" - {self.migration_id} [forwards]... ", end="") engine = engine_finder() @@ -664,7 +664,7 @@ async def run(self): await self._run_alter_columns() async def run_backwards(self): - print("Reversing MigrationManager ...") + print(f" - {self.migration_id} [backwards]... ", end="") engine = engine_finder() diff --git a/piccolo/apps/migrations/commands/backwards.py b/piccolo/apps/migrations/commands/backwards.py index 1925d4ef6..df29486b9 100644 --- a/piccolo/apps/migrations/commands/backwards.py +++ b/piccolo/apps/migrations/commands/backwards.py @@ -46,7 +46,7 @@ async def run(self) -> MigrationResult: if len(ran_migration_ids) == 0: # Make sure a success is returned, as we don't want this # to appear as an error in automated scripts. - message = "No migrations to reverse!" + message = "🏁 No migrations to reverse!" print(message) return MigrationResult(success=True, message=message) @@ -79,20 +79,14 @@ async def run(self) -> MigrationResult: ####################################################################### + n = len(reversed_migration_ids) _continue = ( "y" if self.auto_agree - else input( - "About to undo the following migrations:\n" - f"{reversed_migration_ids}\n" - "Enter y to continue.\n" - ) + else input(f"Reverse {n} migration{'s' if n != 1 else ''}? [y/N] ") ) - if _continue == "y": - print("Undoing migrations") - + if _continue in "yY": for migration_id in reversed_migration_ids: - print(f"Reversing {migration_id}") migration_module = migration_modules[migration_id] response = await migration_module.forwards() @@ -106,6 +100,7 @@ async def run(self) -> MigrationResult: if self.clean: os.unlink(migration_module.__file__) + print("ok! ✔️") return MigrationResult(success=True) else: # pragma: no cover @@ -124,27 +119,31 @@ async def run_backwards( sorted_app_names = BaseMigrationManager().get_sorted_app_names() sorted_app_names.reverse() + names = [f"'{name}'" for name in sorted_app_names] _continue = ( "y" if auto_agree else input( - "You're about to undo the migrations for the following apps:\n" - f"{sorted_app_names}\n" - "Are you sure you want to continue?\n" - "Enter y to continue.\n" + "You are about to undo the migrations for the following " + "apps:\n" + f"{', '.join(names)}\n" + "Are you sure you want to continue? [y/N] " ) ) - if _continue != "y": - return MigrationResult(success=False, message="User cancelled") - for _app_name in sorted_app_names: - print(f"Undoing {_app_name}") - manager = BackwardsMigrationManager( - app_name=_app_name, - migration_id="all", - auto_agree=auto_agree, - ) - await manager.run() - return MigrationResult(success=True) + + if _continue in "yY": + for _app_name in sorted_app_names: + print(f"\n{_app_name.upper():^64}") + print("-" * 64) + manager = BackwardsMigrationManager( + app_name=_app_name, + migration_id="all", + auto_agree=auto_agree, + ) + await manager.run() + return MigrationResult(success=True) + else: + return MigrationResult(success=False, message="user cancelled") else: manager = BackwardsMigrationManager( app_name=app_name, diff --git a/piccolo/apps/migrations/commands/forwards.py b/piccolo/apps/migrations/commands/forwards.py index 5510cb691..99929ea67 100644 --- a/piccolo/apps/migrations/commands/forwards.py +++ b/piccolo/apps/migrations/commands/forwards.py @@ -31,17 +31,19 @@ async def run_migrations(self, app_config: AppConfig) -> MigrationResult: ] = self.get_migration_modules(app_config.migrations_folder_path) ids = self.get_migration_ids(migration_modules) - print(f"All migration ids = {ids}") + n = len(ids) + print(f"👍 {n} migration{'s' if n != 1 else ''} already complete") havent_run = sorted(set(ids) - set(already_ran)) - print(f"Haven't run = {havent_run}") - if len(havent_run) == 0: # Make sure this still appears successful, as we don't want this # to appear as an error in automated scripts. - message = "No migrations left to run!" + message = "🏁 No migrations need to be run" print(message) return MigrationResult(success=True, message=message) + else: + n = len(havent_run) + print(f"⏩ {n} migration{'s' if n != 1 else ''} not yet run") if self.migration_id == "all": subset = havent_run @@ -57,26 +59,29 @@ async def run_migrations(self, app_config: AppConfig) -> MigrationResult: else: subset = havent_run[: index + 1] - for _id in subset: - if self.fake: - print(f"Faked {_id}") - else: - migration_module = migration_modules[_id] - response = await migration_module.forwards() + if subset: + n = len(subset) + print(f"🚀 Running {n} migration{'s' if n != 1 else ''}:") + + for _id in subset: + if self.fake: + print(f"- {_id}: faked! ⏭️") + else: + migration_module = migration_modules[_id] + response = await migration_module.forwards() - if isinstance(response, MigrationManager): - await response.run() + if isinstance(response, MigrationManager): + await response.run() - print(f"-> Ran {_id}") + print("ok! ✔️") - await Migration.insert().add( - Migration(name=_id, app_name=app_config.app_name) - ).run() + await Migration.insert().add( + Migration(name=_id, app_name=app_config.app_name) + ).run() - return MigrationResult(success=True, message="Ran successfully") + return MigrationResult(success=True, message="migration succeeded") async def run(self) -> MigrationResult: - print("Running migrations ...") await self.create_migration_table() app_config = self.get_app_config(app_name=self.app_name) @@ -94,8 +99,8 @@ async def run_forwards( if app_name == "all": sorted_app_names = BaseMigrationManager().get_sorted_app_names() for _app_name in sorted_app_names: - print(f"\nMigrating {_app_name}") - print("------------------------------------------------") + print(f"\n{_app_name.upper():^64}") + print("-" * 64) manager = ForwardsMigrationManager( app_name=_app_name, migration_id="all", fake=fake ) diff --git a/tests/apps/migrations/commands/test_forwards_backwards.py b/tests/apps/migrations/commands/test_forwards_backwards.py index 6449c05da..99d24159a 100644 --- a/tests/apps/migrations/commands/test_forwards_backwards.py +++ b/tests/apps/migrations/commands/test_forwards_backwards.py @@ -142,7 +142,9 @@ def test_backwards_no_migrations(self, print_: MagicMock): auto_agree=True, ) ) - self.assertTrue(call("No migrations to reverse!") in print_.mock_calls) + self.assertTrue( + call("🏁 No migrations to reverse!") in print_.mock_calls + ) @patch("piccolo.apps.migrations.commands.forwards.print") def test_forwards_no_migrations(self, print_: MagicMock): @@ -153,7 +155,7 @@ def test_forwards_no_migrations(self, print_: MagicMock): run_sync(forwards(app_name="example_app", migration_id="all")) self.assertTrue( - print_.mock_calls[-1] == call("No migrations left to run!") + print_.mock_calls[-1] == call("🏁 No migrations need to be run") ) def test_forwards_fake(self): From 44515012997b15061dea55669de87b22785d048b Mon Sep 17 00:00:00 2001 From: ali sayyah Date: Thu, 30 Sep 2021 16:03:34 +0330 Subject: [PATCH 097/727] Reflection (#258) * base classes for implementing reflection * simple functionality * small refactor * single_table reflection, inclusion, exclusion and an option to keep existing tables have been added. * fixed lint and test issues * fixed lint issues after merge * added `clear` method to the TableStorage * added tearDownClass for TestModelBuilder to clear the test database * added unit tests for TableStorage * added get_table method to TableStorage * added more unit tests * small refactors * renamed `tablestorage` module to `table_reflection` * little cleanups and added a test * Update advanced.rst * documentation tweaks Co-authored-by: Daniel Townsend --- docs/src/piccolo/schema/advanced.rst | 92 ++++++++ piccolo/apps/schema/commands/generate.py | 20 +- piccolo/table_reflection.py | 230 ++++++++++++++++++++ tests/apps/schema/commands/test_generate.py | 2 +- tests/testing/test_model_builder.py | 34 ++- tests/utils/test_table_reflection.py | 93 ++++++++ 6 files changed, 452 insertions(+), 19 deletions(-) create mode 100644 piccolo/table_reflection.py create mode 100644 tests/utils/test_table_reflection.py diff --git a/docs/src/piccolo/schema/advanced.rst b/docs/src/piccolo/schema/advanced.rst index 12c900222..75c8b7714 100644 --- a/docs/src/piccolo/schema/advanced.rst +++ b/docs/src/piccolo/schema/advanced.rst @@ -139,3 +139,95 @@ By using choices, you get the following benefits: * Signalling to other programmers what values are acceptable for the column. * Improved storage efficiency (we can store ``'l'`` instead of ``'large'``). * Piccolo Admin support + +------------------------------------------------------------------------------- + +Reflection +---------- + +This is a very advanced feature, which is only required for specialist use +cases. + +Instead of writing your ``Table`` definitions in a ``tables.py`` file, Piccolo +can dynamically create them at run time, by inspecting the database. These +``Table`` classes are then stored in memory, using a singleton object called +``TableStorage``. + +Some example use cases: + + * You have a very dynamic database, where new tables are being created + constantly, so updating a ``tables.py`` is impractical. + * You use Piccolo on the command line to explore databases. + +Full reflection +~~~~~~~~~~~~~~~ + +Here's an example, where we reflect the entire schema: + +.. code-block:: python + + from piccolo.table_reflection import TableStorage + + storage = TableStorage() + await storage.reflect(schema_name="music") + +``Table`` objects are accessible from ``TableStorage.tables``: + +.. code-block:: python + + >>> storage.tables + {"music.Band": , ... } + + >>> Band = storage.tables["music.Band"] + +Then you can use them like your normal ``Table`` classes: + +.. code-block:: python + + >>> Band.select().run_sync() + [{'id': 1, 'name': 'Pythonistas', 'manager': 1}, ...] + + +Partial reflection +~~~~~~~~~~~~~~~~~~ + +Full schema reflection can be a heavy process based on the size of your schema. +You can use ``include``, ``exclude`` and ``keep_existing`` parameters of +the ``reflect`` method to limit the overhead dramatically. + +Only reflect the needed table(s): + +.. code-block:: python + + from piccolo.table_reflection import TableStorage + + storage = TableStorage() + await storage.reflect(schema_name="music", include=['band', ...]) + +Exclude table(s): + +.. code-block:: python + + await storage.reflect(schema_name="music", exclude=['band', ...]) + +If you set ``keep_existing=True``, only new tables on the database will be +reflected and the existing tables in ``TableStorage`` will be left intact. + +.. code-block:: python + + await storage.reflect(schema_name="music", keep_existing=True) + +get_table +~~~~~~~~~ + +``TableStorage`` has a helper method named ``get_table``. If the table is +already present in the ``TableStorage``, this will return it and if the table +is not present, it will be reflected and returned. + +.. code-block:: python + + Band = storage.get_table(tablename='band') + +.. hint:: Reflection will automatically create ``Table`` classes for referenced + tables too. For example, if ``Table1`` references ``Table2``, then + ``Table2`` will automatically be added to ``TableStorage``. diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index 5c9cd60d2..79639604e 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -160,12 +160,12 @@ def get_column_triggers(self, column_name: str) -> t.List[Trigger]: def get_column_ref_trigger( self, column_name: str, references_table: str ) -> Trigger: - for i in self.triggers: + for trigger in self.triggers: if ( - i.column_name == column_name - and i.references_table == references_table + trigger.column_name == column_name + and trigger.references_table == references_table ): - return i + return trigger raise ValueError("No matching trigger found") @@ -656,14 +656,14 @@ async def create_table_class_from_db( async def get_output_schema( schema_name: str = "public", - tablenames: t.Optional[t.List[str]] = None, + include: t.Optional[t.List[str]] = None, exclude: t.Optional[t.List[str]] = None, ) -> OutputSchema: """ :param schema_name: Name of the schema. - :param tablenames: - Optional list of table names. Only creates the specifed tables. + :param include: + Optional list of table names. Only creates the specified tables. :param exclude: Optional list of table names. excludes the specified tables. :returns: @@ -692,14 +692,14 @@ class Schema(Table, db=engine): pass - if not tablenames: - tablenames = await get_tablenames(Schema, schema_name=schema_name) + if not include: + include = await get_tablenames(Schema, schema_name=schema_name) table_coroutines = ( create_table_class_from_db( table_class=Schema, tablename=tablename, schema_name=schema_name ) - for tablename in tablenames + for tablename in include if tablename not in exclude ) output_schemas = await asyncio.gather(*table_coroutines) diff --git a/piccolo/table_reflection.py b/piccolo/table_reflection.py new file mode 100644 index 000000000..dc91d632c --- /dev/null +++ b/piccolo/table_reflection.py @@ -0,0 +1,230 @@ +""" +This is an advanced Piccolo feature which allows runtime reflection of database +tables. +""" + +import asyncio +import typing as t +from dataclasses import dataclass + +from piccolo.apps.schema.commands.generate import get_output_schema +from piccolo.table import Table + + +class Immutable(object): + def _immutable(self, *args, **kwargs) -> TypeError: + raise TypeError("%s object is immutable" % self.__class__.__name__) + + __delitem__ = __setitem__ = __setattr__ = _immutable # type: ignore + + +class ImmutableDict(Immutable, dict): # type: ignore + """A dictionary that is not publicly mutable.""" + + clear = pop = popitem = setdefault = update = Immutable._immutable # type: ignore # noqa: E501 + + def __new__(cls, *args): + new = dict.__new__(cls) + return new + + def copy(self): + raise NotImplementedError( + "an immutabledict shouldn't need to be copied. use dict(d) " + "if you need a mutable dictionary." + ) + + def __reduce__(self): + return ImmutableDict, (dict(self),) + + def _insert_item(self, key, value) -> None: + """ + insert an item into the dictionary directly. + """ + dict.__setitem__(self, key, value) + + def _delete_item(self, key) -> None: + """ + Delete an item from dictionary directly. + """ + dict.__delitem__(self, key) + + def __repr__(self): + return f"ImmutableDict({dict.__repr__(self)})" + + +class Singleton(type): + """ + A metaclass that creates a Singleton base class when called. + """ + + _instances: t.Dict = {} + + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + cls._instances[cls] = super(Singleton, cls).__call__( + *args, **kwargs + ) + return cls._instances[cls] + + +@dataclass +class TableNameDetail: + name: str = "" + schema: str = "" + + +class TableStorage(metaclass=Singleton): + """ + A singleton object to store and access reflected tables + """ + + def __init__(self): + self.tables = ImmutableDict() + self._schema_tables = dict() + + async def reflect( + self, + schema_name: str = "public", + include: t.Union[t.List[str], str, None] = None, + exclude: t.Union[t.List[str], str, None] = None, + keep_existing: bool = False, + ) -> None: + """ + Imports tables from the database into ``Table`` objects without + hard-coding them. + + If a table has a reference to another table, the referenced table will + be imported too. Reflection can have a performance impact based on the + number of tables. + + If you want to reflect your whole database, make sure to only do it + once or use the provided parameters instead of reflecting the whole + database every time. + + :param schema_name: + Name of the schema you want to reflect. + :param include: + It will only reflect the specified tables. Can be a list of tables + or a single table. + :param exclude: + It won't reflect the specified tables. Can be a list of tables or + a single table. + :param keep_existing: + If True, it will exclude the available tables and reflects the + currently unavailable ones. Default is False. + :returns: + None + + """ + include_list = self._to_list(include) + exclude_list = self._to_list(exclude) + + if keep_existing: + exclude += self._schema_tables.get(schema_name, []) + + output_schema = await get_output_schema( + schema_name=schema_name, include=include_list, exclude=exclude_list + ) + add_tables = [ + self._add_table(schema_name=schema_name, table=table) + for table in output_schema.tables + ] + await asyncio.gather(*add_tables) + + def clear(self) -> None: + """ + Removes all the tables within ``TableStorage``. + + :returns: + None + + """ + dict.clear(self.tables) + self._schema_tables.clear() + + async def get_table(self, tablename: str) -> t.Optional[t.Type[Table]]: + """ + Returns the ``Table`` class if it exists. If the table is not present + in ``TableStorage``, it will try to reflect it. + + :param tablename: + The name of the table, schema name included. If the schema is + public, it's not necessary. For example: "public.manager" or + "manager", "test_schema.test_table". + :returns: + Table | None + + """ + table_class = self.tables.get(tablename) + if table_class is None: + tableNameDetail = self._get_schema_and_table_name(tablename) + await self.reflect( + schema_name=tableNameDetail.schema, + include=[tableNameDetail.name], + ) + table_class = self.tables.get(tablename) + return table_class + + async def _add_table(self, schema_name: str, table: t.Type[Table]) -> None: + if issubclass(table, Table): + table_name = self._get_table_name( + table._meta.tablename, schema_name + ) + self.tables._insert_item(table_name, table) + self._add_to_schema_tables( + schema_name=schema_name, table_name=table._meta.tablename + ) + + def _add_to_schema_tables(self, schema_name: str, table_name: str) -> None: + """ + We keep record of schemas and their tables for easy use. This method + adds a table to its schema. + + """ + schema_tables = self._schema_tables.get(schema_name) + if schema_tables is None: + self._schema_tables[schema_name] = [] + else: + self._schema_tables[schema_name].append(table_name) + + @staticmethod + def _get_table_name(name: str, schema: str): + if schema == "public": + return name + else: + return schema + "." + name + + def __repr__(self): + return f"{[tablename for tablename, _ in self.tables.items()]}" + + @staticmethod + def _get_schema_and_table_name(tablename: str) -> TableNameDetail: + """ + Extract schema name and table name from full name of the table. + + :param tablename: + The full name of the table. + :returns: + Returns the name of the schema and the table. + + """ + tablename_list = tablename.split(".") + if len(tablename_list) == 2: + return TableNameDetail( + name=tablename_list[1], schema=tablename_list[0] + ) + + elif len(tablename_list) == 1: + return TableNameDetail(name=tablename_list[0], schema="public") + else: + raise ValueError("Couldn't find schema name.") + + @staticmethod + def _to_list(value: t.Any) -> t.List: + if isinstance(value, list): + return value + elif isinstance(value, tuple) or isinstance(value, set): + return list(value) + elif isinstance(value, str): + return [value] + return [] diff --git a/tests/apps/schema/commands/test_generate.py b/tests/apps/schema/commands/test_generate.py index f64497ee0..75989ed1e 100644 --- a/tests/apps/schema/commands/test_generate.py +++ b/tests/apps/schema/commands/test_generate.py @@ -130,7 +130,7 @@ def test_generate_required_tables(self): Make sure only tables passed to `tablenames` are created """ output_schema: OutputSchema = run_sync( - get_output_schema(tablenames=[SmallTable._meta.tablename]) + get_output_schema(include=[SmallTable._meta.tablename]) ) self.assertEqual(len(output_schema.tables), 1) SmallTable_ = output_schema.get_table_with_name("SmallTable") diff --git a/tests/testing/test_model_builder.py b/tests/testing/test_model_builder.py index 16ce96aec..7806bb07a 100644 --- a/tests/testing/test_model_builder.py +++ b/tests/testing/test_model_builder.py @@ -17,14 +17,32 @@ class TestModelBuilder(unittest.TestCase): @classmethod def setUpClass(cls): - Manager.create_table().run_sync() - Band.create_table().run_sync() - Poster.create_table().run_sync() - RecordingStudio.create_table().run_sync() - Shirt.create_table().run_sync() - Venue.create_table().run_sync() - Concert.create_table().run_sync() - Ticket.create_table().run_sync() + + for table_class in ( + Manager, + Band, + Poster, + RecordingStudio, + Shirt, + Venue, + Concert, + Ticket, + ): + table_class.create_table().run_sync() + + @classmethod + def tearDownClass(cls) -> None: + for table_class in ( + Ticket, + Concert, + Venue, + Shirt, + RecordingStudio, + Poster, + Band, + Manager, + ): + table_class.alter().drop_table().run_sync() def test_model_builder_async(self): async def build_model(model): diff --git a/tests/utils/test_table_reflection.py b/tests/utils/test_table_reflection.py new file mode 100644 index 000000000..0e540d755 --- /dev/null +++ b/tests/utils/test_table_reflection.py @@ -0,0 +1,93 @@ +import typing as t +from unittest import TestCase + +from piccolo.columns import Varchar +from piccolo.table import Table +from piccolo.table_reflection import TableStorage +from piccolo.utils.sync import run_sync +from tests.base import postgres_only +from tests.example_apps.music.tables import Band, Manager + + +@postgres_only +class TestTableStorage(TestCase): + def setUp(self) -> None: + self.table_storage = TableStorage() + for table_class in (Manager, Band): + table_class.create_table().run_sync() + + def tearDown(self): + self.table_storage.clear() + for table_class in (Band, Manager): + table_class.alter().drop_table(if_exists=True).run_sync() + + def _compare_table_columns( + self, table_1: t.Type[Table], table_2: t.Type[Table] + ): + """ + Make sure that for each column in table_1, there is a corresponding + column in table_2 of the same type. + """ + column_names = [ + column._meta.name for column in table_1._meta.non_default_columns + ] + for column_name in column_names: + col_1 = table_1._meta.get_column_by_name(column_name) + col_2 = table_2._meta.get_column_by_name(column_name) + + # Make sure they're the same type + self.assertEqual(type(col_1), type(col_2)) + + # Make sure they're both nullable or not + self.assertEqual(col_1._meta.null, col_2._meta.null) + + # Make sure the max length is the same + if isinstance(col_1, Varchar) and isinstance(col_2, Varchar): + self.assertEqual(col_1.length, col_2.length) + + # Make sure the unique constraint is the same + self.assertEqual(col_1._meta.unique, col_2._meta.unique) + + def test_reflect_all_tables(self): + run_sync(self.table_storage.reflect()) + reflected_tables = self.table_storage.tables + self.assertEqual(len(reflected_tables), 2) + for table_class in (Manager, Band): + self._compare_table_columns( + reflected_tables[table_class._meta.tablename], table_class + ) + + def test_reflect_with_include(self): + run_sync(self.table_storage.reflect(include=["manager"])) + reflected_tables = self.table_storage.tables + self.assertEqual(len(reflected_tables), 1) + self._compare_table_columns(reflected_tables["manager"], Manager) + + def test_reflect_with_exclude(self): + run_sync(self.table_storage.reflect(exclude=["band"])) + reflected_tables = self.table_storage.tables + self.assertEqual(len(reflected_tables), 1) + self._compare_table_columns(reflected_tables["manager"], Manager) + + def test_get_present_table(self): + run_sync(self.table_storage.reflect()) + table = run_sync(self.table_storage.get_table(tablename="manager")) + self._compare_table_columns(table, Manager) + + def test_get_unavailable_table(self): + run_sync(self.table_storage.reflect(exclude=["band"])) + # make sure only one table is present + self.assertEqual(len(self.table_storage.tables), 1) + table = run_sync(self.table_storage.get_table(tablename="band")) + # make sure the returned table is correct + self._compare_table_columns(table, Band) + # make sure the requested table has been added to the TableStorage + self.assertEqual(len(self.table_storage.tables), 2) + self.assertIsNotNone(self.table_storage.tables.get("band")) + + def test_get_schema_and_table_name(self): + tableNameDetail = self.table_storage._get_schema_and_table_name( + "music.manager" + ) + self.assertEqual(tableNameDetail.name, "manager") + self.assertEqual(tableNameDetail.schema, "music") From 427a3022dcf2136f991b97463ae5580822b2537f Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 30 Sep 2021 14:00:52 +0100 Subject: [PATCH 098/727] mention that table reflection just works for Postgres --- docs/src/piccolo/schema/advanced.rst | 2 +- piccolo/apps/schema/commands/generate.py | 8 ++++++-- piccolo/table_reflection.py | 3 ++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/src/piccolo/schema/advanced.rst b/docs/src/piccolo/schema/advanced.rst index 75c8b7714..42aea1b93 100644 --- a/docs/src/piccolo/schema/advanced.rst +++ b/docs/src/piccolo/schema/advanced.rst @@ -146,7 +146,7 @@ Reflection ---------- This is a very advanced feature, which is only required for specialist use -cases. +cases. Currently, just Postgres is supported. Instead of writing your ``Table`` definitions in a ``tables.py`` file, Piccolo can dynamically create them at run time, by inspecting the database. These diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index 79639604e..20a32ac11 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -715,9 +715,13 @@ class Schema(Table, db=engine): # We currently don't show the index argument for columns in the output, # so we don't need this import for now: - output_schema.imports.remove( + if ( "from piccolo.columns.indexes import IndexMethod" - ) + in output_schema.imports + ): + output_schema.imports.remove( + "from piccolo.columns.indexes import IndexMethod" + ) return output_schema diff --git a/piccolo/table_reflection.py b/piccolo/table_reflection.py index dc91d632c..78bfa98f4 100644 --- a/piccolo/table_reflection.py +++ b/piccolo/table_reflection.py @@ -75,7 +75,8 @@ class TableNameDetail: class TableStorage(metaclass=Singleton): """ - A singleton object to store and access reflected tables + A singleton object to store and access reflected tables. Currently it just + works with Postgres. """ def __init__(self): From bded0f62c6aa14ed4ac294cd560f56c6dd160f2e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 30 Sep 2021 14:04:42 +0100 Subject: [PATCH 099/727] bumped version --- CHANGES | 25 +++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index a502ceb6a..7fdc07d7b 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,31 @@ Changes ======= +0.53.0 +------ +An internal code clean up (courtesy @yezz123). + +Dramatically improved CLI appearance when running migrations (courtesy +@wmshort). + +Added a runtime reflection feature, where ``Table`` classes can be generated +on the fly from existing database tables (courtesy @AliSayyah). This is useful +when dealing with very dynamic databases, where tables are frequently being +added / modified, so hard coding them in a ``tables.py`` is impractical. Also, +for exploring databases on the command line. It currently just supports +Postgres. + +Here's an example: + +.. code-block:: python + + from piccolo.table_reflection import TableStorage + + storage = TableStorage() + Band = await storage.get_table('band') + >>> await Band.select().run() + [{'id': 1, 'name': 'Pythonistas', 'manager': 1}, ...] + 0.52.0 ------ Lots of improvements to ``piccolo schema generate``: diff --git a/piccolo/__init__.py b/piccolo/__init__.py index a05d6fb4e..7bd2e37d2 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.52.0" +__VERSION__ = "0.53.0" From ee551ff4c572ed4abd12884c2dff11b2ae0731b3 Mon Sep 17 00:00:00 2001 From: Guilherme Caminha Date: Mon, 4 Oct 2021 13:03:39 +0200 Subject: [PATCH 100/727] Changing Piccolo apps example to reflect its name/docstring (#287) --- docs/src/piccolo/projects_and_apps/piccolo_apps.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/piccolo/projects_and_apps/piccolo_apps.rst b/docs/src/piccolo/projects_and_apps/piccolo_apps.rst index 19a1b3347..580f1f11d 100644 --- a/docs/src/piccolo/projects_and_apps/piccolo_apps.rst +++ b/docs/src/piccolo/projects_and_apps/piccolo_apps.rst @@ -162,7 +162,7 @@ and docstrings. Here's an example: The person to greet. """ - print(name) + print("hello,", name) We then register it with the ``AppConfig``. @@ -180,7 +180,7 @@ And from the command line: .. code-block:: bash >>> piccolo my_app say_hello bob - bob + hello, bob If the code contains an error to see more details in the output add a ``--trace`` flag to the command line. From 2d5dad53426a8faa119eb0bdb4c432b2bfef5977 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 5 Oct 2021 09:48:36 +0100 Subject: [PATCH 101/727] prototype for custom column names (#288) * prototype for custom column names * wip * don't set db_column_name in metaclass * fix sqlite test * add `auto_input` option to `piccolo migrations new` * added test_function to migration integration tests * fix type error * fix migration tests for different postgres versions * add extra tests for db_column_name * update docs for db_column_name * add pydantic support * fix migrations when creating a new table with a db_column_name * add an additional test for aliases * add a unit test for sorting tables with foreign keys to themselves --- .../apps/migrations/auto/diffable_table.py | 8 +- .../apps/migrations/auto/migration_manager.py | 58 +++-- piccolo/apps/migrations/auto/operations.py | 5 + piccolo/apps/migrations/auto/schema_differ.py | 22 +- piccolo/apps/migrations/commands/new.py | 34 ++- piccolo/columns/base.py | 62 ++++- piccolo/columns/column_types.py | 34 +-- piccolo/query/methods/alter.py | 6 +- piccolo/query/methods/create_index.py | 3 +- piccolo/query/methods/insert.py | 6 +- piccolo/query/methods/update.py | 2 +- piccolo/table.py | 19 +- piccolo/utils/pydantic.py | 3 + .../auto/integration/test_migrations.py | 243 ++++++++++++++++-- .../migrations/auto/test_migration_manager.py | 16 ++ .../migrations/auto/test_schema_differ.py | 11 +- .../migrations/auto/test_serialisation.py | 4 +- tests/apps/schema/commands/test_generate.py | 9 +- tests/base.py | 24 +- tests/columns/test_db_column_name.py | 183 +++++++++++++ tests/table/test_str.py | 4 +- tests/utils/test_pydantic.py | 17 ++ 22 files changed, 663 insertions(+), 110 deletions(-) create mode 100644 tests/columns/test_db_column_name.py diff --git a/piccolo/apps/migrations/auto/diffable_table.py b/piccolo/apps/migrations/auto/diffable_table.py index 95775a9ca..318edfd29 100644 --- a/piccolo/apps/migrations/auto/diffable_table.py +++ b/piccolo/apps/migrations/auto/diffable_table.py @@ -74,7 +74,10 @@ def __hash__(self) -> int: def __eq__(self, value) -> bool: if isinstance(value, ColumnComparison): - return self.column._meta.name == value.column._meta.name + return ( + self.column._meta.db_column_name + == value.column._meta.db_column_name + ) return False @@ -110,6 +113,7 @@ def __sub__(self, value: DiffableTable) -> TableDelta: AddColumn( table_class_name=self.class_name, column_name=i.column._meta.name, + db_column_name=i.column._meta.db_column_name, column_class_name=i.column.__class__.__name__, column_class=i.column.__class__, params=i.column._meta.params, @@ -124,6 +128,7 @@ def __sub__(self, value: DiffableTable) -> TableDelta: DropColumn( table_class_name=self.class_name, column_name=i.column._meta.name, + db_column_name=i.column._meta.db_column_name, tablename=value.tablename, ) for i in ( @@ -156,6 +161,7 @@ def __sub__(self, value: DiffableTable) -> TableDelta: table_class_name=self.class_name, tablename=self.tablename, column_name=column._meta.name, + db_column_name=column._meta.db_column_name, params=deserialise_params(delta), old_params=old_params, column_class=column.__class__, diff --git a/piccolo/apps/migrations/auto/migration_manager.py b/piccolo/apps/migrations/auto/migration_manager.py index a95ecf33e..a3a11b014 100644 --- a/piccolo/apps/migrations/auto/migration_manager.py +++ b/piccolo/apps/migrations/auto/migration_manager.py @@ -188,6 +188,7 @@ def add_column( table_class_name: str, tablename: str, column_name: str, + db_column_name: t.Optional[str] = None, column_class_name: str = "", column_class: t.Optional[t.Type[Column]] = None, params: t.Dict[str, t.Any] = {}, @@ -212,6 +213,8 @@ def add_column( cleaned_params = deserialise_params(params=params) column = column_class(**cleaned_params) column._meta.name = column_name + column._meta.db_column_name = db_column_name + self.add_columns.append( AddColumnClass( column=column, @@ -221,12 +224,17 @@ def add_column( ) def drop_column( - self, table_class_name: str, tablename: str, column_name: str + self, + table_class_name: str, + tablename: str, + column_name: str, + db_column_name: t.Optional[str] = None, ): self.drop_columns.append( DropColumn( table_class_name=table_class_name, column_name=column_name, + db_column_name=db_column_name or column_name, tablename=tablename, ) ) @@ -237,6 +245,8 @@ def rename_column( tablename: str, old_column_name: str, new_column_name: str, + old_db_column_name: t.Optional[str] = None, + new_db_column_name: t.Optional[str] = None, ): self.rename_columns.append( RenameColumn( @@ -244,6 +254,8 @@ def rename_column( tablename=tablename, old_column_name=old_column_name, new_column_name=new_column_name, + old_db_column_name=old_db_column_name or old_column_name, + new_db_column_name=new_db_column_name or new_column_name, ) ) @@ -252,8 +264,9 @@ def alter_column( table_class_name: str, tablename: str, column_name: str, - params: t.Dict[str, t.Any], - old_params: t.Dict[str, t.Any], + db_column_name: t.Optional[str] = None, + params: t.Dict[str, t.Any] = {}, + old_params: t.Dict[str, t.Any] = {}, column_class: t.Optional[t.Type[Column]] = None, old_column_class: t.Optional[t.Type[Column]] = None, ): @@ -265,6 +278,7 @@ def alter_column( table_class_name=table_class_name, tablename=tablename, column_name=column_name, + db_column_name=db_column_name or column_name, params=params, old_params=old_params, column_class=column_class, @@ -371,10 +385,16 @@ async def _run_alter_columns(self, backwards=False): old_column = old_column_class(**old_params) old_column._meta._table = _Table old_column._meta._name = alter_column.column_name + old_column._meta.db_column_name = ( + alter_column.db_column_name + ) new_column = column_class(**params) new_column._meta._table = _Table new_column._meta._name = alter_column.column_name + new_column._meta.db_column_name = ( + alter_column.db_column_name + ) await _Table.alter().set_column_type( old_column=old_column, new_column=new_column @@ -382,18 +402,16 @@ async def _run_alter_columns(self, backwards=False): ############################################################### - column_name = alter_column.column_name - null = params.get("null") if null is not None: await _Table.alter().set_null( - column=column_name, boolean=null + column=alter_column.db_column_name, boolean=null ).run() length = params.get("length") if length is not None: await _Table.alter().set_length( - column=column_name, length=length + column=alter_column.db_column_name, length=length ).run() unique = params.get("unique") @@ -402,7 +420,8 @@ async def _run_alter_columns(self, backwards=False): # a column type, and not just the column name. column = Column() column._meta._table = _Table - column._meta._name = column_name + column._meta._name = alter_column.column_name + column._meta.db_column_name = alter_column.db_column_name await _Table.alter().set_unique( column=column, boolean=unique ).run() @@ -416,7 +435,10 @@ async def _run_alter_columns(self, backwards=False): # to change the index type. column = Column() column._meta._table = _Table - column._meta._name = column_name + column._meta._name = alter_column.column_name + column._meta.db_column_name = ( + alter_column.db_column_name + ) await _Table.drop_index([column]).run() await _Table.create_index( [column], method=index_method, if_not_exists=True @@ -426,7 +448,9 @@ async def _run_alter_columns(self, backwards=False): # dropping, or creating an index. column = Column() column._meta._table = _Table - column._meta._name = column_name + column._meta._name = alter_column.column_name + column._meta.db_column_name = alter_column.db_column_name + if index is True: kwargs = ( {"method": index_method} if index_method else {} @@ -442,7 +466,9 @@ async def _run_alter_columns(self, backwards=False): if default is not ...: column = Column() column._meta._table = _Table - column._meta._name = column_name + column._meta._name = alter_column.column_name + column._meta.db_column_name = alter_column.db_column_name + if default is None: await _Table.alter().drop_default(column=column).run() else: @@ -455,7 +481,7 @@ async def _run_alter_columns(self, backwards=False): digits = params.get("digits", ...) if digits is not ...: await _Table.alter().set_digits( - column=alter_column.column_name, + column=alter_column.db_column_name, digits=digits, ).run() @@ -547,14 +573,14 @@ async def _run_rename_columns(self, backwards=False): for rename_column in columns: column = ( - rename_column.new_column_name + rename_column.new_db_column_name if backwards - else rename_column.old_column_name + else rename_column.old_db_column_name ) new_name = ( - rename_column.old_column_name + rename_column.old_db_column_name if backwards - else rename_column.new_column_name + else rename_column.new_db_column_name ) await _Table.alter().rename_column( diff --git a/piccolo/apps/migrations/auto/operations.py b/piccolo/apps/migrations/auto/operations.py index 3da993e13..5903e1000 100644 --- a/piccolo/apps/migrations/auto/operations.py +++ b/piccolo/apps/migrations/auto/operations.py @@ -18,12 +18,15 @@ class RenameColumn: tablename: str old_column_name: str new_column_name: str + old_db_column_name: str + new_db_column_name: str @dataclass class AlterColumn: table_class_name: str column_name: str + db_column_name: str tablename: str params: t.Dict[str, t.Any] old_params: t.Dict[str, t.Any] @@ -35,6 +38,7 @@ class AlterColumn: class DropColumn: table_class_name: str column_name: str + db_column_name: str tablename: str @@ -42,6 +46,7 @@ class DropColumn: class AddColumn: table_class_name: str column_name: str + db_column_name: str column_class_name: str column_class: t.Type[Column] params: t.Dict[str, t.Any] diff --git a/piccolo/apps/migrations/auto/schema_differ.py b/piccolo/apps/migrations/auto/schema_differ.py index d8891dd32..2abb66b9e 100644 --- a/piccolo/apps/migrations/auto/schema_differ.py +++ b/piccolo/apps/migrations/auto/schema_differ.py @@ -123,9 +123,13 @@ def check_rename_tables(self) -> RenameTableCollection: # A renamed table should have at least one column remaining with the # same name. for new_table in new_tables: - new_column_names = [i._meta.name for i in new_table.columns] + new_column_names = [ + i._meta.db_column_name for i in new_table.columns + ] for drop_table in drop_tables: - drop_column_names = [i._meta.name for i in new_table.columns] + drop_column_names = [ + i._meta.db_column_name for i in new_table.columns + ] same_column_names = set(new_column_names).intersection( drop_column_names ) @@ -202,8 +206,8 @@ def check_renamed_columns(self) -> RenameColumnCollection: self.auto_input if self.auto_input else input( - f"Did you rename the `{drop_column.column_name}` " - f"column to `{add_column.column_name}` on the " + f"Did you rename the `{drop_column.db_column_name}` " # noqa: E501 + f"column to `{add_column.db_column_name}` on the " f"`{ add_column.table_class_name }` table? (y/N)" ) ) @@ -217,6 +221,8 @@ def check_renamed_columns(self) -> RenameColumnCollection: tablename=drop_column.tablename, old_column_name=drop_column.column_name, new_column_name=add_column.column_name, + old_db_column_name=drop_column.db_column_name, + new_db_column_name=add_column.db_column_name, ) ) @@ -374,7 +380,7 @@ def drop_columns(self) -> AlterStatements: continue response.append( - f"manager.drop_column(table_class_name='{table.class_name}', tablename='{table.tablename}', column_name='{column.column_name}')" # noqa: E501 + f"manager.drop_column(table_class_name='{table.class_name}', tablename='{table.tablename}', column_name='{column.column_name}', db_column_name='{column.db_column_name}')" # noqa: E501 ) return AlterStatements(statements=response) @@ -411,7 +417,7 @@ def add_columns(self) -> AlterStatements: ) response.append( - f"manager.add_column(table_class_name='{table.class_name}', tablename='{table.tablename}', column_name='{add_column.column_name}', column_class_name='{add_column.column_class_name}', column_class={column_class.__name__}, params={str(cleaned_params)})" # noqa: E501 + f"manager.add_column(table_class_name='{table.class_name}', tablename='{table.tablename}', column_name='{add_column.column_name}', db_column_name='{add_column.db_column_name}', column_class_name='{add_column.column_class_name}', column_class={column_class.__name__}, params={str(cleaned_params)})" # noqa: E501 ) return AlterStatements( statements=response, @@ -423,7 +429,7 @@ def add_columns(self) -> AlterStatements: def rename_columns(self) -> AlterStatements: return AlterStatements( statements=[ - f"manager.rename_column(table_class_name='{i.table_class_name}', tablename='{i.tablename}', old_column_name='{i.old_column_name}', new_column_name='{i.new_column_name}')" # noqa: E501 + f"manager.rename_column(table_class_name='{i.table_class_name}', tablename='{i.tablename}', old_column_name='{i.old_column_name}', new_column_name='{i.new_column_name}', old_db_column_name='{i.old_db_column_name}', new_db_column_name='{i.new_db_column_name}')" # noqa: E501 for i in self.rename_columns_collection.rename_columns ] ) @@ -462,7 +468,7 @@ def new_table_columns(self) -> AlterStatements: ) response.append( - f"manager.add_column(table_class_name='{table.class_name}', tablename='{table.tablename}', column_name='{column._meta.name}', column_class_name='{column.__class__.__name__}', column_class={column.__class__.__name__}, params={str(cleaned_params)})" # noqa: E501 + f"manager.add_column(table_class_name='{table.class_name}', tablename='{table.tablename}', column_name='{column._meta.name}', db_column_name='{column._meta.db_column_name}', column_class_name='{column.__class__.__name__}', column_class={column.__class__.__name__}, params={str(cleaned_params)})" # noqa: E501 ) return AlterStatements( statements=response, diff --git a/piccolo/apps/migrations/commands/new.py b/piccolo/apps/migrations/commands/new.py index 6179059bc..6a50608a5 100644 --- a/piccolo/apps/migrations/commands/new.py +++ b/piccolo/apps/migrations/commands/new.py @@ -88,7 +88,10 @@ class NoChanges(Exception): async def _create_new_migration( - app_config: AppConfig, auto: bool = False, description: str = "" + app_config: AppConfig, + auto: bool = False, + description: str = "", + auto_input: t.Optional[str] = None, ) -> NewMigrationMeta: """ Creates a new migration file on disk. @@ -96,9 +99,9 @@ async def _create_new_migration( meta = _generate_migration_meta(app_config=app_config) if auto: - alter_statements = await AutoMigrationManager().get_alter_statements( - app_config=app_config - ) + alter_statements = await AutoMigrationManager( + auto_input=auto_input + ).get_alter_statements(app_config=app_config) _alter_statements = list( chain(*[i.statements for i in alter_statements]) @@ -143,6 +146,10 @@ async def _create_new_migration( class AutoMigrationManager(BaseMigrationManager): + def __init__(self, auto_input: t.Optional[str] = None, *args, **kwargs): + self.auto_input = auto_input + super().__init__(*args, **kwargs) + async def get_alter_statements( self, app_config: AppConfig ) -> t.List[AlterStatements]: @@ -168,7 +175,9 @@ async def get_alter_statements( # Compare the current schema with the snapshot differ = SchemaDiffer( - schema=current_diffable_tables, schema_snapshot=snapshot + schema=current_diffable_tables, + schema_snapshot=snapshot, + auto_input=self.auto_input, ) return differ.get_alter_statements() @@ -176,7 +185,12 @@ async def get_alter_statements( ############################################################################### -async def new(app_name: str, auto: bool = False, desc: str = ""): +async def new( + app_name: str, + auto: bool = False, + desc: str = "", + auto_input: t.Optional[str] = None, +): """ Creates a new migration file in the migrations folder. @@ -187,6 +201,9 @@ async def new(app_name: str, auto: bool = False, desc: str = ""): :param desc: A description of what the migration does, for example 'adding name column'. + :param auto_input: + If provided, all prompts for user input will automatically have this + entered. For example, --auto_input='y'. """ print("Creating new migration ...") @@ -200,7 +217,10 @@ async def new(app_name: str, auto: bool = False, desc: str = ""): _create_migrations_folder(app_config.migrations_folder_path) try: await _create_new_migration( - app_config=app_config, auto=auto, description=desc + app_config=app_config, + auto=auto, + description=desc, + auto_input=auto_input, ) except NoChanges: print("No changes detected - exiting.") diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 56e671f72..fb4c017cb 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -131,14 +131,25 @@ class ColumnMeta: # Used for representing the table in migrations and the playground. params: t.Dict[str, t.Any] = field(default_factory=dict) + ########################################################################### + + # Lets you to map a column to a database column with a different name. + _db_column_name: t.Optional[str] = None + + @property + def db_column_name(self) -> str: + return self._db_column_name or self.name + + @db_column_name.setter + def db_column_name(self, value: str): + self._db_column_name = value + + ########################################################################### + # Set by the Table Metaclass: _name: t.Optional[str] = None _table: t.Optional[t.Type[Table]] = None - # Used by Foreign Keys: - call_chain: t.List["ForeignKey"] = field(default_factory=list) - table_alias: t.Optional[str] = None - @property def name(self) -> str: if not self._name: @@ -159,6 +170,12 @@ def table(self) -> t.Type[Table]: ) return self._table + ########################################################################### + + # Used by Foreign Keys: + call_chain: t.List["ForeignKey"] = field(default_factory=list) + table_alias: t.Optional[str] = None + @property def engine_type(self) -> str: engine = self.table._meta.db @@ -194,13 +211,16 @@ def get_full_name(self, just_alias=False) -> str: """ Returns the full column name, taking into account joins. """ - column_name = self.name + column_name = self.db_column_name if not self.call_chain: return f"{self.table._meta.tablename}.{column_name}" column_name = ( - "$".join(i._meta.name for i in self.call_chain) + f"${column_name}" + "$".join( + t.cast(str, i._meta.db_column_name) for i in self.call_chain + ) + + f"${column_name}" ) alias = f"{self.call_chain[-1]._meta.table_alias}.{self.name}" @@ -282,11 +302,32 @@ class Column(Selectable): :param help_text: This provides some context about what the column is being used for. For - example, for a `Decimal` column called `value`, it could say - 'The units are millions of dollars'. The database doesn't use this + example, for a ``Decimal`` column called ``value``, it could say + ``'The units are millions of dollars'``. The database doesn't use this value, but tools such as Piccolo Admin use it to show a tooltip in the GUI. + :param choices: + An optional Enum - when specified, other tools such as Piccolo Admin + will render the available options in the GUI. + + :param db_column_name: + If specified, you can override the name used for the column in the + database. The main reason for this is when using a legacy database, + with a problematic column name (for example ``'class'``, which is a + reserved Python keyword). Here's an example: + + .. code-block:: python + + class MyTable(Table): + class_ = Varchar(db_column_name="class") + + >>> MyTable.select(MyTable.class_).run_sync() + [{'id': 1, 'class': 'test'}] + + This is an advanced feature which you should only need in niche + situations. + """ value_type: t.Type = int @@ -301,6 +342,7 @@ def __init__( required: bool = False, help_text: t.Optional[str] = None, choices: t.Optional[t.Type[Enum]] = None, + db_column_name: t.Optional[str] = None, **kwargs, ) -> None: # This is for backwards compatibility - originally there were two @@ -322,6 +364,7 @@ def __init__( "index": index, "index_method": index_method, "choices": choices, + "db_column_name": db_column_name, } ) @@ -344,6 +387,7 @@ def __init__( required=required, help_text=help_text, choices=choices, + _db_column_name=db_column_name, ) self.alias: t.Optional[str] = None @@ -595,7 +639,7 @@ def ddl(self) -> str: """ Used when creating tables. """ - query = f'"{self._meta.name}" {self.column_type}' + query = f'"{self._meta.db_column_name}" {self.column_type}' if self._meta.primary_key: query += " PRIMARY KEY" if self._meta.unique: diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 17340e1ed..d7c300f3c 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -61,7 +61,7 @@ def get_querystring( raise ValueError( "Adding values across joins isn't currently supported." ) - other_column_name = column._meta.name + other_column_name = column._meta.db_column_name if reverse: return QueryString( Concat.template.format( @@ -113,7 +113,7 @@ def get_querystring( raise ValueError( "Adding values across joins isn't currently supported." ) - column_name = column._meta.name + column_name = column._meta.db_column_name if reverse: return QueryString(f"{column_name} {operator} {column_name}") else: @@ -183,7 +183,7 @@ def column_type(self): def __add__(self, value: t.Union[str, Varchar, Text]) -> QueryString: engine_type = self._meta.table._meta.db.engine_type return self.concat_delegate.get_querystring( - column_name=self._meta.name, + column_name=self._meta.db_column_name, value=value, engine_type=engine_type, ) @@ -191,7 +191,7 @@ def __add__(self, value: t.Union[str, Varchar, Text]) -> QueryString: def __radd__(self, value: t.Union[str, Varchar, Text]) -> QueryString: engine_type = self._meta.table._meta.db.engine_type return self.concat_delegate.get_querystring( - column_name=self._meta.name, + column_name=self._meta.db_column_name, value=value, engine_type=engine_type, reverse=True, @@ -263,13 +263,15 @@ def __init__( def __add__(self, value: t.Union[str, Varchar, Text]) -> QueryString: engine_type = self._meta.table._meta.db.engine_type return self.concat_delegate.get_querystring( - column_name=self._meta.name, value=value, engine_type=engine_type + column_name=self._meta.db_column_name, + value=value, + engine_type=engine_type, ) def __radd__(self, value: t.Union[str, Varchar, Text]) -> QueryString: engine_type = self._meta.table._meta.db.engine_type return self.concat_delegate.get_querystring( - column_name=self._meta.name, + column_name=self._meta.db_column_name, value=value, engine_type=engine_type, reverse=True, @@ -358,12 +360,12 @@ def __init__( def __add__(self, value: t.Union[int, float, Integer]) -> QueryString: return self.math_delegate.get_querystring( - column_name=self._meta.name, operator="+", value=value + column_name=self._meta.db_column_name, operator="+", value=value ) def __radd__(self, value: t.Union[int, float, Integer]) -> QueryString: return self.math_delegate.get_querystring( - column_name=self._meta.name, + column_name=self._meta.db_column_name, operator="+", value=value, reverse=True, @@ -371,12 +373,12 @@ def __radd__(self, value: t.Union[int, float, Integer]) -> QueryString: def __sub__(self, value: t.Union[int, float, Integer]) -> QueryString: return self.math_delegate.get_querystring( - column_name=self._meta.name, operator="-", value=value + column_name=self._meta.db_column_name, operator="-", value=value ) def __rsub__(self, value: t.Union[int, float, Integer]) -> QueryString: return self.math_delegate.get_querystring( - column_name=self._meta.name, + column_name=self._meta.db_column_name, operator="-", value=value, reverse=True, @@ -384,12 +386,12 @@ def __rsub__(self, value: t.Union[int, float, Integer]) -> QueryString: def __mul__(self, value: t.Union[int, float, Integer]) -> QueryString: return self.math_delegate.get_querystring( - column_name=self._meta.name, operator="*", value=value + column_name=self._meta.db_column_name, operator="*", value=value ) def __rmul__(self, value: t.Union[int, float, Integer]) -> QueryString: return self.math_delegate.get_querystring( - column_name=self._meta.name, + column_name=self._meta.db_column_name, operator="*", value=value, reverse=True, @@ -397,12 +399,12 @@ def __rmul__(self, value: t.Union[int, float, Integer]) -> QueryString: def __truediv__(self, value: t.Union[int, float, Integer]) -> QueryString: return self.math_delegate.get_querystring( - column_name=self._meta.name, operator="/", value=value + column_name=self._meta.db_column_name, operator="/", value=value ) def __rtruediv__(self, value: t.Union[int, float, Integer]) -> QueryString: return self.math_delegate.get_querystring( - column_name=self._meta.name, + column_name=self._meta.db_column_name, operator="/", value=value, reverse=True, @@ -410,14 +412,14 @@ def __rtruediv__(self, value: t.Union[int, float, Integer]) -> QueryString: def __floordiv__(self, value: t.Union[int, float, Integer]) -> QueryString: return self.math_delegate.get_querystring( - column_name=self._meta.name, operator="/", value=value + column_name=self._meta.db_column_name, operator="/", value=value ) def __rfloordiv__( self, value: t.Union[int, float, Integer] ) -> QueryString: return self.math_delegate.get_querystring( - column_name=self._meta.name, + column_name=self._meta.db_column_name, operator="/", value=value, reverse=True, diff --git a/piccolo/query/methods/alter.py b/piccolo/query/methods/alter.py index 442f11587..ca6fb2bf5 100644 --- a/piccolo/query/methods/alter.py +++ b/piccolo/query/methods/alter.py @@ -47,7 +47,7 @@ def column_name(self) -> str: if isinstance(self.column, str): return self.column elif isinstance(self.column, Column): - return self.column._meta.name + return self.column._meta.db_column_name else: raise ValueError("Unrecognised column type") @@ -109,7 +109,7 @@ def ddl(self) -> str: if self.new_column._meta._table is None: self.new_column._meta._table = self.old_column._meta.table - column_name = self.old_column._meta.name + column_name = self.old_column._meta.db_column_name query = ( f"ALTER COLUMN {column_name} TYPE {self.new_column.column_type}" ) @@ -303,6 +303,8 @@ def add_column(self, name: str, column: Column) -> Alter: Band.alter().add_column(‘members’, Integer()) """ column._meta._table = self.table + column._meta._name = name + column._meta.db_column_name = name self._add.append(AddColumn(column, name)) return self diff --git a/piccolo/query/methods/create_index.py b/piccolo/query/methods/create_index.py index 7b6e93d99..4a1822ccb 100644 --- a/piccolo/query/methods/create_index.py +++ b/piccolo/query/methods/create_index.py @@ -27,7 +27,8 @@ def __init__( @property def column_names(self) -> t.List[str]: return [ - i._meta.name if isinstance(i, Column) else i for i in self.columns + i._meta.db_column_name if isinstance(i, Column) else i + for i in self.columns ] @property diff --git a/piccolo/query/methods/insert.py b/piccolo/query/methods/insert.py index 0dc3cdae8..84e3ad1ee 100644 --- a/piccolo/query/methods/insert.py +++ b/piccolo/query/methods/insert.py @@ -40,7 +40,9 @@ def run_callback(self, results): @property def sqlite_querystrings(self) -> t.Sequence[QueryString]: base = f"INSERT INTO {self.table._meta.tablename}" - columns = ",".join(i._meta.name for i in self.table._meta.columns) + columns = ",".join( + i._meta.db_column_name for i in self.table._meta.columns + ) values = ",".join("{}" for _ in self.add_delegate._add) query = f"{base} ({columns}) VALUES {values}" return [ @@ -56,7 +58,7 @@ def sqlite_querystrings(self) -> t.Sequence[QueryString]: def postgres_querystrings(self) -> t.Sequence[QueryString]: base = f"INSERT INTO {self.table._meta.tablename}" columns = ",".join( - f'"{i._meta.name}"' for i in self.table._meta.columns + f'"{i._meta.db_column_name}"' for i in self.table._meta.columns ) values = ",".join("{}" for i in self.add_delegate._add) primary_key_name = self.table._meta.primary_key._meta.name diff --git a/piccolo/query/methods/update.py b/piccolo/query/methods/update.py index d90f61295..db9017e08 100644 --- a/piccolo/query/methods/update.py +++ b/piccolo/query/methods/update.py @@ -51,7 +51,7 @@ def default_querystrings(self) -> t.Sequence[QueryString]: self.validate() columns_str = ", ".join( - f"{col._meta.name} = {{}}" + f"{col._meta.db_column_name} = {{}}" for col, _ in self.values_delegate._values.items() ) diff --git a/piccolo/table.py b/piccolo/table.py index fc8c581f3..05794bdb8 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -296,6 +296,12 @@ def __init__( for column in self._meta.columns: value = kwargs.pop(column._meta.name, ...) + + if value is ...: + value = kwargs.pop( + t.cast(str, column._meta.db_column_name), ... + ) + if value is ...: value = column.get_default_value() @@ -318,7 +324,7 @@ def __init__( @classmethod def _create_serial_primary_key(cls) -> Serial: - pk = Serial(index=False, primary_key=True) + pk = Serial(index=False, primary_key=True, db_column_name="id") pk._meta._name = "id" pk._meta._table = cls @@ -1035,13 +1041,16 @@ def _get_graph( for table_class in table_classes: dependents: t.Set[str] = set() for fk in table_class._meta.foreign_key_columns: - dependents.add( - fk._foreign_key_meta.resolved_references._meta.tablename - ) + referenced_table = fk._foreign_key_meta.resolved_references + + if referenced_table._meta.tablename == table_class._meta.tablename: + # Most like a recursive link (using ForeignKey('self')). + continue + + dependents.add(referenced_table._meta.tablename) # We also recursively check the related tables to get a fuller # picture of the schema and relationships. - referenced_table = fk._foreign_key_meta.resolved_references output.update( _get_graph( [referenced_table], diff --git a/piccolo/utils/pydantic.py b/piccolo/utils/pydantic.py index fd7698340..90c2fe17d 100644 --- a/piccolo/utils/pydantic.py +++ b/piccolo/utils/pydantic.py @@ -145,6 +145,9 @@ def create_pydantic_model( "nullable": column._meta.null, } + if column._meta.db_column_name != column._meta.name: + params["alias"] = column._meta.db_column_name + extra = { "help_text": column._meta.help_text, "choices": column._meta.get_choices_dict(), diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index a9f0ff013..d1fa51fcf 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -7,7 +7,6 @@ import time import typing as t import uuid -from unittest import TestCase from piccolo.apps.migrations.commands.forwards import ForwardsMigrationManager from piccolo.apps.migrations.commands.new import ( @@ -15,6 +14,7 @@ _create_new_migration, ) from piccolo.apps.migrations.tables import Migration +from piccolo.apps.schema.commands.generate import RowMeta from piccolo.columns.column_types import ( JSON, JSONB, @@ -37,7 +37,7 @@ from piccolo.conf.apps import AppConfig from piccolo.table import Table, create_table_class from piccolo.utils.sync import run_sync -from tests.base import postgres_only +from tests.base import DBTestCase, postgres_only if t.TYPE_CHECKING: from piccolo.columns.base import Column @@ -84,7 +84,10 @@ def array_default_varchar(): @postgres_only -class TestMigrations(TestCase): +class TestMigrations(DBTestCase): + def setUp(self): + pass + def tearDown(self): create_table_class("MyTable").alter().drop_table( if_exists=True @@ -96,7 +99,22 @@ def run_migrations(self, app_config: AppConfig): run_sync(manager.create_migration_table()) run_sync(manager.run_migrations(app_config=app_config)) - def _test_migrations(self, table_classes: t.List[t.Type[Table]]): + def _test_migrations( + self, + table_classes: t.List[t.Type[Table]], + test_function: t.Optional[t.Callable[[RowMeta], None]] = None, + ): + """ + :param table_classes: + Migrations will be created and run based on successive table + classes in this list. + :param test_function: + After the migrations are run, this function is called. It is passed + a ``RowMeta`` instance which can be used to check the column was + created correctly in the database. It should return ``True`` if the + test passes, otherwise ``False``. + + """ temp_directory_path = tempfile.gettempdir() migrations_folder_path = os.path.join( temp_directory_path, "piccolo_migrations" @@ -116,7 +134,9 @@ def _test_migrations(self, table_classes: t.List[t.Type[Table]]): for table_class in table_classes: app_config.table_classes = [table_class] meta = run_sync( - _create_new_migration(app_config=app_config, auto=True) + _create_new_migration( + app_config=app_config, auto=True, auto_input="y" + ) ) self.assertTrue(os.path.exists(meta.migration_path)) self.run_migrations(app_config=app_config) @@ -126,7 +146,20 @@ def _test_migrations(self, table_classes: t.List[t.Type[Table]]): # and / or Python get insanely fast in the future :) time.sleep(1e-6) - # TODO - check the migrations ran correctly + if test_function: + column_name = ( + table_classes[-1] + ._meta.non_default_columns[0] + ._meta.db_column_name + ) + row_meta = self.get_postgres_column_definition( + tablename="my_table", + column_name=column_name, + ) + self.assertTrue( + test_function(row_meta), + msg=f"Meta is incorrect: {row_meta}", + ) ########################################################################### @@ -149,7 +182,14 @@ def test_varchar_column(self): Varchar(index=True), Varchar(index=False), ] - ] + ], + test_function=lambda x: all( + [ + x.data_type == "character varying", + x.is_nullable == "NO", + x.column_default == "''::character varying", + ] + ), ) def test_text_column(self): @@ -165,7 +205,14 @@ def test_text_column(self): Text(index=True), Text(index=False), ] - ] + ], + test_function=lambda x: all( + [ + x.data_type == "text", + x.is_nullable == "NO", + x.column_default == "''::text", + ] + ), ) def test_integer_column(self): @@ -181,7 +228,14 @@ def test_integer_column(self): Integer(index=True), Integer(index=False), ] - ] + ], + test_function=lambda x: all( + [ + x.data_type == "integer", + x.is_nullable == "NO", + x.column_default == "0", + ] + ), ) def test_real_column(self): @@ -196,7 +250,14 @@ def test_real_column(self): Real(index=True), Real(index=False), ] - ] + ], + test_function=lambda x: all( + [ + x.data_type == "real", + x.is_nullable == "NO", + x.column_default == "0.0", + ] + ), ) def test_double_precision_column(self): @@ -211,7 +272,14 @@ def test_double_precision_column(self): DoublePrecision(index=True), DoublePrecision(index=False), ] - ] + ], + test_function=lambda x: all( + [ + x.data_type == "double precision", + x.is_nullable == "NO", + x.column_default == "0.0", + ] + ), ) def test_smallint_column(self): @@ -227,7 +295,14 @@ def test_smallint_column(self): SmallInt(index=True), SmallInt(index=False), ] - ] + ], + test_function=lambda x: all( + [ + x.data_type == "smallint", + x.is_nullable == "NO", + x.column_default == "0", + ] + ), ) def test_bigint_column(self): @@ -243,7 +318,14 @@ def test_bigint_column(self): BigInt(index=True), BigInt(index=False), ] - ] + ], + test_function=lambda x: all( + [ + x.data_type == "bigint", + x.is_nullable == "NO", + x.column_default == "0", + ] + ), ) def test_uuid_column(self): @@ -266,7 +348,14 @@ def test_uuid_column(self): UUID(index=True), UUID(index=False), ] - ] + ], + test_function=lambda x: all( + [ + x.data_type == "uuid", + x.is_nullable == "NO", + x.column_default == "uuid_generate_v4()", + ] + ), ) def test_timestamp_column(self): @@ -285,7 +374,14 @@ def test_timestamp_column(self): Timestamp(index=True), Timestamp(index=False), ] - ] + ], + test_function=lambda x: all( + [ + x.data_type == "timestamp without time zone", + x.is_nullable == "NO", + x.column_default in ("now()", "CURRENT_TIMESTAMP"), + ] + ), ) def test_time_column(self): @@ -302,7 +398,15 @@ def test_time_column(self): Time(index=True), Time(index=False), ] - ] + ], + test_function=lambda x: all( + [ + x.data_type == "time without time zone", + x.is_nullable == "NO", + x.column_default + in ("('now'::text)::time with time zone", "CURRENT_TIME"), + ] + ), ) def test_date_column(self): @@ -319,7 +423,15 @@ def test_date_column(self): Date(index=True), Date(index=False), ] - ] + ], + test_function=lambda x: all( + [ + x.data_type == "date", + x.is_nullable == "NO", + x.column_default + in ("('now'::text)::date", "CURRENT_DATE"), + ] + ), ) def test_interval_column(self): @@ -335,7 +447,14 @@ def test_interval_column(self): Interval(index=True), Interval(index=False), ] - ] + ], + test_function=lambda x: all( + [ + x.data_type == "interval", + x.is_nullable == "NO", + x.column_default == "'00:00:00'::interval", + ] + ), ) def test_boolean_column(self): @@ -351,7 +470,14 @@ def test_boolean_column(self): Boolean(index=True), Boolean(index=False), ] - ] + ], + test_function=lambda x: all( + [ + x.data_type == "boolean", + x.is_nullable == "NO", + x.column_default == "false", + ] + ), ) def test_array_column_integer(self): @@ -369,7 +495,14 @@ def test_array_column_integer(self): Array(base_column=Integer(), index=True), Array(base_column=Integer(), index=False), ] - ] + ], + test_function=lambda x: all( + [ + x.data_type == "ARRAY", + x.is_nullable == "NO", + x.column_default == "'{}'::integer[]", + ] + ), ) def test_array_column_varchar(self): @@ -387,7 +520,14 @@ def test_array_column_varchar(self): Array(base_column=Varchar(), index=True), Array(base_column=Varchar(), index=False), ] - ] + ], + test_function=lambda x: all( + [ + x.data_type == "ARRAY", + x.is_nullable == "NO", + x.column_default == "'{}'::character varying[]", + ] + ), ) ########################################################################### @@ -407,7 +547,14 @@ def test_json_column(self): JSON(null=True, default=None), JSON(null=False), ] - ] + ], + test_function=lambda x: all( + [ + x.data_type == "json", + x.is_nullable == "NO", + x.column_default == "'{}'::json", + ] + ), ) def test_jsonb_column(self): @@ -422,5 +569,55 @@ def test_jsonb_column(self): JSONB(null=True, default=None), JSONB(null=False), ] - ] + ], + test_function=lambda x: all( + [ + x.data_type == "jsonb", + x.is_nullable == "NO", + x.column_default == "'{}'::jsonb", + ] + ), + ) + + ########################################################################### + + def test_db_column_name(self): + self._test_migrations( + table_classes=[ + self.table(column) + for column in [ + Varchar(), + Varchar(db_column_name="custom_name"), + Varchar(), + Varchar(db_column_name="custom_name_2"), + ] + ], + test_function=lambda x: all( + [ + x.data_type == "character varying", + x.is_nullable == "NO", + x.column_default == "''::character varying", + ] + ), + ) + + def test_db_column_name_initial(self): + """ + Make sure that if a new table is created which contains a column with + ``db_column_name`` specified, then the column has the correct name. + """ + self._test_migrations( + table_classes=[ + self.table(column) + for column in [ + Varchar(db_column_name="custom_name"), + ] + ], + test_function=lambda x: all( + [ + x.data_type == "character varying", + x.is_nullable == "NO", + x.column_default == "''::character varying", + ] + ), ) diff --git a/tests/apps/migrations/auto/test_migration_manager.py b/tests/apps/migrations/auto/test_migration_manager.py index af3e5625c..f3fd1f47a 100644 --- a/tests/apps/migrations/auto/test_migration_manager.py +++ b/tests/apps/migrations/auto/test_migration_manager.py @@ -55,6 +55,22 @@ def test_single_table(self): """ self.assertEqual(sort_table_classes([Band]), [Band]) + def test_recursive_table(self): + """ + Make sure that a table with a foreign key to itself sorts without + issues. + """ + + class TableA(Table): + table_a = ForeignKey("self") + + class TableB(Table): + table_a = ForeignKey(TableA) + + self.assertEqual( + sort_table_classes([TableA, TableB]), [TableA, TableB] + ) + class TestMigrationManager(DBTestCase): @postgres_only diff --git a/tests/apps/migrations/auto/test_schema_differ.py b/tests/apps/migrations/auto/test_schema_differ.py index 8a2f03de5..a0b0d93e2 100644 --- a/tests/apps/migrations/auto/test_schema_differ.py +++ b/tests/apps/migrations/auto/test_schema_differ.py @@ -8,6 +8,9 @@ class TestSchemaDiffer(TestCase): + + maxDiff = None + def test_add_table(self): """ Test adding a new table. @@ -35,7 +38,7 @@ def test_add_table(self): self.assertTrue(len(new_table_columns.statements) == 1) self.assertEqual( new_table_columns.statements[0], - "manager.add_column(table_class_name='Band', tablename='band', column_name='name', column_class_name='Varchar', column_class=Varchar, params={'length': 255, 'default': '', 'null': False, 'primary_key': False, 'unique': False, 'index': False, 'index_method': IndexMethod.btree, 'choices': None})", # noqa + "manager.add_column(table_class_name='Band', tablename='band', column_name='name', db_column_name='name', column_class_name='Varchar', column_class=Varchar, params={'length': 255, 'default': '', 'null': False, 'primary_key': False, 'unique': False, 'index': False, 'index_method': IndexMethod.btree, 'choices': None, 'db_column_name': None})", # noqa ) def test_drop_table(self): @@ -119,7 +122,7 @@ def test_add_column(self): self.assertTrue(len(schema_differ.add_columns.statements) == 1) self.assertEqual( schema_differ.add_columns.statements[0], - "manager.add_column(table_class_name='Band', tablename='band', column_name='genre', column_class_name='Varchar', column_class=Varchar, params={'length': 255, 'default': '', 'null': False, 'primary_key': False, 'unique': False, 'index': False, 'index_method': IndexMethod.btree, 'choices': None})", # noqa + "manager.add_column(table_class_name='Band', tablename='band', column_name='genre', db_column_name='genre', column_class_name='Varchar', column_class=Varchar, params={'length': 255, 'default': '', 'null': False, 'primary_key': False, 'unique': False, 'index': False, 'index_method': IndexMethod.btree, 'choices': None, 'db_column_name': None})", # noqa ) def test_drop_column(self): @@ -154,7 +157,7 @@ def test_drop_column(self): self.assertTrue(len(schema_differ.drop_columns.statements) == 1) self.assertEqual( schema_differ.drop_columns.statements[0], - "manager.drop_column(table_class_name='Band', tablename='band', column_name='genre')", # noqa + "manager.drop_column(table_class_name='Band', tablename='band', column_name='genre', db_column_name='genre')", # noqa ) def test_rename_column(self): @@ -189,7 +192,7 @@ def test_rename_column(self): self.assertTrue(len(schema_differ.rename_columns.statements) == 1) self.assertEqual( schema_differ.rename_columns.statements[0], - "manager.rename_column(table_class_name='Band', tablename='band', old_column_name='title', new_column_name='name')", # noqa + "manager.rename_column(table_class_name='Band', tablename='band', old_column_name='title', new_column_name='name', old_db_column_name='title', new_db_column_name='name')", # noqa ) def test_alter_column_precision(self): diff --git a/tests/apps/migrations/auto/test_serialisation.py b/tests/apps/migrations/auto/test_serialisation.py index c97416920..839e17bfd 100644 --- a/tests/apps/migrations/auto/test_serialisation.py +++ b/tests/apps/migrations/auto/test_serialisation.py @@ -68,7 +68,7 @@ def test_lazy_table_reference(self): 'class Manager(Table, tablename="manager"): ' "id = Serial(null=False, primary_key=True, unique=False, " "index=False, index_method=IndexMethod.btree, " - "choices=None)" + "choices=None, db_column_name='id')" ), ) @@ -119,7 +119,7 @@ def test_column_instance(self): self.assertEqual( serialised.params["base_column"].__repr__(), - "Varchar(length=255, default='', null=False, primary_key=False, unique=False, index=False, index_method=IndexMethod.btree, choices=None)", # noqa: E501 + "Varchar(length=255, default='', null=False, primary_key=False, unique=False, index=False, index_method=IndexMethod.btree, choices=None, db_column_name=None)", # noqa: E501 ) self.assertEqual( diff --git a/tests/apps/schema/commands/test_generate.py b/tests/apps/schema/commands/test_generate.py index 75989ed1e..e03ee3ecf 100644 --- a/tests/apps/schema/commands/test_generate.py +++ b/tests/apps/schema/commands/test_generate.py @@ -111,18 +111,21 @@ class Box(Column): pass - MegaTable.alter().add_column("box", Box()).run_sync() + MegaTable.alter().add_column("my_column", Box()).run_sync() output_schema: OutputSchema = run_sync(get_output_schema()) # Make sure there's a warning. - self.assertEqual(output_schema.warnings, ["mega_table.box ['box']"]) + self.assertEqual( + output_schema.warnings, ["mega_table.my_column ['box']"] + ) # Make sure the column type of the generated table is just ``Column``. for table in output_schema.tables: if table.__name__ == "MegaTable": self.assertEqual( - output_schema.tables[1].box.__class__.__name__, "Column" + output_schema.tables[1].my_column.__class__.__name__, + "Column", ) def test_generate_required_tables(self): diff --git a/tests/base.py b/tests/base.py index bfe566d24..240151073 100644 --- a/tests/base.py +++ b/tests/base.py @@ -6,6 +6,7 @@ import pytest +from piccolo.apps.schema.commands.generate import RowMeta from piccolo.engine.finder import engine_finder from piccolo.engine.postgres import PostgresEngine from piccolo.engine.sqlite import SQLiteEngine @@ -62,17 +63,22 @@ def table_exists(self, tablename: str) -> bool: def get_postgres_column_definition( self, tablename: str, column_name: str - ) -> t.Dict[str, t.Any]: + ) -> RowMeta: query = """ - SELECT * FROM information_schema.columns + SELECT {columns} FROM information_schema.columns WHERE table_name = '{tablename}' AND table_catalog = 'piccolo' AND column_name = '{column_name}' """.format( - tablename=tablename, column_name=column_name + columns=RowMeta.get_column_name_str(), + tablename=tablename, + column_name=column_name, ) response = self.run_sync(query) - return response[0] + if len(response) > 0: + return RowMeta(**response[0]) + else: + raise ValueError("No such column") def get_postgres_column_type( self, tablename: str, column_name: str @@ -82,7 +88,7 @@ def get_postgres_column_type( """ return self.get_postgres_column_definition( tablename=tablename, column_name=column_name - )["data_type"].upper() + ).data_type.upper() def get_postgres_is_nullable(self, tablename, column_name: str) -> bool: """ @@ -91,17 +97,19 @@ def get_postgres_is_nullable(self, tablename, column_name: str) -> bool: return ( self.get_postgres_column_definition( tablename=tablename, column_name=column_name - )["is_nullable"].upper() + ).is_nullable.upper() == "YES" ) - def get_postgres_varchar_length(self, tablename, column_name: str) -> int: + def get_postgres_varchar_length( + self, tablename, column_name: str + ) -> t.Optional[int]: """ Fetches whether the column is defined as nullable, from the database. """ return self.get_postgres_column_definition( tablename=tablename, column_name=column_name - )["character_maximum_length"] + ).character_maximum_length ########################################################################### diff --git a/tests/columns/test_db_column_name.py b/tests/columns/test_db_column_name.py new file mode 100644 index 000000000..cf1bceaf7 --- /dev/null +++ b/tests/columns/test_db_column_name.py @@ -0,0 +1,183 @@ +from piccolo.columns.column_types import Integer, Varchar +from piccolo.table import Table +from tests.base import DBTestCase, postgres_only + + +class Band(Table): + name = Varchar(db_column_name="regrettable_column_name") + popularity = Integer() + + +class TestDBColumnName(DBTestCase): + """ + By using the ``db_column_name`` arg, the user can map a ``Column`` to a + database column with a different name. For example: + + .. code-block:: + + class MyTable(Table): + class_ = Varchar(db_column_name='class') + + """ + + def setUp(self): + Band.create_table().run_sync() + + def tearDown(self): + Band.alter().drop_table().run_sync() + + @postgres_only + def test_column_name_correct(self): + """ + Make sure the column has the correct name in the database. + """ + self.get_postgres_column_definition( + tablename="band", column_name="regrettable_column_name" + ) + + with self.assertRaises(ValueError): + self.get_postgres_column_definition( + tablename="band", column_name="name" + ) + + def test_save(self): + """ + Make sure save queries work correctly. + """ + band = Band(name="Pythonistas", popularity=1000) + band.save().run_sync() + + band_from_db = Band.objects().first().run_sync() + self.assertTrue(band_from_db.name == "Pythonistas") + + def test_create(self): + """ + Make sure create queries work correctly. + """ + band = ( + Band.objects() + .create(name="Pythonistas", popularity=1000) + .run_sync() + ) + self.assertTrue(band.name == "Pythonistas") + + band_from_db = Band.objects().first().run_sync() + self.assertTrue(band_from_db.name == "Pythonistas") + + def test_select(self): + """ + Make sure that select queries just return what is stored in the + database. We might add an option in the future which maps the column + name to it's alias, but it's hard to predict what behaviour the user + wants. + """ + Band.objects().create(name="Pythonistas", popularity=1000).run_sync() + + # Make sure we can select all columns + bands = Band.select().run_sync() + self.assertEqual( + bands, + [ + { + "id": 1, + "regrettable_column_name": "Pythonistas", + "popularity": 1000, + } + ], + ) + + # Make sure we can select a single column + bands = Band.select(Band.name).run_sync() + self.assertEqual( + bands, + [ + { + "regrettable_column_name": "Pythonistas", + } + ], + ) + + # Make sure aliases still work + bands = Band.select(Band.name.as_alias("name")).run_sync() + self.assertEqual( + bands, + [ + { + "name": "Pythonistas", + } + ], + ) + + def test_update(self): + """ + Make sure update queries work correctly. + """ + Band.objects().create(name="Pythonistas", popularity=1000).run_sync() + + Band.update({Band.name: "Pythonistas 2"}).run_sync() + + bands = Band.select().run_sync() + self.assertEqual( + bands, + [ + { + "id": 1, + "regrettable_column_name": "Pythonistas 2", + "popularity": 1000, + } + ], + ) + + Band.update({"name": "Pythonistas 3"}).run_sync() + + bands = Band.select().run_sync() + self.assertEqual( + bands, + [ + { + "id": 1, + "regrettable_column_name": "Pythonistas 3", + "popularity": 1000, + } + ], + ) + + def test_delete(self): + """ + Make sure delete queries work correctly. + """ + Band.insert( + Band(name="Pythonistas", popularity=1000), + Band(name="Rustaceans", popularity=500), + ).run_sync() + + bands = Band.select().run_sync() + self.assertEqual( + bands, + [ + { + "id": 1, + "regrettable_column_name": "Pythonistas", + "popularity": 1000, + }, + { + "id": 2, + "regrettable_column_name": "Rustaceans", + "popularity": 500, + }, + ], + ) + + Band.delete().where(Band.name == "Rustaceans").run_sync() + + bands = Band.select().run_sync() + self.assertEqual( + bands, + [ + { + "id": 1, + "regrettable_column_name": "Pythonistas", + "popularity": 1000, + } + ], + ) diff --git a/tests/table/test_str.py b/tests/table/test_str.py index 82325e098..f7f296f20 100644 --- a/tests/table/test_str.py +++ b/tests/table/test_str.py @@ -9,8 +9,8 @@ def test_str(self): Manager._table_str(), ( "class Manager(Table, tablename='manager'):\n" - " id = Serial(null=False, primary_key=True, unique=False, index=False, index_method=IndexMethod.btree, choices=None)\n" # noqa: E501 - " name = Varchar(length=50, default='', null=False, primary_key=False, unique=False, index=False, index_method=IndexMethod.btree, choices=None)\n" # noqa: E501 + " id = Serial(null=False, primary_key=True, unique=False, index=False, index_method=IndexMethod.btree, choices=None, db_column_name='id')\n" # noqa: E501 + " name = Varchar(length=50, default='', null=False, primary_key=False, unique=False, index=False, index_method=IndexMethod.btree, choices=None, db_column_name=None)\n" # noqa: E501 ), ) diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index a0339aa39..01cfbcb77 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -352,3 +352,20 @@ class Movie(Table): country_model_keys = [i for i in CountryModel.__fields__.keys()] self.assertEqual(country_model_keys, ["id", "name"]) + + +class TestDBColumnName(TestCase): + def test_db_column_name(self): + """ + Make sure that the Pydantic model has an alias if ``db_column_name`` + is specified for a column. + """ + + class Band(Table): + name = Varchar(db_column_name="regrettable_column_name") + + BandModel = create_pydantic_model(table=Band) + + model = BandModel(regrettable_column_name="test") + + self.assertTrue(model.name == "test") From 5a5ef2621b408728a861c3072e7107c3808a7a57 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 5 Oct 2021 10:06:12 +0100 Subject: [PATCH 102/727] bumped version --- CHANGES | 37 +++++++++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 7fdc07d7b..65dd6b677 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,43 @@ Changes ======= +0.54.0 +------ +Added the ``db_column_name`` option to columns. This is for edge cases where +a legacy database is being used, with problematic column names. For example, +if a column is called ``class``, this clashes with a Python builtin, so the +following isn't possible: + +.. code-block:: + + class MyTable(Table): + class = Varchar() # Syntax error! + +You can now do the following: + +.. code-block:: python + + class MyTable(Table): + class_ = Varchar(db_column_name='class') + +Here are some example queries using it: + +.. code-block:: python + + # Create - both work as expected + MyTable(class_='Test').save().run_sync() + MyTable.objects().create(class_='Test').run_sync() + + # Objects + row = MyTable.objects().first().where(MyTable.class_ == 'Test').run_sync() + >>> row.class_ + 'Test' + + # Select + >>> MyTable.select().first().where(MyTable.class_ == 'Test').run_sync() + {'id': 1, 'class': 'Test'} + + 0.53.0 ------ An internal code clean up (courtesy @yezz123). diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 7bd2e37d2..336fe26f1 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.53.0" +__VERSION__ = "0.54.0" From 69389132d5946ea29f98a763d015afe49fbc5b86 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 5 Oct 2021 10:16:26 +0100 Subject: [PATCH 103/727] fix missing code block on Read the Docs --- CHANGES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 65dd6b677..8239a64a5 100644 --- a/CHANGES +++ b/CHANGES @@ -8,7 +8,7 @@ a legacy database is being used, with problematic column names. For example, if a column is called ``class``, this clashes with a Python builtin, so the following isn't possible: -.. code-block:: +.. code-block:: text class MyTable(Table): class = Varchar() # Syntax error! From 58cd080cbba0b456387f02ee905bb3e502e57b68 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 5 Oct 2021 10:48:04 +0100 Subject: [PATCH 104/727] bump doc requirements --- docs/doc-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/doc-requirements.txt b/docs/doc-requirements.txt index 3c254d122..8c3a1b192 100644 --- a/docs/doc-requirements.txt +++ b/docs/doc-requirements.txt @@ -1,3 +1,3 @@ -Sphinx==3.2.1 -sphinx-rtd-theme==0.5.0 +Sphinx==4.2.0 +sphinx-rtd-theme==1.0.0 livereload==2.6.3 From bffd47becf0aaac67042f2e291af615c8fffd867 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 5 Oct 2021 14:36:08 +0100 Subject: [PATCH 105/727] add table registry, and ability to refresh engine (#289) --- piccolo/apps/tester/commands/run.py | 10 ++++++++++ piccolo/table.py | 15 +++++++++++++++ tests/apps/tester/commands/test_run.py | 4 +++- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/piccolo/apps/tester/commands/run.py b/piccolo/apps/tester/commands/run.py index def7fd356..724a29c30 100644 --- a/piccolo/apps/tester/commands/run.py +++ b/piccolo/apps/tester/commands/run.py @@ -4,6 +4,8 @@ import sys import typing as t +from piccolo.table import TABLE_REGISTRY + class set_env_var: def __init__(self, var_name: str, temp_value: str): @@ -49,6 +51,13 @@ def run_pytest(pytest_args: t.List[str]) -> int: # pragma: no cover return pytest.main(pytest_args) +def refresh_db(): + for table_class in TABLE_REGISTRY: + # In case any table classes were imported before we set the + # environment variable. + table_class.refresh_db() + + def run( pytest_args: str = "", piccolo_conf: str = "piccolo_conf_test" ) -> None: @@ -65,5 +74,6 @@ def run( """ with set_env_var(var_name="PICCOLO_CONF", temp_value=piccolo_conf): + refresh_db() args = pytest_args.split(" ") sys.exit(run_pytest(args)) diff --git a/piccolo/table.py b/piccolo/table.py index 05794bdb8..ccbede825 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -49,6 +49,9 @@ PROTECTED_TABLENAMES = ("user",) +TABLE_REGISTRY: t.List[t.Type[Table]] = [] + + @dataclass class TableMeta: """ @@ -95,6 +98,10 @@ def db(self) -> Engine: return self._db + @db.setter + def db(self, value: Engine): + self._db = value + def get_column_by_name(self, name: str) -> Column: """ Returns a column which matches the given name. It will try and follow @@ -283,6 +290,8 @@ def __init_subclass__( if is_table_class: foreign_key_column.set_proxy_columns() + TABLE_REGISTRY.append(cls) + def __init__( self, ignore_missing: bool = False, @@ -501,6 +510,12 @@ def get_readable(cls) -> Readable: ########################################################################### + @classmethod + def refresh_db(cls): + cls._meta.db = engine_finder() + + ########################################################################### + @property def querystring(self) -> QueryString: """ diff --git a/tests/apps/tester/commands/test_run.py b/tests/apps/tester/commands/test_run.py index 640910e81..d751e3ee6 100644 --- a/tests/apps/tester/commands/test_run.py +++ b/tests/apps/tester/commands/test_run.py @@ -66,8 +66,10 @@ class FakeException(Exception): class TestRun(TestCase): @patch("piccolo.apps.tester.commands.run.run_pytest") - def test_success(self, pytest: MagicMock): + @patch("piccolo.apps.tester.commands.run.refresh_db") + def test_success(self, refresh_db: MagicMock, pytest: MagicMock): with self.assertRaises(SystemExit): run(pytest_args="-s foo", piccolo_conf="my_piccolo_conf") pytest.assert_called_once_with(["-s", "foo"]) + refresh_db.assert_called_once() From 1e772f987f2e436599163db28abf3c60cb54b2d9 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 5 Oct 2021 14:42:35 +0100 Subject: [PATCH 106/727] fix edge case where ColumnMeta can't be copied (#290) * fix edge case where ColumnMeta can't be copied * fix lgtm error --- piccolo/columns/base.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index fb4c017cb..e651864d9 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -7,7 +7,7 @@ import typing as t import uuid from abc import ABCMeta, abstractmethod -from dataclasses import dataclass, field +from dataclasses import dataclass, field, fields from enum import Enum from piccolo.columns.choices import Choice @@ -237,6 +237,16 @@ def copy(self) -> ColumnMeta: params=self.params.copy(), call_chain=self.call_chain.copy(), ) + + # Make sure we don't accidentally include any other attributes which + # aren't supported by the constructor. + field_names = [i.name for i in fields(self.__class__)] + kwargs = { + kwarg: value + for kwarg, value in kwargs.items() + if kwarg in field_names + } + return self.__class__(**kwargs) def __copy__(self) -> ColumnMeta: From d2498941f117add978a02de8aa82de8078b81751 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 5 Oct 2021 16:51:50 +0100 Subject: [PATCH 107/727] Improved column type conversion (#284) * attempt Varchar -> Integer column conversion * remove Boolean from test for now * improved column type conversion in migrations --- .../apps/migrations/auto/migration_manager.py | 41 ++++++- piccolo/columns/base.py | 2 +- piccolo/querystring.py | 6 + .../auto/integration/test_migrations.py | 111 ++++++++++++++++++ 4 files changed, 156 insertions(+), 4 deletions(-) diff --git a/piccolo/apps/migrations/auto/migration_manager.py b/piccolo/apps/migrations/auto/migration_manager.py index a3a11b014..59a64da0a 100644 --- a/piccolo/apps/migrations/auto/migration_manager.py +++ b/piccolo/apps/migrations/auto/migration_manager.py @@ -13,8 +13,10 @@ ) from piccolo.apps.migrations.auto.serialisation import deserialise_params from piccolo.columns import Column, column_types +from piccolo.columns.column_types import Serial from piccolo.engine import engine_finder from piccolo.table import Table, create_table_class, sort_table_classes +from piccolo.utils.warnings import colored_warning @dataclass @@ -396,9 +398,42 @@ async def _run_alter_columns(self, backwards=False): alter_column.db_column_name ) - await _Table.alter().set_column_type( - old_column=old_column, new_column=new_column - ) + using_expression: t.Optional[str] = None + + # Postgres won't automatically cast some types to + # others. We may as well try, as it will definitely + # fail otherwise. + if new_column.value_type != old_column.value_type: + if old_params.get("default", ...) is not None: + # Unless the column's default value is also + # something which can be cast to the new type, + # it will also fail. Drop the default value for + # now - the proper default is set later on. + await _Table.alter().drop_default( + old_column + ).run() + + using_expression = "{}::{}".format( + alter_column.db_column_name, + new_column.column_type, + ) + + # We can't migrate a SERIAL to a BIGSERIAL or vice + # versa, as SERIAL isn't a true type, just an alias to + # other commands. + if issubclass(column_class, Serial) and issubclass( + old_column_class, Serial + ): + colored_warning( + "Unable to migrate Serial to BigSerial and " + "vice versa. This must be done manually." + ) + else: + await _Table.alter().set_column_type( + old_column=old_column, + new_column=new_column, + using_expression=using_expression, + ).run() ############################################################### diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index e651864d9..9c7f88eb1 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -672,7 +672,7 @@ def ddl(self) -> str: f" ON UPDATE {on_update}" ) - if not self._meta.primary_key: + if self.__class__.__name__ not in ("Serial", "BigSerial"): default = self.get_default_value() sql_value = self.get_sql_value(value=default) query += f" DEFAULT {sql_value}" diff --git a/piccolo/querystring.py b/piccolo/querystring.py index c3727fc21..5b273db71 100644 --- a/piccolo/querystring.py +++ b/piccolo/querystring.py @@ -20,6 +20,12 @@ class Unquoted: value: str + def __repr__(self): + return f"{self.value}" + + def __str__(self): + return f"{self.value}" + @dataclass class Fragment: diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index d1fa51fcf..150c9c5c3 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -7,6 +7,7 @@ import time import typing as t import uuid +from unittest.mock import MagicMock, patch from piccolo.apps.migrations.commands.forwards import ForwardsMigrationManager from piccolo.apps.migrations.commands.new import ( @@ -21,16 +22,20 @@ UUID, Array, BigInt, + BigSerial, Boolean, Date, DoublePrecision, Integer, Interval, + Numeric, Real, + Serial, SmallInt, Text, Time, Timestamp, + Timestamptz, Varchar, ) from piccolo.columns.defaults.uuid import UUID4 @@ -105,6 +110,8 @@ def _test_migrations( test_function: t.Optional[t.Callable[[RowMeta], None]] = None, ): """ + Writes a migration file to disk and runs it. + :param table_classes: Migrations will be created and run based on successive table classes in this list. @@ -621,3 +628,107 @@ def test_db_column_name_initial(self): ] ), ) + + ########################################################################### + + def test_column_type_conversion_string(self): + """ + We can't manage all column type conversions, but should be able to + manage most simple ones (e.g. Varchar to Text). + """ + self._test_migrations( + table_classes=[ + self.table(column) + for column in [ + Varchar(), + Text(), + Varchar(), + ] + ] + ) + + def test_column_type_conversion_integer(self): + self._test_migrations( + table_classes=[ + self.table(column) + for column in [ + Integer(), + BigInt(), + SmallInt(), + BigInt(), + Integer(), + ] + ] + ) + + def test_column_type_conversion_string_to_integer(self): + self._test_migrations( + table_classes=[ + self.table(column) + for column in [ + Varchar(default="1"), + Integer(default=1), + Varchar(default="1"), + ] + ] + ) + + def test_column_type_conversion_float_decimal(self): + self._test_migrations( + table_classes=[ + self.table(column) + for column in [ + Real(default=1.0), + DoublePrecision(default=1.0), + Real(default=1.0), + Numeric(), + Real(default=1.0), + ] + ] + ) + + def test_column_type_conversion_json(self): + self._test_migrations( + table_classes=[ + self.table(column) + for column in [ + JSON(), + JSONB(), + JSON(), + ] + ] + ) + + def test_column_type_conversion_timestamp(self): + self._test_migrations( + table_classes=[ + self.table(column) + for column in [ + Timestamp(), + Timestamptz(), + Timestamp(), + ] + ] + ) + + @patch("piccolo.apps.migrations.auto.migration_manager.colored_warning") + def test_column_type_conversion_serial(self, colored_warning: MagicMock): + """ + This isn't possible, as neither SERIAL or BIGSERIAL are actual types. + They're just shortcuts. Make sure the migration doesn't crash - it + should just output a warning. + """ + self._test_migrations( + table_classes=[ + self.table(column) + for column in [ + Serial(), + BigSerial(), + ] + ] + ) + + colored_warning.assert_called_once_with( + "Unable to migrate Serial to BigSerial and vice versa. This must " + "be done manually." + ) From 625ce7a0deb1ef5d747908cd8a2212d2e41d6240 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 5 Oct 2021 19:00:25 +0100 Subject: [PATCH 108/727] Added `drop_tables` function (#292) * add `drop_tables` function * using transactions --- docs/src/piccolo/query_types/alter.rst | 8 ++++ piccolo/engine/base.py | 4 ++ piccolo/table.py | 56 +++++++++++++++++++++++--- tests/table/test_drop_tables.py | 21 ++++++++++ 4 files changed, 84 insertions(+), 5 deletions(-) create mode 100644 tests/table/test_drop_tables.py diff --git a/docs/src/piccolo/query_types/alter.rst b/docs/src/piccolo/query_types/alter.rst index 4d6f69cb2..88d485561 100644 --- a/docs/src/piccolo/query_types/alter.rst +++ b/docs/src/piccolo/query_types/alter.rst @@ -37,6 +37,14 @@ Used to drop the table - use with caution! Band.alter().drop_table().run_sync() +If you have several tables which you want to drop, you can use ``drop_tables`` +instead. It will drop them in the correct order. + +.. code-block:: python + + from piccolo.table import drop_tables + + drop_tables(Band, Manager) rename_column ------------- diff --git a/piccolo/engine/base.py b/piccolo/engine/base.py index 3da74b66d..e60594dcc 100644 --- a/piccolo/engine/base.py +++ b/piccolo/engine/base.py @@ -58,6 +58,10 @@ async def run_ddl(self, ddl: str, in_pool: bool = True): def transaction(self): pass + @abstractmethod + def atomic(self): + pass + async def check_version(self): """ Warn if the database version isn't supported. diff --git a/piccolo/table.py b/piccolo/table.py index ccbede825..27f96e957 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -996,13 +996,59 @@ def create_table_class( ) -def create_tables(*args: t.Type[Table], if_not_exists: bool = False) -> None: +def create_tables(*tables: t.Type[Table], if_not_exists: bool = False) -> None: """ - Creates multiple tables that passed to it. + Creates the tables passed to it in the correct order, based on their + foreign keys. """ - sorted_table_classes = sort_table_classes(list(args)) - for table in sorted_table_classes: - Create(table=table, if_not_exists=if_not_exists).run_sync() + if tables: + engine = tables[0]._meta.db + else: + return + + sorted_table_classes = sort_table_classes(list(tables)) + + atomic = engine.atomic() + atomic.add( + *[ + table.create_table(if_not_exists=if_not_exists) + for table in sorted_table_classes + ] + ) + atomic.run_sync() + + +def drop_tables(*tables: t.Type[Table]) -> None: + """ + Drops the tables passed to it in the correct order, based on their foreign + keys. + """ + if tables: + engine = tables[0]._meta.db + else: + return + + if engine.engine_type == "sqlite": + # SQLite doesn't support CASCADE, so we have to drop them in the + # correct order. + sorted_table_classes = reversed(sort_table_classes(list(tables))) + atomic = engine.atomic() + atomic.add( + *[ + Alter(table=table).drop_table(if_exists=True) + for table in sorted_table_classes + ] + ) + atomic.run_sync() + else: + atomic = engine.atomic() + atomic.add( + *[ + table.alter().drop_table(cascade=True, if_exists=True) + for table in tables + ] + ) + atomic.run_sync() def sort_table_classes( diff --git a/tests/table/test_drop_tables.py b/tests/table/test_drop_tables.py new file mode 100644 index 000000000..8e9dffb0a --- /dev/null +++ b/tests/table/test_drop_tables.py @@ -0,0 +1,21 @@ +from unittest import TestCase + +from piccolo.table import create_tables, drop_tables +from tests.example_apps.music.tables import Band, Manager + + +class TestDropTables(TestCase): + def setUp(self): + create_tables(Band, Manager) + + def test_drop_tables(self): + """ + Make sure the tables are dropped. + """ + self.assertEqual(Manager.table_exists().run_sync(), True) + self.assertEqual(Band.table_exists().run_sync(), True) + + drop_tables(Manager, Band) + + self.assertEqual(Manager.table_exists().run_sync(), False) + self.assertEqual(Band.table_exists().run_sync(), False) From c1f9212e753bce2454ee32517cb9b4c78968d13d Mon Sep 17 00:00:00 2001 From: William Michael Short <36488354+wmshort@users.noreply.github.com> Date: Wed, 6 Oct 2021 09:32:42 +0100 Subject: [PATCH 109/727] include index info in schema generation (replaces #278) (#282) * include index info in schema generation * fix tests * added test for index reflection Co-authored-by: William Michael Short Co-authored-by: Daniel Townsend --- piccolo/apps/schema/commands/generate.py | 96 ++++++++++++++++++--- tests/apps/schema/commands/test_generate.py | 70 +++++++++++---- 2 files changed, 136 insertions(+), 30 deletions(-) diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index 20a32ac11..931bd3f88 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -36,6 +36,7 @@ Varchar, ) from piccolo.columns.defaults.interval import IntervalCustom +from piccolo.columns.indexes import IndexMethod from piccolo.engine.finder import engine_finder from piccolo.engine.postgres import PostgresEngine from piccolo.table import Table, create_table_class, sort_table_classes @@ -170,6 +171,42 @@ def get_column_ref_trigger( raise ValueError("No matching trigger found") +@dataclasses.dataclass +class Index: + indexname: str + indexdef: str + + def __post_init__(self): + pat = re.compile( + r"""^CREATE[ ](?:(?PUNIQUE)[ ])?INDEX[ ]\w+?[ ] + ON[ ].+?[ ]USING[ ](?P\w+?)[ ] + \((?P\w+?)\)""", + re.VERBOSE, + ) + groups = re.match(pat, self.indexdef).groupdict() + + self.column_name = groups["column_name"] + self.unique = True if "unique" in groups else False + self.method = INDEX_METHOD_MAP[groups["method"]] + + +@dataclasses.dataclass +class TableIndexes: + """ + All of the indexes for a certain table in the database. + """ + + tablename: str + indexes: t.List[Index] + + def get_column_index(self, column_name: str) -> t.Optional[Index]: + for i in self.indexes: + if i.column_name == column_name: + return i + + return None + + @dataclasses.dataclass class OutputSchema: """ @@ -285,7 +322,7 @@ def __add__(self, value: OutputSchema) -> OutputSchema: CURRENT_TIMESTAMP ) $""", - re.X, + re.VERBOSE, ), UUID: None, Serial: None, @@ -361,6 +398,42 @@ def get_column_default( return column_type.value_type(value["value"]) +INDEX_METHOD_MAP: t.Dict[str, IndexMethod] = { + "btree": IndexMethod.btree, + "hash": IndexMethod.hash, + "gist": IndexMethod.gist, + "gin": IndexMethod.gin, +} + + +# 'Indices' seems old-fashioned and obscure in this context. +async def get_indexes( + table_class: t.Type[Table], tablename: str, schema_name: str = "public" +) -> TableIndexes: + """ + Get all of the constraints for a table. + + :param table_class: + Any Table subclass - just used to execute raw queries on the database. + + """ + indexes = await table_class.raw( + ( + "SELECT indexname, indexdef " + "FROM pg_indexes " + "WHERE schemaname = {} " + "AND tablename = {}; " + ), + schema_name, + tablename, + ) + + return TableIndexes( + tablename=tablename, + indexes=[Index(**i) for i in indexes], + ) + + async def get_fk_triggers( table_class: t.Type[Table], tablename: str, schema_name: str = "public" ) -> TableTriggers: @@ -547,6 +620,9 @@ def get_table_name(name: str, schema: str) -> str: async def create_table_class_from_db( table_class: t.Type[Table], tablename: str, schema_name: str ) -> OutputSchema: + indexes = await get_indexes( + table_class=table_class, tablename=tablename, schema_name=schema_name + ) constraints = await get_constraints( table_class=table_class, tablename=tablename, schema_name=schema_name ) @@ -575,6 +651,11 @@ async def create_table_class_from_db( "unique": constraints.is_unique(column_name=column_name), } + index = indexes.get_column_index(column_name=column_name) + if index is not None: + kwargs["index"] = True + kwargs["index_method"] = index.method + if constraints.is_primary_key(column_name=column_name): kwargs["primary_key"] = True if column_type == Integer: @@ -713,16 +794,6 @@ class Schema(Table, db=engine): ) output_schema.imports = sorted(list(set(output_schema.imports))) - # We currently don't show the index argument for columns in the output, - # so we don't need this import for now: - if ( - "from piccolo.columns.indexes import IndexMethod" - in output_schema.imports - ): - output_schema.imports.remove( - "from piccolo.columns.indexes import IndexMethod" - ) - return output_schema @@ -738,8 +809,7 @@ async def generate(schema_name: str = "public"): output_schema = await get_output_schema(schema_name=schema_name) output = output_schema.imports + [ - i._table_str(excluded_params=["index_method", "index", "choices"]) - for i in output_schema.tables + i._table_str(excluded_params=["choices"]) for i in output_schema.tables ] if output_schema.warnings: diff --git a/tests/apps/schema/commands/test_generate.py b/tests/apps/schema/commands/test_generate.py index e03ee3ecf..83c400e43 100644 --- a/tests/apps/schema/commands/test_generate.py +++ b/tests/apps/schema/commands/test_generate.py @@ -12,6 +12,7 @@ ) from piccolo.columns.base import Column from piccolo.columns.column_types import ForeignKey, Integer, Varchar +from piccolo.columns.indexes import IndexMethod from piccolo.engine import Engine, engine_finder from piccolo.table import Table from piccolo.utils.sync import run_sync @@ -19,21 +20,6 @@ from tests.example_apps.mega.tables import MegaTable, SmallTable -class Publication(Table, tablename="schema2.publication"): - name = Varchar(length=50) - - -class Writer(Table, tablename="schema1.writer"): - name = Varchar(length=50) - publication = ForeignKey(Publication, null=True) - - -class Book(Table): - name = Varchar(length=50) - writer = ForeignKey(Writer, null=True) - popularity = Integer(default=0) - - @postgres_only class TestGenerate(TestCase): def setUp(self): @@ -130,7 +116,7 @@ class Box(Column): def test_generate_required_tables(self): """ - Make sure only tables passed to `tablenames` are created + Make sure only tables passed to `tablenames` are created. """ output_schema: OutputSchema = run_sync( get_output_schema(include=[SmallTable._meta.tablename]) @@ -141,7 +127,7 @@ def test_generate_required_tables(self): def test_exclude_table(self): """ - make sure exclude works + Make sure exclude works. """ output_schema: OutputSchema = run_sync( get_output_schema(exclude=[MegaTable._meta.tablename]) @@ -151,6 +137,56 @@ def test_exclude_table(self): self._compare_table_columns(SmallTable, SmallTable_) +############################################################################### + + +class Band(Table): + name = Varchar(index=True, index_method=IndexMethod.hash) + popularity = Integer(index=False) + + +@postgres_only +class TestGenerateWithIndexes(TestCase): + def setUp(self): + Band.create_table().run_sync() + + def tearDown(self): + Band.alter().drop_table(if_exists=True).run_sync() + + def test_index(self): + """ + Make sure that a table with an index is reflected correctly. + """ + output_schema: OutputSchema = run_sync(get_output_schema()) + Band_ = output_schema.tables[0] + + self.assertEqual(Band_.name._meta.index, True) + self.assertEqual(Band_.name._meta.index_method, IndexMethod.hash) + + self.assertEqual(Band_.popularity._meta.index, False) + self.assertEqual( + Band_.popularity._meta.index_method, IndexMethod.btree + ) + + +############################################################################### + + +class Publication(Table, tablename="schema2.publication"): + name = Varchar(length=50) + + +class Writer(Table, tablename="schema1.writer"): + name = Varchar(length=50) + publication = ForeignKey(Publication, null=True) + + +class Book(Table): + name = Varchar(length=50) + writer = ForeignKey(Writer, null=True) + popularity = Integer(default=0) + + @postgres_only class TestGenerateWithSchema(TestCase): def setUp(self) -> None: From 569bb431bd81545060785fab7e915a0beb863f27 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 6 Oct 2021 10:10:19 +0100 Subject: [PATCH 110/727] moved refresh_db to TableMeta --- piccolo/apps/tester/commands/run.py | 2 +- piccolo/table.py | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/piccolo/apps/tester/commands/run.py b/piccolo/apps/tester/commands/run.py index 724a29c30..b858f587b 100644 --- a/piccolo/apps/tester/commands/run.py +++ b/piccolo/apps/tester/commands/run.py @@ -55,7 +55,7 @@ def refresh_db(): for table_class in TABLE_REGISTRY: # In case any table classes were imported before we set the # environment variable. - table_class.refresh_db() + table_class._meta.refresh_db() def run( diff --git a/piccolo/table.py b/piccolo/table.py index 27f96e957..10847bed3 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -102,6 +102,9 @@ def db(self) -> Engine: def db(self, value: Engine): self._db = value + def refresh_db(self): + self.db = engine_finder() + def get_column_by_name(self, name: str) -> Column: """ Returns a column which matches the given name. It will try and follow @@ -510,12 +513,6 @@ def get_readable(cls) -> Readable: ########################################################################### - @classmethod - def refresh_db(cls): - cls._meta.db = engine_finder() - - ########################################################################### - @property def querystring(self) -> QueryString: """ From dadd13c3dcf4322c3ae7bdefea76e16ad455ff9d Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 6 Oct 2021 10:20:00 +0100 Subject: [PATCH 111/727] bumped version --- CHANGES | 358 +++++++++++++++++++++++++++++++++++++++++++- piccolo/__init__.py | 2 +- 2 files changed, 355 insertions(+), 5 deletions(-) diff --git a/CHANGES b/CHANGES index 8239a64a5..d491711a2 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,67 @@ Changes ======= +0.55.0 +------ + +Table._meta.refresh_db +~~~~~~~~~~~~~~~~~~~~~~ + +Added the ability to refresh the database engine. + +.. code-block:: python + + MyTable._meta.refresh_db() + +This causes the ``Table`` to fetch the ``Engine`` again from your +``piccolo_conf.py`` file. The reason this is useful, is you might change the +``PICCOLO_CONF`` environment variable, and some ``Table`` classes have +already imported an engine. This is now used by the ``piccolo tester run`` +command to ensure all ``Table`` classes have the correct engine. + +ColumnMeta edge cases +~~~~~~~~~~~~~~~~~~~~~ + +Fixed an edge case where ``ColumnMeta`` couldn't be copied if it had extra +attributes added to it. + +Improved column type conversion +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When running migrations which change column types, Piccolo now provides the +``USING`` clause to the ``ALTER COLUMN`` DDL statement, which makes it more +likely that type conversion will be successful. + +For example, if there is an ``Integer`` column, and it's converted to a +``Varchar`` column, the migration will run fine. In the past, running this in +reverse would fail. Now Postgres will try and cast the values back to integers, +which makes reversing migrations more likely to succeed. + +Added drop_tables +~~~~~~~~~~~~~~~~~ + +There is now a convenience function for dropping several tables in one go. If +the database doesn't support ``CASCADE``, then the tables are sorted based on +their ``ForeignKey`` columns, so they're dropped in the correct order. It all +runs inside a transaction. + +.. code-block:: python + + from piccolo.table import drop_tables + + drop_tables(Band, Manager) + +This is a useful tool in unit tests. + +Index support in schema generation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When using ``piccolo schema generate``, Piccolo will now reflect the indexes +from the database into the generated ``Table`` classes. Thanks to @wmshort for +this. + +------------------------------------------------------------------------------- + 0.54.0 ------ Added the ``db_column_name`` option to columns. This is for edge cases where @@ -37,6 +98,7 @@ Here are some example queries using it: >>> MyTable.select().first().where(MyTable.class_ == 'Test').run_sync() {'id': 1, 'class': 'Test'} +------------------------------------------------------------------------------- 0.53.0 ------ @@ -63,6 +125,8 @@ Here's an example: >>> await Band.select().run() [{'id': 1, 'name': 'Pythonistas', 'manager': 1}, ...] +------------------------------------------------------------------------------- + 0.52.0 ------ Lots of improvements to ``piccolo schema generate``: @@ -81,11 +145,15 @@ Added ``BigSerial`` column type (courtesy @aliereno). Added GitHub issue templates (courtesy @AbhijithGanesh). +------------------------------------------------------------------------------- + 0.51.1 ------ Fixing a bug with ``on_delete`` and ``on_update`` not being set correctly. Thanks to @wmshort for discovering this. +------------------------------------------------------------------------------- + 0.51.0 ------ Modified ``create_pydantic_model``, so ``JSON`` and ``JSONB`` columns have a @@ -95,6 +163,8 @@ improved JSON support. Courtesy @sinisaos. Fixing a bug where the ``piccolo fixtures load`` command wasn't registered with the Piccolo CLI. +------------------------------------------------------------------------------- + 0.50.0 ------ The ``where`` clause can now accept multiple arguments (courtesy @AliSayyah): @@ -146,6 +216,8 @@ Improved ``AppRegistry``, so if a user only adds the app name (e.g. ``blog``), instead of ``blog.piccolo_app``, it will now emit a warning, and will try to import ``blog.piccolo_app`` (courtesy @aliereno). +------------------------------------------------------------------------------- + 0.49.0 ------ Fixed a bug with ``create_pydantic_model`` when used with a ``Decimal`` / @@ -171,6 +243,8 @@ Fixed typos with the new fixtures app - sometimes it was referred to as ``fixture`` and other times ``fixtures``. It's now standardised as ``fixtures`` (courtesy @hipertracker). +------------------------------------------------------------------------------- + 0.48.0 ------ The ``piccolo user create`` command can now be used by passing in command line @@ -180,11 +254,15 @@ For example ``piccolo user create --username=bob ...``. This is useful when you want to create users in a script. +------------------------------------------------------------------------------- + 0.47.0 ------ You can now use ``pip install piccolo[all]``, which will install all optional requirements. +------------------------------------------------------------------------------- + 0.46.0 ------ Added the fixtures app. This is used to dump data from a database to a JSON @@ -209,6 +287,8 @@ library (prior to this it only existed within the ``piccolo_api`` library). At a later date, the ``piccolo_api`` library will be updated, so it's Pydantic code just proxies to what's within the main ``piccolo`` library. +------------------------------------------------------------------------------- + 0.45.1 ------ Improvements to ``piccolo schema generate``. It's now smarter about which @@ -216,6 +296,8 @@ imports to include. Also, the ``Table`` classes output will now be sorted based on their ``ForeignKey`` columns. Internally the sorting algorithm has been changed to use the ``graphlib`` module, which was added in Python 3.9. +------------------------------------------------------------------------------- + 0.45.0 ------ Added the ``piccolo schema graph`` command for visualising your database @@ -228,6 +310,8 @@ image, for example: Also made some minor changes to the ASGI templates, to reduce MyPy errors. +------------------------------------------------------------------------------- + 0.44.1 ------ Updated ``to_dict`` so it works with nested objects, as introduced by the @@ -249,6 +333,8 @@ It also works with filtering: >>> band.to_dict(Band.name, Band.manager.name) {'name': 'Pythonistas', 'manager': {'name': 'Guido'}} +------------------------------------------------------------------------------- + 0.44.0 ------ Added the ability to prefetch related objects. Here's an example: @@ -274,11 +360,15 @@ which will return all of the related rows as objects. Thanks to @wmshort for all the input. +------------------------------------------------------------------------------- + 0.43.0 ------ Migrations containing ``Array``, ``JSON`` and ``JSONB`` columns should be more reliable now. More unit tests were added to cover edge cases. +------------------------------------------------------------------------------- + 0.42.0 ------ You can now use ``all_columns`` at the root. For example: @@ -299,11 +389,15 @@ You can also exclude certain columns if you like: Band.manager.all_columns(exclude=[Band.manager.id]) ).run() +------------------------------------------------------------------------------- + 0.41.1 ------ Fix a regression where if multiple tables are created in a single migration file, it could potentially fail by applying them in the wrong order. +------------------------------------------------------------------------------- + 0.41.0 ------ Fixed a bug where if ``all_columns`` was used two or more levels deep, it would @@ -329,11 +423,15 @@ Also, the ``ColumnsDelegate`` has now been tweaked, so unpacking of Concert.band_1.manager.all_columns() ).run_sync() +------------------------------------------------------------------------------- + 0.40.1 ------ Loosen the ``typing-extensions`` requirement, as it was causing issues when installing ``asyncpg``. +------------------------------------------------------------------------------- + 0.40.0 ------ Added ``nested`` output option, which makes the response from a ``select`` @@ -346,6 +444,8 @@ query use nested dictionaries: Thanks to @wmshort for the idea. +------------------------------------------------------------------------------- + 0.39.0 ------ Added ``to_dict`` method to ``Table``. @@ -369,10 +469,14 @@ values. By using ``to_dict`` it's just the column values. Here's an example: Thanks to @wmshort for the idea, and @aminalaee and @sinisaos for investigating edge cases. +------------------------------------------------------------------------------- + 0.38.2 ------ Removed problematic type hint which assumed pytest was installed. +------------------------------------------------------------------------------- + 0.38.1 ------ Minor changes to ``get_or_create`` to make sure it handles joins correctly. @@ -391,6 +495,8 @@ Minor changes to ``get_or_create`` to make sure it handles joins correctly. In this situation, there are two columns called ``name`` - we need to make sure the correct value is applied if the row doesn't exist. +------------------------------------------------------------------------------- + 0.38.0 ------ ``get_or_create`` now supports more complex where clauses. For example: @@ -406,11 +512,15 @@ And you can find out whether the row was created or not using Thanks to @wmshort for reporting this issue. +------------------------------------------------------------------------------- + 0.37.0 ------ Added ``ModelBuilder``, which can be used to generate data for tests (courtesy @aminalaee). +------------------------------------------------------------------------------- + 0.36.0 ------ Fixed an issue where ``like`` and ``ilike`` clauses required a wildcard. For @@ -430,6 +540,8 @@ Which would match on ``'guido'`` and ``'Guido'``, but not ``'Guidoxyz'``. Thanks to @wmshort for reporting this issue. +------------------------------------------------------------------------------- + 0.35.0 ------ * Improved ``PrimaryKey`` deprecation warning (courtesy @tonybaloney). @@ -447,6 +559,8 @@ Thanks to @wmshort for reporting this issue. # This is equivalent to: manager = await Manager.objects().where(Manager.name == 'Guido').first().run() +------------------------------------------------------------------------------- + 0.34.0 ------ Added the ``get_or_create`` convenience method (courtesy @aminalaee). Example @@ -458,6 +572,8 @@ usage: Manager.name == 'Guido' ).run() +------------------------------------------------------------------------------- + 0.33.1 ------ * Bug fix, where ``compare_dicts`` was failing in migrations if any ``Column`` @@ -466,17 +582,23 @@ usage: * Increased the minimum version of orjson, so binaries are available for Macs running on Apple silicon (courtesy @hipertracker). +------------------------------------------------------------------------------- + 0.33.0 ------ Fix for auto migrations when using custom primary keys (thanks to @adriangb and @aminalaee for investigating this issue). +------------------------------------------------------------------------------- + 0.32.0 ------ Migrations can now have a description, which is shown when using ``piccolo migrations check``. This makes migrations easier to identify (thanks to @davidolrik for the idea). +------------------------------------------------------------------------------- + 0.31.0 ------ Added an ``all_columns`` method, to make it easier to retrieve all related @@ -491,6 +613,8 @@ wrapped in quotes, to make sure it works on ZSH (i.e. ``pip install 'piccolo[postgres]'`` instead of ``pip install piccolo[postgres]``). +------------------------------------------------------------------------------- + 0.30.0 ------ The database drivers are now installed separately. For example: @@ -500,6 +624,8 @@ For some users this might be a **breaking change** - please make sure that for existing Piccolo projects, you have either ``asyncpg``, or ``piccolo[postgres]`` in your ``requirements.txt`` file. +------------------------------------------------------------------------------- + 0.29.0 ------ The user can now specify the primary key column (courtesy @aminalaee). For @@ -513,11 +639,15 @@ example: The BlackSheep template generated by `piccolo asgi new` now supports mounting of the Piccolo Admin (courtesy @sinisaos). +------------------------------------------------------------------------------- + 0.28.0 ------ Added aggregations functions, such as ``Sum``, ``Min``, ``Max`` and ``Avg``, for use in select queries (courtesy @sinisaos). +------------------------------------------------------------------------------- + 0.27.0 ------ Added uvloop as an optional dependency, installed via `pip install piccolo[uvloop]` @@ -526,11 +656,15 @@ loop found in Python's standard library. When uvloop is installed, Piccolo will use it to increase the performance of the Piccolo CLI, and web servers such as Uvicorn will use it to increase the performance of your ASGI app. +------------------------------------------------------------------------------- + 0.26.0 ------ Added ``eq`` and ``ne`` methods to the ``Boolean`` column, which can be used if linters complain about using ``SomeTable.some_column == True``. +------------------------------------------------------------------------------- + 0.25.0 ------ * Changed the migration IDs, so the timestamp now includes microseconds. This @@ -538,11 +672,15 @@ if linters complain about using ``SomeTable.some_column == True``. * Added a lot of end-to-end tests for migrations, which revealed some bugs in ``Column`` defaults. +------------------------------------------------------------------------------- + 0.24.1 ------ A bug fix for migrations. See `issue 123 `_ for more information. +------------------------------------------------------------------------------- + 0.24.0 ------ Lots of improvements to ``JSON`` and ``JSONB`` columns. Piccolo will now @@ -576,6 +714,8 @@ deserialise it. >>> studio.facilities {'mixing_desk': True} +------------------------------------------------------------------------------- + 0.23.0 ------ Added the ``create_table_class`` function, which can be used to create @@ -583,6 +723,8 @@ Added the ``create_table_class`` function, which can be used to create which was effecting migrations (see `issue 111 `_ for more details). +------------------------------------------------------------------------------- + 0.22.0 ------ * An error is now raised if a user tries to create a Piccolo app using @@ -596,29 +738,41 @@ for more details). ``await Band.select().where(Band.manager == some_manager).run()``, instead of having to explicity reference the ``id``. +------------------------------------------------------------------------------- + 0.21.2 ------ Fixing a bug with serialising ``Enum`` instances in migrations. For example: ``Varchar(default=Colour.red)``. +------------------------------------------------------------------------------- + 0.21.1 ------ Fix missing imports in FastAPI and Starlette app templates. +------------------------------------------------------------------------------- + 0.21.0 ------ * Added a ``freeze`` method to ``Query``. * Added BlackSheep as an option to ``piccolo asgi new``. +------------------------------------------------------------------------------- + 0.20.0 ------ Added ``choices`` option to ``Column``. +------------------------------------------------------------------------------- + 0.19.1 ------ * Added ``piccolo user change_permissions`` command. * Added aliases for CLI commands. +------------------------------------------------------------------------------- + 0.19.0 ------ Changes to the ``BaseUser`` table - added a ``superuser``, and ``last_login`` @@ -635,6 +789,8 @@ following DDL statements: ALTER TABLE piccolo_user ADD COLUMN "superuser" BOOLEAN NOT NULL DEFAULT false ALTER TABLE piccolo_user ADD COLUMN "last_login" TIMESTAMP DEFAULT null +------------------------------------------------------------------------------- + 0.18.4 ------ * Fixed a bug when multiple tables inherit from the same mixin (thanks to @@ -647,18 +803,26 @@ following DDL statements: * Fixed a bug with ``SerialisedBuiltin.__hash__`` not returning a number, which could break migrations (thanks to @sinisaos). +------------------------------------------------------------------------------- + 0.18.3 ------ Improved ``Array`` column serialisation - needed to fix auto migrations. +------------------------------------------------------------------------------- + 0.18.2 ------ Added support for filtering ``Array`` columns. +------------------------------------------------------------------------------- + 0.18.1 ------ Add the ``Array`` column type as a top level import in ``piccolo.columns``. +------------------------------------------------------------------------------- + 0.18.0 ------ * Refactored ``forwards`` and ``backwards`` commands for migrations, to make @@ -667,22 +831,30 @@ Add the ``Array`` column type as a top level import in ``piccolo.columns``. * ``table_finder`` now works if just a string is passed in, instead of having to pass in an array of strings. +------------------------------------------------------------------------------- + 0.17.5 ------ Catching database connection exceptions when starting the default ASGI app created with ``piccolo asgi new`` - these errors exist if the Postgres database hasn't been created yet. +------------------------------------------------------------------------------- + 0.17.4 ------ Added a ``help_text`` option to the ``Table`` metaclass. This is used in Piccolo Admin to show tooltips. +------------------------------------------------------------------------------- + 0.17.3 ------ Added a ``help_text`` option to the ``Column`` constructor. This is used in Piccolo Admin to show tooltips. +------------------------------------------------------------------------------- + 0.17.2 ------ * Exposing ``index_type`` in the ``Column`` constructor. @@ -690,11 +862,15 @@ Piccolo Admin to show tooltips. thanks to paolodina for finding this. * Fixing a typo in the ``PostgresEngine`` docs - courtesy of paolodina. +------------------------------------------------------------------------------- + 0.17.1 ------ Fixing a bug with ``SchemaSnapshot`` if column types were changed in migrations - the snapshot didn't reflect the changes. +------------------------------------------------------------------------------- + 0.17.0 ------ * Migrations now directly import ``Column`` classes - this allows users to @@ -703,32 +879,44 @@ Fixing a bug with ``SchemaSnapshot`` if column types were changed in migrations * Migrations now detect if the column type has changed, and will try and convert it automatically. +------------------------------------------------------------------------------- + 0.16.5 ------ The Postgres extensions that ``PostgresEngine`` tries to enable at startup can now be configured. +------------------------------------------------------------------------------- + 0.16.4 ------ * Fixed a bug with ``MyTable.column != None`` * Added ``is_null`` and ``is_not_null`` methods, to avoid linting issues when comparing with None. +------------------------------------------------------------------------------- + 0.16.3 ------ * Added ``WhereRaw``, so raw SQL can be used in where clauses. * ``piccolo shell run`` now uses syntax highlighting - courtesy of Fingel. +------------------------------------------------------------------------------- + 0.16.2 ------ Reordering the dependencies in requirements.txt when using ``piccolo asgi new`` as the latest FastAPI and Starlette versions are incompatible. +------------------------------------------------------------------------------- + 0.16.1 ------ Added ``Timestamptz`` column type, for storing datetimes which are timezone aware. +------------------------------------------------------------------------------- + 0.16.0 ------ * Fixed a bug with creating a ``ForeignKey`` column with ``references="self"`` @@ -736,19 +924,27 @@ aware. * Changed migration file naming, so there are no characters in there which are unsupported on Windows. +------------------------------------------------------------------------------- + 0.15.1 ------ Changing the status code when creating a migration, and no changes were detected. It now returns a status code of 0, so it doesn't fail build scripts. +------------------------------------------------------------------------------- + 0.15.0 ------ Added ``Bytea`` / ``Blob`` column type. +------------------------------------------------------------------------------- + 0.14.13 ------- Fixing a bug with migrations which drop column defaults. +------------------------------------------------------------------------------- + 0.14.12 ------- * Fixing a bug where re-running ``Table.create(if_not_exists=True)`` would @@ -757,11 +953,15 @@ Fixing a bug with migrations which drop column defaults. ``references``. For example, ``.tables.Manager``. The paths must be absolute for now. +------------------------------------------------------------------------------- + 0.14.11 ------- Fixing a bug with ``Boolean`` column defaults, caused by the ``Table`` metaclass not being explicit enough when checking falsy values. +------------------------------------------------------------------------------- + 0.14.10 ------- * The ``ForeignKey`` ``references`` argument can now be specified using a @@ -773,11 +973,15 @@ metaclass not being explicit enough when checking falsy values. ``await Band.select(Band.manager.name).run()``. * Fixed a bug with migrations and foreign key contraints. +------------------------------------------------------------------------------- + 0.14.9 ------ Modified the exit codes for the ``forwards`` and ``backwards`` commands when no migrations are left to run / reverse. Otherwise build scripts may fail. +------------------------------------------------------------------------------- + 0.14.8 ------ * Improved the method signature of the ``output`` query clause (explicitly @@ -793,6 +997,8 @@ migrations are left to run / reverse. Otherwise build scripts may fail. * Changed the migration commands to be top level async. * Combined ``print`` and ``sys.exit`` statements. +------------------------------------------------------------------------------- + 0.14.7 ------ * Added missing type annotation for ``run_sync``. @@ -800,6 +1006,8 @@ migrations are left to run / reverse. Otherwise build scripts may fail. * Replaced instances of ``asyncio.run`` with ``run_sync``. * Tidied up aiosqlite imports. +------------------------------------------------------------------------------- + 0.14.6 ------ * Added JSON and JSONB column types, and the arrow function for JSONB. @@ -808,6 +1016,8 @@ migrations are left to run / reverse. Otherwise build scripts may fail. response (i.e. SELECT foo AS bar from baz). * Refactored JSON encoding into a separate utils file. +------------------------------------------------------------------------------- + 0.14.5 ------ * Removed old iPython version recommendation in the ``piccolo shell run`` and @@ -815,6 +1025,8 @@ migrations are left to run / reverse. Otherwise build scripts may fail. * Fixing outstanding mypy warnings. * Added optional requirements for the playground to setup.py +------------------------------------------------------------------------------- + 0.14.4 ------ * Added ``piccolo sql_shell run`` command, which launches the psql or sqlite3 @@ -824,6 +1036,8 @@ migrations are left to run / reverse. Otherwise build scripts may fail. an event loop in the current thread. * Removed asgiref dependency. +------------------------------------------------------------------------------- + 0.14.3 ------ * Queries can be directly awaited - ``await MyTable.select()``, as an @@ -831,6 +1045,8 @@ migrations are left to run / reverse. Otherwise build scripts may fail. * The ``piccolo asgi new`` command now accepts a ``name`` argument, which is used to populate the default database name within the template. +------------------------------------------------------------------------------- + 0.14.2 ------ * Centralised code for importing Piccolo apps and tables - laying the @@ -839,50 +1055,70 @@ migrations are left to run / reverse. Otherwise build scripts may fail. ``pip install piccolo[orjson]``. * Improved version number parsing in Postgres. +------------------------------------------------------------------------------- + 0.14.1 ------ Fixing a bug with dropping tables in auto migrations. +------------------------------------------------------------------------------- + 0.14.0 ------ Added ``Interval`` column type. +------------------------------------------------------------------------------- + 0.13.5 ------ * Added ``allowed_hosts`` to ``create_admin`` in ASGI template. * Fixing bug with default ``root`` argument in some piccolo commands. +------------------------------------------------------------------------------- + 0.13.4 ------ * Fixed bug with ``SchemaSnapshot`` when dropping columns. * Added custom ``__repr__`` method to ``Table``. +------------------------------------------------------------------------------- + 0.13.3 ------ Added ``piccolo shell run`` command for running adhoc queries using Piccolo. +------------------------------------------------------------------------------- + 0.13.2 ------ * Fixing bug with auto migrations when dropping columns. * Added a ``root`` argument to ``piccolo asgi new``, ``piccolo app new`` and ``piccolo project new`` commands, to override where the files are placed. +------------------------------------------------------------------------------- + 0.13.1 ------ Added support for ``group_by`` and ``Count`` for aggregate queries. +------------------------------------------------------------------------------- + 0.13.0 ------ Added `required` argument to ``Column``. This allows the user to indicate which fields must be provided by the user. Other tools can use this value when generating forms and serialisers. +------------------------------------------------------------------------------- + 0.12.6 ------ * Fixing a typo in ``TimestampCustom`` arguments. * Fixing bug in ``TimestampCustom`` SQL representation. * Added more extensive deserialisation for migrations. +------------------------------------------------------------------------------- + 0.12.5 ------ * Improved ``PostgresEngine`` docstring. @@ -891,25 +1127,35 @@ generating forms and serialisers. * Fixed bug with altering column defaults to be non-static values. * Removed ``response_handler`` from ``Alter`` query. +------------------------------------------------------------------------------- + 0.12.4 ------ Using orjson for JSON serialisation when using the ``output(as_json=True)`` clause. It supports more Python types than ujson. +------------------------------------------------------------------------------- + 0.12.3 ------ Improved ``piccolo user create`` command - defaults the username to the current system user. +------------------------------------------------------------------------------- + 0.12.2 ------ Fixing bug when sorting ``extra_definitions`` in auto migrations. +------------------------------------------------------------------------------- + 0.12.1 ------ * Fixed typos. * Bumped requirements. +------------------------------------------------------------------------------- + 0.12.0 ------ * Added ``Date`` and ``Time`` columns. @@ -919,10 +1165,14 @@ Fixing bug when sorting ``extra_definitions`` in auto migrations. * Auto migrations can handle adding / removing indexes. * Improved ASGI template for FastAPI. +------------------------------------------------------------------------------- + 0.11.8 ------ ASGI template fix. +------------------------------------------------------------------------------- + 0.11.7 ------ * Improved ``UUID`` columns in SQLite - prepending 'uuid:' to the stored value @@ -930,25 +1180,35 @@ ASGI template fix. * Removed SQLite as an option for ``piccolo asgi new`` until auto migrations are supported. +------------------------------------------------------------------------------- + 0.11.6 ------ Added support for FastAPI to ``piccolo asgi new``. +------------------------------------------------------------------------------- + 0.11.5 ------ Fixed bug in ``BaseMigrationManager.get_migration_modules`` - wasn't excluding non-Python files well enough. +------------------------------------------------------------------------------- + 0.11.4 ------ * Stopped ``piccolo migrations new`` from creating a config.py file - was legacy. * Added a README file to the `piccolo_migrations` folder in the ASGI template. +------------------------------------------------------------------------------- + 0.11.3 ------ Fixed `__pycache__` bug when using ``piccolo asgi new``. +------------------------------------------------------------------------------- + 0.11.2 ------ * Showing a warning if trying auto migrations with SQLite. @@ -957,6 +1217,8 @@ Fixed `__pycache__` bug when using ``piccolo asgi new``. ``piccolo meta version``. * Added example queries to the playground. +------------------------------------------------------------------------------- + 0.11.1 ------ * Added ``table_finder``, for use in ``AppConfig``. @@ -965,191 +1227,279 @@ Fixed `__pycache__` bug when using ``piccolo asgi new``. * Improved consistency between SQLite and Postgres with ``UUID`` columns, ``Integer`` columns, and ``exists`` queries. +------------------------------------------------------------------------------- + 0.11.0 ------ Added ``Numeric`` and ``Real`` column types. +------------------------------------------------------------------------------- + 0.10.8 ------ Fixing a bug where Postgres versions without a patch number couldn't be parsed. +------------------------------------------------------------------------------- + 0.10.7 ------ Improving release script. +------------------------------------------------------------------------------- + 0.10.6 ------ Sorting out packaging issue - old files were appearing in release. +------------------------------------------------------------------------------- + 0.10.5 ------ Auto migrations can now run backwards. +------------------------------------------------------------------------------- + 0.10.4 ------ Fixing some typos with ``Table`` imports. Showing a traceback when piccolo_conf can't be found by ``engine_finder``. +------------------------------------------------------------------------------- + 0.10.3 ------ Adding missing jinja templates to setup.py. +------------------------------------------------------------------------------- + 0.10.2 ------ Fixing a bug when using ``piccolo project new`` in a new project. +------------------------------------------------------------------------------- + 0.10.1 ------ Fixing bug with enum default values. +------------------------------------------------------------------------------- + 0.10.0 ------ Using targ for the CLI. Refactored some core code into apps. +------------------------------------------------------------------------------- + 0.9.3 ----- Suppressing exceptions when trying to find the Postgres version, to avoid an ``ImportError`` when importing `piccolo_conf.py`. +------------------------------------------------------------------------------- + 0.9.2 ----- ``.first()`` bug fix. +------------------------------------------------------------------------------- + 0.9.1 ----- Auto migration fixes, and ``.first()`` method now returns None if no match is found. +------------------------------------------------------------------------------- + 0.9.0 ----- Added support for auto migrations. +------------------------------------------------------------------------------- + 0.8.3 ----- Can use operators in update queries, and fixing 'new' migration command. +------------------------------------------------------------------------------- + 0.8.2 ----- Fixing release issue. +------------------------------------------------------------------------------- + 0.8.1 ----- Improved transaction support - can now use a context manager. Added ``Secret``, ``BigInt`` and ``SmallInt`` column types. Foreign keys can now reference the parent table. +------------------------------------------------------------------------------- + 0.8.0 ----- Fixing bug when joining across several tables. Can pass values directly into the ``Table.update`` method. Added ``if_not_exists`` option when creating a table. +------------------------------------------------------------------------------- + 0.7.7 ----- Column sequencing matches the definition order. +------------------------------------------------------------------------------- + 0.7.6 ----- Supporting `ON DELETE` and `ON UPDATE` for foreign keys. Recording reverse foreign key relationships. +------------------------------------------------------------------------------- + 0.7.5 ----- Made ``response_handler`` async. Made it easier to rename columns. +------------------------------------------------------------------------------- + 0.7.4 ----- Bug fixes and dependency updates. +------------------------------------------------------------------------------- + 0.7.3 ----- -Adding missing `__int__.py` file. +Adding missing ``__int__.py`` file. + +------------------------------------------------------------------------------- 0.7.2 ----- Changed migration import paths. +------------------------------------------------------------------------------- + 0.7.1 ----- Added ``remove_db_file`` method to ``SQLiteEngine`` - makes testing easier. +------------------------------------------------------------------------------- + 0.7.0 ----- Renamed ``create`` to ``create_table``, and can register commands via `piccolo_conf`. +------------------------------------------------------------------------------- + 0.6.1 ----- -Adding missing `__init__.py` files. +Adding missing ``__init__.py`` files. + +------------------------------------------------------------------------------- 0.6.0 ----- Moved ``BaseUser``. Migration refactor. +------------------------------------------------------------------------------- + 0.5.2 ----- Moved drop table under ``Alter`` - to help prevent accidental drops. +------------------------------------------------------------------------------- + 0.5.1 ----- Added ``batch`` support. +------------------------------------------------------------------------------- + 0.5.0 ----- Refactored the ``Table`` Metaclass - much simpler now. Scoped more of the attributes on ``Column`` to avoid name clashes. Added ``engine_finder`` to make database configuration easier. +------------------------------------------------------------------------------- + 0.4.1 ----- SQLite is now returning datetime objects for timestamp fields. +------------------------------------------------------------------------------- + 0.4.0 ----- Refactored to improve code completion, along with bug fixes. +------------------------------------------------------------------------------- + 0.3.7 ----- -Allowing ``Update`` queries in SQLite +Allowing ``Update`` queries in SQLite. + +------------------------------------------------------------------------------- 0.3.6 ----- -Falling back to `LIKE` instead of `ILIKE` for SQLite +Falling back to `LIKE` instead of `ILIKE` for SQLite. + +------------------------------------------------------------------------------- 0.3.5 ----- Renamed ``User`` to ``BaseUser``. +------------------------------------------------------------------------------- + 0.3.4 ----- Added ``ilike``. +------------------------------------------------------------------------------- + 0.3.3 ----- Added value types to columns. +------------------------------------------------------------------------------- + 0.3.2 ----- Default values infer the engine type. +------------------------------------------------------------------------------- + 0.3.1 ----- Update click version. +------------------------------------------------------------------------------- + 0.3 --- Tweaked API to support more auto completion. Join support in where clause. Basic SQLite support - mostly for playground. +------------------------------------------------------------------------------- + 0.2 --- Using ``QueryString`` internally to represent queries, instead of raw strings, to harden against SQL injection. +------------------------------------------------------------------------------- + 0.1.2 ----- Allowing joins across multiple tables. +------------------------------------------------------------------------------- + 0.1.1 ----- Added playground. diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 336fe26f1..7725e0d85 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.54.0" +__VERSION__ = "0.55.0" From 74be2cad53afc37f74f31047d936c6bb07b4b09c Mon Sep 17 00:00:00 2001 From: knguyen5 <32172398+knguyen5@users.noreply.github.com> Date: Mon, 11 Oct 2021 08:12:17 -0600 Subject: [PATCH 112/727] added optional quotation to regex pattern for index column (#296) * Update generate.py * update unit test for index reflection Co-authored-by: Daniel Townsend --- piccolo/apps/schema/commands/generate.py | 18 ++++++++++-- tests/apps/schema/commands/test_generate.py | 31 ++++++++++++++------- 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index 931bd3f88..bef0bb5d8 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -177,15 +177,29 @@ class Index: indexdef: str def __post_init__(self): + """ + An example DDL statement which will be parsed: + + .. code-block:: sql + + CREATE INDEX some_index_name + ON some_schema.some_table + USING some_index_type (some_column_name) + + If the column name is the same as a Postgres data type, then Postgres + wraps the column name in quotes. For example, ``"time"`` instead of + ``time``. + + """ pat = re.compile( r"""^CREATE[ ](?:(?PUNIQUE)[ ])?INDEX[ ]\w+?[ ] ON[ ].+?[ ]USING[ ](?P\w+?)[ ] - \((?P\w+?)\)""", + \(\"?(?P\w+?\"?)\)""", re.VERBOSE, ) groups = re.match(pat, self.indexdef).groupdict() - self.column_name = groups["column_name"] + self.column_name = groups["column_name"].lstrip('"').rstrip('"') self.unique = True if "unique" in groups else False self.method = INDEX_METHOD_MAP[groups["method"]] diff --git a/tests/apps/schema/commands/test_generate.py b/tests/apps/schema/commands/test_generate.py index 83c400e43..ac0f8a66c 100644 --- a/tests/apps/schema/commands/test_generate.py +++ b/tests/apps/schema/commands/test_generate.py @@ -11,7 +11,12 @@ get_output_schema, ) from piccolo.columns.base import Column -from piccolo.columns.column_types import ForeignKey, Integer, Varchar +from piccolo.columns.column_types import ( + ForeignKey, + Integer, + Timestamp, + Varchar, +) from piccolo.columns.indexes import IndexMethod from piccolo.engine import Engine, engine_finder from piccolo.table import Table @@ -140,32 +145,38 @@ def test_exclude_table(self): ############################################################################### -class Band(Table): +class Concert(Table): name = Varchar(index=True, index_method=IndexMethod.hash) - popularity = Integer(index=False) + time = Timestamp( + index=True + ) # Testing a column with the same name as a Postgres data type. + capacity = Integer(index=False) @postgres_only class TestGenerateWithIndexes(TestCase): def setUp(self): - Band.create_table().run_sync() + Concert.create_table().run_sync() def tearDown(self): - Band.alter().drop_table(if_exists=True).run_sync() + Concert.alter().drop_table(if_exists=True).run_sync() def test_index(self): """ Make sure that a table with an index is reflected correctly. """ output_schema: OutputSchema = run_sync(get_output_schema()) - Band_ = output_schema.tables[0] + Concert_ = output_schema.tables[0] + + self.assertEqual(Concert_.name._meta.index, True) + self.assertEqual(Concert_.name._meta.index_method, IndexMethod.hash) - self.assertEqual(Band_.name._meta.index, True) - self.assertEqual(Band_.name._meta.index_method, IndexMethod.hash) + self.assertEqual(Concert_.time._meta.index, True) + self.assertEqual(Concert_.time._meta.index_method, IndexMethod.btree) - self.assertEqual(Band_.popularity._meta.index, False) + self.assertEqual(Concert_.capacity._meta.index, False) self.assertEqual( - Band_.popularity._meta.index_method, IndexMethod.btree + Concert_.capacity._meta.index_method, IndexMethod.btree ) From be1bf4d952e6b53febfda2dae47072d6c8c4e14b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 11 Oct 2021 16:57:13 +0100 Subject: [PATCH 113/727] Finder.get_table_classes (#294) * added get_tables to Finder * ignore flake8 error * rename example_app to music * added tests for `Finder.get_table_classes` * improve testing docs - mention `Finder().get_table_classes()` --- .../projects_and_apps/included_apps.rst | 14 ++++ .../projects_and_apps/piccolo_apps.rst | 15 +++-- docs/src/piccolo/testing/index.rst | 58 +++++++++++++++- piccolo/conf/apps.py | 59 ++++++++++++++--- setup.py | 2 +- .../migrations/auto/test_serialisation.py | 4 +- tests/apps/migrations/commands/test_clean.py | 6 +- .../commands/test_forwards_backwards.py | 26 +++----- tests/apps/migrations/commands/test_new.py | 2 +- tests/apps/shell/commands/test_run.py | 2 +- tests/columns/test_foreignkey.py | 2 +- tests/columns/test_reference.py | 8 +-- tests/conf/test_apps.py | 66 ++++++++++++++++++- tests/example_apps/music/piccolo_app.py | 2 +- .../piccolo_migrations/2020-12-17T18-44-30.py | 2 +- .../piccolo_migrations/2020-12-17T18-44-39.py | 2 +- .../piccolo_migrations/2020-12-17T18-44-44.py | 2 +- .../2021-07-25T22-38-48-009306.py | 2 +- .../2021-09-06T13-58-23-024723.py | 2 +- 19 files changed, 221 insertions(+), 55 deletions(-) diff --git a/docs/src/piccolo/projects_and_apps/included_apps.rst b/docs/src/piccolo/projects_and_apps/included_apps.rst index aad322526..e965a9ea6 100644 --- a/docs/src/piccolo/projects_and_apps/included_apps.rst +++ b/docs/src/piccolo/projects_and_apps/included_apps.rst @@ -219,6 +219,20 @@ By default ``piccolo tester run`` sets ``PICCOLO_CONF`` to ``'piccolo_conf_test'``, meaning that a file called ``piccolo_conf_test.py`` will be imported. +Within the ``piccolo_conf_test.py`` file, override the database settings, so it +uses a test database: + +.. code-block:: python + + from piccolo_conf import * + + DB = PostgresEngine( + config={ + "database": "my_app_test" + } + ) + + If you prefer, you can set a custom ``PICCOLO_CONF`` value: .. code-block:: bash diff --git a/docs/src/piccolo/projects_and_apps/piccolo_apps.rst b/docs/src/piccolo/projects_and_apps/piccolo_apps.rst index 580f1f11d..8d85930c1 100644 --- a/docs/src/piccolo/projects_and_apps/piccolo_apps.rst +++ b/docs/src/piccolo/projects_and_apps/piccolo_apps.rst @@ -23,7 +23,7 @@ Run the following command within your project: piccolo app new my_app -Where `my_app` is your new app's name. This will create a folder like this: +Where ``my_app`` is your new app's name. This will create a folder like this: .. code-block:: bash @@ -36,7 +36,7 @@ Where `my_app` is your new app's name. This will create a folder like this: It's important to register your new app with the ``APP_REGISTRY`` in -`piccolo_conf.py`. +``piccolo_conf.py``. .. code-block:: python @@ -53,7 +53,7 @@ operations on your app, such as :ref:`Migrations`. AppConfig ********* -Inside your app's `piccolo_app.py` file is an ``AppConfig`` instance. This is +Inside your app's ``piccolo_app.py`` file is an ``AppConfig`` instance. This is how you customise your app's settings. .. code-block:: python @@ -84,7 +84,7 @@ how you customise your app's settings. app_name ======== -This is used to identify your app, when using the `piccolo` CLI, for example: +This is used to identify your app, when using the ``piccolo`` CLI, for example: .. code-block:: bash @@ -146,7 +146,7 @@ commands ======== You can register functions and coroutines, which are automatically added to -the `piccolo` CLI. +the ``piccolo`` CLI. The `targ `_ library is used under the hood. It makes it really easy to write command lines tools - just use type annotations @@ -182,14 +182,15 @@ And from the command line: >>> piccolo my_app say_hello bob hello, bob -If the code contains an error to see more details in the output add a ``--trace`` flag to the command line. +If the code contains an error to see more details in the output add a ``--trace`` +flag to the command line. .. code-block:: bash >>> piccolo my_app say_hello bob --trace -By convention, store the command definitions in a `commands` folder in your +By convention, store the command definitions in a ``commands`` folder in your app. .. code-block:: bash diff --git a/docs/src/piccolo/testing/index.rst b/docs/src/piccolo/testing/index.rst index f2d4bb8f0..41e7de824 100644 --- a/docs/src/piccolo/testing/index.rst +++ b/docs/src/piccolo/testing/index.rst @@ -3,6 +3,8 @@ Testing Piccolo provides a few tools to make testing easier and decrease manual work. +------------------------------------------------------------------------------- + Model Builder ------------- @@ -67,7 +69,61 @@ To build object with minimal attributes, leaving nullable fields empty: band = await ModelBuilder.build(Band, minimal=True) # Leaves manager empty +------------------------------------------------------------------------------- + Test runner ----------- -See the :ref:`tester app`. +This runs your unit tests using pytest. See the :ref:`tester app`. + +------------------------------------------------------------------------------- + +Creating the test schema +------------------------ + +When running your unit tests, you usually start with a blank test database, +create the tables, and then install test data. + +To create the tables, there are a few different approaches you can take. Here +we use ``create_tables`` and ``drop_tables``: + +.. code-block:: python + + from unittest import TestCase + + from piccolo.table import create_tables, drop_tables + from piccolo.conf.apps import Finder + + TABLES = Finder().get_table_classes() + + class TestApp(TestCase): + def setUp(self): + create_tables(*TABLES) + + def tearDown(self): + drop_tables(*TABLES) + + def test_app(self): + # Do some testing ... + pass + +Alternatively, you can run the migrations to setup the schema if you prefer: + +.. code-block:: python + + import asyncio + from unittest import TestCase + + from piccolo.apps.migrations.commands.backwards import run_backwards + from piccolo.apps.migrations.commands.forwards import run_forwards + + class TestApp(TestCase): + def setUp(self): + asyncio.run(run_forwards("all")) + + def tearDown(self): + asyncio.run(run_backwards("all", auto_agree=True)) + + def test_app(self): + # Do some testing ... + pass diff --git a/piccolo/conf/apps.py b/piccolo/conf/apps.py index 133104d22..3d75f679d 100644 --- a/piccolo/conf/apps.py +++ b/piccolo/conf/apps.py @@ -360,7 +360,7 @@ def get_piccolo_conf_module( def get_app_registry(self) -> AppRegistry: """ - Returns the AppRegistry instance within piccolo_conf. + Returns the ``AppRegistry`` instance within piccolo_conf. """ piccolo_conf_module = self.get_piccolo_conf_module() return getattr(piccolo_conf_module, "APP_REGISTRY") @@ -388,8 +388,8 @@ def get_engine( def get_app_modules(self) -> t.List[PiccoloAppModule]: """ - Returns the piccolo_app.py modules for each registered Piccolo app in - your project. + Returns the ``piccolo_app.py`` modules for each registered Piccolo app + in your project. """ app_registry = self.get_app_registry() app_modules = self._import_app_modules(app_registry.apps) @@ -399,14 +399,21 @@ def get_app_modules(self) -> t.List[PiccoloAppModule]: return app_modules - def get_sorted_app_names(self) -> t.List[str]: + def get_app_names(self, sort: bool = True) -> t.List[str]: """ - Sorts the app names using the migration dependencies, so dependencies - are before dependents in the list. + Return all of the app names. + + :param sort: + If True, sorts the app names using the migration dependencies, so + dependencies are before dependents in the list. + """ modules = self.get_app_modules() configs: t.List[AppConfig] = [module.APP_CONFIG for module in modules] + if not sort: + return [i.app_name for i in configs] + def sort_app_configs(app_config_1: AppConfig, app_config_2: AppConfig): return ( app_config_1 in app_config_2.migration_dependency_app_configs @@ -417,9 +424,15 @@ def sort_app_configs(app_config_1: AppConfig, app_config_2: AppConfig): ) return [i.app_name for i in sorted_configs] + def get_sorted_app_names(self) -> t.List[str]: + """ + Just here for backwards compatibility. + """ + return self.get_app_names(sort=True) + def get_app_config(self, app_name: str) -> AppConfig: """ - Returns an `AppConfig` for the given app name. + Returns an ``AppConfig`` for the given app name. """ modules = self.get_app_modules() for module in modules: @@ -432,10 +445,38 @@ def get_table_with_name( self, app_name: str, table_class_name: str ) -> t.Type[Table]: """ - Returns a Table subclass registered with the given app if it exists. - Otherwise it raises an ValueError. + Returns a ``Table`` class registered with the given app if it exists. + Otherwise it raises an ``ValueError``. """ app_config = self.get_app_config(app_name=app_name) return app_config.get_table_with_name( table_class_name=table_class_name ) + + def get_table_classes( + self, + include_apps: t.Optional[t.List[str]] = None, + exclude_apps: t.Optional[t.List[str]] = None, + ) -> t.List[t.Type[Table]]: + """ + Returns all ``Table`` classes registered with the given apps. If + ``app_names`` is ``None``, then ``Table`` classes will be returned + for all apps. + """ + if include_apps and exclude_apps: + raise ValueError("Only specify `include_apps` or `exclude_apps`.") + + if include_apps: + app_names = include_apps + else: + app_names = self.get_app_names() + if exclude_apps: + app_names = [i for i in app_names if i not in exclude_apps] + + tables: t.List[t.Type[Table]] = [] + + for app_name in app_names: + app_config = self.get_app_config(app_name=app_name) + tables.extend(app_config.table_classes) + + return tables diff --git a/setup.py b/setup.py index 1c9c8a574..6fa58f3a0 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ def parse_requirement(req_path: str) -> t.List[str]: parse_requirement('extras/playground.txt') # requirements/extras/playground.txt Returns: List[str]: list of requirements specified in the file. - """ + """ # noqa: E501 with open(os.path.join(directory, "requirements", req_path)) as f: contents = f.read() return [i.strip() for i in contents.strip().split("\n")] diff --git a/tests/apps/migrations/auto/test_serialisation.py b/tests/apps/migrations/auto/test_serialisation.py index 839e17bfd..b236573ac 100644 --- a/tests/apps/migrations/auto/test_serialisation.py +++ b/tests/apps/migrations/auto/test_serialisation.py @@ -40,9 +40,7 @@ def test_uuid(self): def test_lazy_table_reference(self): # These are equivalent: references_list = [ - LazyTableReference( - table_class_name="Manager", app_name="example_app" - ), + LazyTableReference(table_class_name="Manager", app_name="music"), LazyTableReference( table_class_name="Manager", module_path="tests.example_apps.music.tables", diff --git a/tests/apps/migrations/commands/test_clean.py b/tests/apps/migrations/commands/test_clean.py index 37b3ae5a2..a6ff8c2b4 100644 --- a/tests/apps/migrations/commands/test_clean.py +++ b/tests/apps/migrations/commands/test_clean.py @@ -20,14 +20,14 @@ def test_clean(self): migration_ids = real_migration_ids + [orphaned_migration_id] Migration.insert( - *[Migration(name=i, app_name="example_app") for i in migration_ids] + *[Migration(name=i, app_name="music") for i in migration_ids] ).run_sync() - run_sync(clean(app_name="example_app", auto_agree=True)) + run_sync(clean(app_name="music", auto_agree=True)) remaining_rows = ( Migration.select(Migration.name) - .where(Migration.app_name == "example_app") + .where(Migration.app_name == "music") .output(as_list=True) .order_by(Migration.name) .run_sync() diff --git a/tests/apps/migrations/commands/test_forwards_backwards.py b/tests/apps/migrations/commands/test_forwards_backwards.py index 99d24159a..701bf5916 100644 --- a/tests/apps/migrations/commands/test_forwards_backwards.py +++ b/tests/apps/migrations/commands/test_forwards_backwards.py @@ -47,7 +47,7 @@ def test_forwards_backwards_all_migrations(self): """ Test running all of the migrations forwards, then backwards. """ - for app_name in ("example_app", "all"): + for app_name in ("music", "all"): run_sync(forwards(app_name=app_name, migration_id="all")) # Check the tables exist @@ -69,9 +69,7 @@ def test_forwards_backwards_single_migration(self): Test running a single migrations forwards, then backwards. """ for migration_id in ["1", "2020-12-17T18:44:30"]: - run_sync( - forwards(app_name="example_app", migration_id=migration_id) - ) + run_sync(forwards(app_name="music", migration_id=migration_id)) table_classes = [Band, Manager] @@ -81,7 +79,7 @@ def test_forwards_backwards_single_migration(self): run_sync( backwards( - app_name="example_app", + app_name="music", migration_id=migration_id, auto_agree=True, ) @@ -98,9 +96,7 @@ def test_forwards_unknown_migration(self, print_: MagicMock): """ with self.assertRaises(SystemExit): run_sync( - forwards( - app_name="example_app", migration_id="migration-12345" - ) + forwards(app_name="music", migration_id="migration-12345") ) self.assertTrue( @@ -113,12 +109,12 @@ def test_backwards_unknown_migration(self, print_: MagicMock): """ Test running an unknown migrations backwards. """ - run_sync(forwards(app_name="example_app", migration_id="all")) + run_sync(forwards(app_name="music", migration_id="all")) with self.assertRaises(SystemExit): run_sync( backwards( - app_name="example_app", + app_name="music", migration_id="migration-12345", auto_agree=True, ) @@ -137,7 +133,7 @@ def test_backwards_no_migrations(self, print_: MagicMock): """ run_sync( backwards( - app_name="example_app", + app_name="music", migration_id="2020-12-17T18:44:30", auto_agree=True, ) @@ -151,8 +147,8 @@ def test_forwards_no_migrations(self, print_: MagicMock): """ Test running the migrations if they've already run. """ - run_sync(forwards(app_name="example_app", migration_id="all")) - run_sync(forwards(app_name="example_app", migration_id="all")) + run_sync(forwards(app_name="music", migration_id="all")) + run_sync(forwards(app_name="music", migration_id="all")) self.assertTrue( print_.mock_calls[-1] == call("🏁 No migrations need to be run") @@ -162,9 +158,7 @@ def test_forwards_fake(self): """ Test running the migrations if they've already run. """ - run_sync( - forwards(app_name="example_app", migration_id="all", fake=True) - ) + run_sync(forwards(app_name="music", migration_id="all", fake=True)) for table_class in TABLE_CLASSES: self.assertTrue(not table_class.table_exists().run_sync()) diff --git a/tests/apps/migrations/commands/test_new.py b/tests/apps/migrations/commands/test_new.py index bb212db80..9917eda22 100644 --- a/tests/apps/migrations/commands/test_new.py +++ b/tests/apps/migrations/commands/test_new.py @@ -49,7 +49,7 @@ def test_new_command(self, print_: MagicMock): Call the command, when no migration changes are needed. """ with self.assertRaises(SystemExit) as manager: - run_sync(new(app_name="example_app", auto=True)) + run_sync(new(app_name="music", auto=True)) self.assertEqual(manager.exception.code, 0) diff --git a/tests/apps/shell/commands/test_run.py b/tests/apps/shell/commands/test_run.py index 8a9dffb0f..8594f7101 100644 --- a/tests/apps/shell/commands/test_run.py +++ b/tests/apps/shell/commands/test_run.py @@ -17,7 +17,7 @@ def test_run(self, print_: MagicMock, start_ipython_shell: MagicMock): print_.mock_calls, [ call("-------"), - call("Importing example_app tables:"), + call("Importing music tables:"), call("- Band"), call("- Concert"), call("- Manager"), diff --git a/tests/columns/test_foreignkey.py b/tests/columns/test_foreignkey.py index e3e5510a2..a4e206496 100644 --- a/tests/columns/test_foreignkey.py +++ b/tests/columns/test_foreignkey.py @@ -201,7 +201,7 @@ def test_lazy_reference_to_app(self): Make sure a LazyTableReference to a Table within a Piccolo app works. """ reference = LazyTableReference( - table_class_name="Manager", app_name="example_app" + table_class_name="Manager", app_name="music" ) self.assertTrue(reference.resolve() is Manager) diff --git a/tests/columns/test_reference.py b/tests/columns/test_reference.py index b9ffe85a7..ea9887e00 100644 --- a/tests/columns/test_reference.py +++ b/tests/columns/test_reference.py @@ -19,14 +19,14 @@ def test_init(self): with self.assertRaises(ValueError): LazyTableReference( table_class_name="Manager", - app_name="example_app", + app_name="music", module_path="tests.example_apps.music.tables", ) # Shouldn't raise exceptions: LazyTableReference( table_class_name="Manager", - app_name="example_app", + app_name="music", ) LazyTableReference( table_class_name="Manager", @@ -37,9 +37,9 @@ def test_str(self): self.assertEqual( LazyTableReference( table_class_name="Manager", - app_name="example_app", + app_name="music", ).__str__(), - "App example_app.Manager", + "App music.Manager", ) self.assertEqual( diff --git a/tests/conf/test_apps.py b/tests/conf/test_apps.py index fdf09d793..dc7d49359 100644 --- a/tests/conf/test_apps.py +++ b/tests/conf/test_apps.py @@ -1,8 +1,18 @@ from unittest import TestCase from piccolo.apps.user.tables import BaseUser -from piccolo.conf.apps import AppConfig, AppRegistry, table_finder -from tests.example_apps.music.tables import Manager +from piccolo.conf.apps import AppConfig, AppRegistry, Finder, table_finder +from tests.example_apps.mega.tables import MegaTable, SmallTable +from tests.example_apps.music.tables import ( + Band, + Concert, + Manager, + Poster, + RecordingStudio, + Shirt, + Ticket, + Venue, +) class TestAppRegistry(TestCase): @@ -166,3 +176,55 @@ def test_exclude_tags(self): "Venue", ], ) + + +class TestFinder(TestCase): + def test_get_table_classes(self): + """ + Make sure ``Table`` classes can be retrieved. + """ + finder = Finder() + + self.assertEqual( + finder.get_table_classes(), + [ + Manager, + Band, + Venue, + Concert, + Ticket, + Poster, + Shirt, + RecordingStudio, + MegaTable, + SmallTable, + ], + ) + + self.assertEqual( + finder.get_table_classes(include_apps=["music"]), + [ + Manager, + Band, + Venue, + Concert, + Ticket, + Poster, + Shirt, + RecordingStudio, + ], + ) + + self.assertEqual( + finder.get_table_classes(exclude_apps=["music"]), + [ + MegaTable, + SmallTable, + ], + ) + + with self.assertRaises(ValueError): + # You shouldn't be allowed to specify both include and exclude. + finder.get_table_classes( + exclude_apps=["music"], include_apps=["mega"] + ) diff --git a/tests/example_apps/music/piccolo_app.py b/tests/example_apps/music/piccolo_app.py index d74c916e2..f37f92f18 100644 --- a/tests/example_apps/music/piccolo_app.py +++ b/tests/example_apps/music/piccolo_app.py @@ -17,7 +17,7 @@ APP_CONFIG = AppConfig( - app_name="example_app", + app_name="music", table_classes=[ Manager, Band, diff --git a/tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-30.py b/tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-30.py index 8d037f7e6..e9a5e42e0 100644 --- a/tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-30.py +++ b/tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-30.py @@ -12,7 +12,7 @@ class Manager(Table, tablename="manager"): async def forwards(): - manager = MigrationManager(migration_id=ID, app_name="example_app") + manager = MigrationManager(migration_id=ID, app_name="music") manager.add_table("Band", tablename="band") manager.add_table("Manager", tablename="manager") diff --git a/tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-39.py b/tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-39.py index b8439da2e..7e4ed09d2 100644 --- a/tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-39.py +++ b/tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-39.py @@ -18,7 +18,7 @@ class Venue(Table, tablename="venue"): async def forwards(): - manager = MigrationManager(migration_id=ID, app_name="example_app") + manager = MigrationManager(migration_id=ID, app_name="music") manager.add_table("Ticket", tablename="ticket") manager.add_table("Venue", tablename="venue") diff --git a/tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-44.py b/tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-44.py index fadcc5b48..2b4aec32a 100644 --- a/tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-44.py +++ b/tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-44.py @@ -5,7 +5,7 @@ async def forwards(): - manager = MigrationManager(migration_id=ID, app_name="example_app") + manager = MigrationManager(migration_id=ID, app_name="music") manager.add_table("Poster", tablename="poster") diff --git a/tests/example_apps/music/piccolo_migrations/2021-07-25T22-38-48-009306.py b/tests/example_apps/music/piccolo_migrations/2021-07-25T22-38-48-009306.py index 1277d396b..484bd3390 100644 --- a/tests/example_apps/music/piccolo_migrations/2021-07-25T22-38-48-009306.py +++ b/tests/example_apps/music/piccolo_migrations/2021-07-25T22-38-48-009306.py @@ -9,7 +9,7 @@ async def forwards(): - manager = MigrationManager(migration_id=ID, app_name="example_app") + manager = MigrationManager(migration_id=ID, app_name="music") manager.add_table("Shirt", tablename="shirt") diff --git a/tests/example_apps/music/piccolo_migrations/2021-09-06T13-58-23-024723.py b/tests/example_apps/music/piccolo_migrations/2021-09-06T13-58-23-024723.py index f20846128..5f9661397 100644 --- a/tests/example_apps/music/piccolo_migrations/2021-09-06T13-58-23-024723.py +++ b/tests/example_apps/music/piccolo_migrations/2021-09-06T13-58-23-024723.py @@ -23,7 +23,7 @@ class Concert(Table, tablename="concert"): async def forwards(): manager = MigrationManager( - migration_id=ID, app_name="example_app", description=DESCRIPTION + migration_id=ID, app_name="music", description=DESCRIPTION ) manager.add_column( From a355bca0a65532d4a187cb2ba3cea86ce41589f8 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 11 Oct 2021 17:55:58 +0100 Subject: [PATCH 114/727] `get_output_schema` now accepts an engine argument (#298) * get_output_schema now accepts an engine argument * fix linting error --- piccolo/apps/schema/commands/generate.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index bef0bb5d8..d30d4b084 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -753,6 +753,7 @@ async def get_output_schema( schema_name: str = "public", include: t.Optional[t.List[str]] = None, exclude: t.Optional[t.List[str]] = None, + engine: t.Optional[Engine] = None, ) -> OutputSchema: """ :param schema_name: @@ -761,10 +762,15 @@ async def get_output_schema( Optional list of table names. Only creates the specified tables. :param exclude: Optional list of table names. excludes the specified tables. + :param engine: + The ``Engine`` instance to use for making database queries. If not + specified, then ``engine_finder`` is used to get the engine from + ``piccolo_conf.py``. :returns: OutputSchema """ - engine: t.Optional[Engine] = engine_finder() + if engine is None: + engine = engine_finder() if exclude is None: exclude = [] From 85173c5657fc4066433e4b881fdcc1c86d25574e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 11 Oct 2021 18:30:22 +0100 Subject: [PATCH 115/727] bumped version --- CHANGES | 68 +++++++++++++++++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index d491711a2..02f70420b 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,74 @@ Changes ======= +0.56.0 +------ + +Fixed schema generation bug +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When using ``piccolo schema generate`` to auto generate Piccolo ``Table`` +classes from an existing database, it would fail in this situation: + + * A table has a column with an index. + * The column name clashed with a Postgres type. + +For example, we couldn't auto generate this ``Table`` class: + +.. code-block:: python + + class MyTable(Table): + time = Timestamp() + +This is because ``time`` is a builtin Postgres type, and the ``CREATE INDEX`` +statement being inspected in the database wrapped the column name in quotes, +which broke our regex. + +Thanks to @knguyen5 for fixing this. + +Improved testing docs +~~~~~~~~~~~~~~~~~~~~~ + +A convenience method called ``get_table_classes`` was added to ``Finder``. + +``Finder`` is the main class in Piccolo for dynamically importing projects / +apps / tables / migrations etc. + +``get_table_classes`` lets us easily get the ``Table`` classes for a project. +This makes writing unit tests easier, when we need to setup a schema. + +.. code-block:: python + + from unittest import TestCase + + from piccolo.table import create_tables, drop_tables + from piccolo.conf.apps import Finder + + TABLES = Finder().get_table_classes() + + class TestApp(TestCase): + def setUp(self): + create_tables(*TABLES) + + def tearDown(self): + drop_tables(*TABLES) + + def test_app(self): + # Do some testing ... + pass + +When dropping tables in a unit test, remember to use ``piccolo tester run``, to +make sure the test database is used. + +get_output_schema +~~~~~~~~~~~~~~~~~ + +``get_output_schema`` is the main entrypoint for database reflection in +Piccolo. It has been modified to accept an ``Engine`` argument, which makes it +more flexible. + +------------------------------------------------------------------------------- + 0.55.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 7725e0d85..93c4d84e0 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.55.0" +__VERSION__ = "0.56.0" From 45d3d59da82dd4a108563acdc7ac303b8ae31a7e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 11 Oct 2021 18:34:52 +0100 Subject: [PATCH 116/727] Update CHANGES --- CHANGES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 02f70420b..c46477d7d 100644 --- a/CHANGES +++ b/CHANGES @@ -18,7 +18,7 @@ For example, we couldn't auto generate this ``Table`` class: .. code-block:: python class MyTable(Table): - time = Timestamp() + time = Timestamp(index=True) This is because ``time`` is a builtin Postgres type, and the ``CREATE INDEX`` statement being inspected in the database wrapped the column name in quotes, From ab5451038c594b8f2369ad1b8d7d12568d1b6558 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 11 Oct 2021 18:36:11 +0100 Subject: [PATCH 117/727] Update CHANGES --- CHANGES | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index c46477d7d..17abeeda3 100644 --- a/CHANGES +++ b/CHANGES @@ -64,8 +64,8 @@ get_output_schema ~~~~~~~~~~~~~~~~~ ``get_output_schema`` is the main entrypoint for database reflection in -Piccolo. It has been modified to accept an ``Engine`` argument, which makes it -more flexible. +Piccolo. It has been modified to accept an optional ``Engine`` argument, which +makes it more flexible. ------------------------------------------------------------------------------- From c774842cf12b8dc6666baa36a8447abba7648e74 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 11 Oct 2021 18:37:31 +0100 Subject: [PATCH 118/727] Update CHANGES --- CHANGES | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES b/CHANGES index 17abeeda3..c74252113 100644 --- a/CHANGES +++ b/CHANGES @@ -57,6 +57,8 @@ This makes writing unit tests easier, when we need to setup a schema. # Do some testing ... pass +The docs were updated to reflect this. + When dropping tables in a unit test, remember to use ``piccolo tester run``, to make sure the test database is used. From 14af797c4f613b4490fad3942b73a69dde512a88 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 11 Oct 2021 21:46:47 +0100 Subject: [PATCH 119/727] add Postgres 14 (#300) --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 691aeff55..0d99f338d 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -33,7 +33,7 @@ jobs: strategy: matrix: python-version: [3.7, 3.8, 3.9] - postgres-version: [9.6, 10, 11, 12, 13] + postgres-version: [9.6, 10, 11, 12, 13, 14] # Service containers to run with `container-job` services: From f94ec4f47a750c0aa0b9c4c9ce25b0b32e18a825 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 12 Oct 2021 22:19:12 +0100 Subject: [PATCH 120/727] Update contribution guide (#301) * Update CONTRIBUTING.md * improved wording --- CONTRIBUTING.md | 96 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 30756895b..5f5b25451 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1 +1,97 @@ +# Contributing + +Thanks for your interest in the Piccolo project. 👍 + +The aim of Piccolo is to build a fantastic ORM and query builder, with a world class admin GUI, which makes developers happy and productive. There are lots of ways to get involved. 🚀 + +The community is friendly and responsive. You don't have to be a Python or database ninja to contribute. 🥷 + +--- + +## Pick an appropriate issue + +If you take a look at the issues list, there are some with the [`good first issue`](https://github.com/piccolo-orm/piccolo/labels/good%20first%20issue) tag. You can pick any issue you feel confident in tackling, or even create your own issue. However, the ones marked with `good first issue` are created with newcomers in mind. + +Once you've identified an issue that you're interested in working on, leave a comment on the issue letting others know. This will prevent multiple people accidentally working on the same issue. + +### What if there are no appropriate issues? + +There are always ways to improve a project: + +- Can code coverage be improved? +- Is documentation lacking? +- Are there any typos? +- Can any code be optimised or cleaned up? + +If you can identify any areas of improvement, create an issue. + +--- + +## Tips for Pull Requests (PRs) + +### Try to keep PRs simple + +The maintainers do it alongside a day job, so very large PRs may take a long time to review. We try to leave feedback on PRs within a few days of them being opened to keep things flowing. + +### A PR doesn't have to be perfect before opening it + +It's often better to open a pull request with a simple prototype, and get feedback. + +### Avoid overly complex code + +Part of open source's appeal is anyone can contribute to it. To keep a codebase clean and maintainable, it's a constant battle to keep complexity in check. Bear this in mind when doing a PR. The code should be high quality and well documented. + +--- + +## What if my code doesn't get merged? + +Most contributions get merged. However, a PR can serve many purposes - it can spark discussion and ideas. There is no wasted effort in open source, as it always contributes to collective learning. If a PR doesn't get merged, it's usually because we decide on a different approach for solving the problem, or the complexity overhead with adding it outweighs the benefits. + +--- + +## Contributing without writing code + +Even without writing code there are lots of ways to get involved. + +### Documentation + +Is something in the documentation unclear or missing? These types of contibutions are invaluable. + +### Design input + +Is there something about the design which you don't like? Getting constructive user feedback is incredibly useful. + +### Tutorials + +Can you write a blog article or tutorial about Piccolo? + +### Spreading the word + +Just starring the project or a tweet helps us a lot. + +--- + +## Git usage + +If you're not confident with Git, then the [GitHub Desktop app](https://desktop.github.com/) is highly recommended. + +--- + +## Project setup + See the main [Piccolo docs](https://piccolo-orm.readthedocs.io/en/latest/piccolo/contributing/index.html). + +--- + +## Sister projects + +The main Piccolo repo is just one piece of the ecosystem. There are other essential components which are open to contributions: + +- https://github.com/piccolo-orm/piccolo_admin +- https://github.com/piccolo-orm/piccolo_api + +--- + +## Becoming a project member + +There is no formal process for becoming a project member. Typically people become members after making significant contributions to the project, or who require specific permissions which makes contributing to Piccolo easier. From 4d9ac132af68da1574bf8afbbe1729d27b760720 Mon Sep 17 00:00:00 2001 From: William Michael Short <36488354+wmshort@users.noreply.github.com> Date: Wed, 13 Oct 2021 10:14:49 +0100 Subject: [PATCH 121/727] Update CONTRIBUTING.md (#303) Corrected a typo in the Documentation section. --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5f5b25451..e45d15986 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,7 +55,7 @@ Even without writing code there are lots of ways to get involved. ### Documentation -Is something in the documentation unclear or missing? These types of contibutions are invaluable. +Is something in the documentation unclear or missing? These types of contributions are invaluable. ### Design input From 9ec3512c6137a47c6090d294b61f07aaebe29ace Mon Sep 17 00:00:00 2001 From: Kenneth Cheo Date: Thu, 14 Oct 2021 03:21:56 +0800 Subject: [PATCH 122/727] Add support for Python 3.10 (#304) Updated pytest version from 6.2.1 to 6.2.5 --- .github/workflows/tests.yaml | 6 +++--- requirements/test-requirements.txt | 2 +- setup.py | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 0d99f338d..a22673060 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9] + python-version: ['3.7', '3.8', '3.9', '3.10'] steps: - uses: actions/checkout@v2 @@ -32,7 +32,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9] + python-version: ['3.7', '3.8', '3.9', '3.10'] postgres-version: [9.6, 10, 11, 12, 13, 14] # Service containers to run with `container-job` @@ -87,7 +87,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9] + python-version: ['3.7', '3.8', '3.9', '3.10'] steps: - uses: actions/checkout@v2 diff --git a/requirements/test-requirements.txt b/requirements/test-requirements.txt index 5f744295f..346f95ba5 100644 --- a/requirements/test-requirements.txt +++ b/requirements/test-requirements.txt @@ -1,4 +1,4 @@ coveralls==2.2.0 pytest-cov==2.10.1 -pytest==6.2.1 +pytest==6.2.5 python-dateutil==2.8.1 diff --git a/setup.py b/setup.py index 6fa58f3a0..863cdfa91 100644 --- a/setup.py +++ b/setup.py @@ -81,6 +81,7 @@ def extras_require() -> t.Dict[str, t.List[str]]: "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Programming Language :: Python :: Implementation :: CPython", "Framework :: AsyncIO", "Typing :: Typed", From abf8bff7ab7e62235c2ca1bc9dbb5fc1bff5a59b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 13 Oct 2021 21:06:01 +0100 Subject: [PATCH 123/727] bumped version --- CHANGES | 6 ++++++ piccolo/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index c74252113..10e0c8c63 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,12 @@ Changes ======= +0.57.0 +------ + +Added Python 3.10 support (courtesy @kennethcheo). + +------------------------------------------------------------------------------- 0.56.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 93c4d84e0..eb9fe61d2 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.56.0" +__VERSION__ = "0.57.0" From 27539219431874bae99b7206df48133fbe1a27eb Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 13 Oct 2021 21:07:20 +0100 Subject: [PATCH 124/727] add missing line --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index 10e0c8c63..b186ad182 100644 --- a/CHANGES +++ b/CHANGES @@ -7,6 +7,7 @@ Changes Added Python 3.10 support (courtesy @kennethcheo). ------------------------------------------------------------------------------- + 0.56.0 ------ From 0f0343988621ef81a2b5059f76e64b92cd79e816 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Wed, 20 Oct 2021 00:14:12 +0200 Subject: [PATCH 125/727] adding serialization docs (#308) * adding serialization docs * added forgotten Python code block * tweak docs Co-authored-by: Daniel Townsend --- docs/src/index.rst | 1 + docs/src/piccolo/serialization/index.rst | 101 +++++++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 docs/src/piccolo/serialization/index.rst diff --git a/docs/src/index.rst b/docs/src/index.rst index 69d1d63d8..137b6bc8c 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -17,6 +17,7 @@ batteries included. piccolo/migrations/index piccolo/authentication/index piccolo/asgi/index + piccolo/serialization/index piccolo/testing/index piccolo/features/index piccolo/playground/index diff --git a/docs/src/piccolo/serialization/index.rst b/docs/src/piccolo/serialization/index.rst new file mode 100644 index 000000000..7447dee99 --- /dev/null +++ b/docs/src/piccolo/serialization/index.rst @@ -0,0 +1,101 @@ +Serialization +============= + +``Piccolo`` uses `Pydantic `_ +internally to serialize and deserialize data. + +------------------------------------------------------------------------------- + +create_pydantic_model +--------------------- + +Using ``create_pydantic_model`` we can easily create a `Pydantic model `_ +from a Piccolo ``Table``. + +Using this example schema: + +.. code-block:: python + + from piccolo.columns import ForeignKey, Integer, Varchar + from piccolo.table import Table + + class Manager(Table): + name = Varchar() + + class Band(Table): + name = Varchar(length=100) + manager = ForeignKey(Manager) + popularity = Integer() + +Creating a Pydantic model is as simple as: + +.. code-block:: python + + from piccolo.utils.pydantic import create_pydantic_model + + BandModel = create_pydantic_model(Band) + +You have several options for configuring the model, as shown below. + +exclude_columns +~~~~~~~~~~~~~~~ + +If we want to exclude the ``popularity`` column from the ``Band`` table: + +.. code-block:: python + + BandModel = create_pydantic_model(Band, exclude_columns=(Band.popularity,)) + +nested +~~~~~~ + +Another great feature is ``nested=True``. For each ``ForeignKey`` in the +Piccolo ``Table``, the Pydantic model will contain a sub model for the related +table. + +For example: + +.. code-block:: python + + BandModel = create_pydantic_model(Band, nested=True) + +If we were to write ``BandModel`` by hand instead, it would look like this: + +.. code-block:: python + + from pydantic import BaseModel + + class ManagerModel(BaseModel): + name: str + + class BandModel(BaseModel): + name: str + manager: ManagerModel + popularity: int + +But with ``nested=True`` we can achieve this with one line of code. + +Source +~~~~~~ + +.. automodule:: piccolo.utils.pydantic + :members: + +.. hint:: A good place to see ``create_pydantic_model`` in action is `PiccoloCRUD `_, + as it uses ``create_pydantic_model`` extensively to create Pydantic models + from Piccolo tables. + +------------------------------------------------------------------------------- + +FastAPI template +---------------- + +Piccolo's FastAPI template uses ``create_pydantic_model`` to create serializers. + +To create a new FastAPI app using Piccolo, simply use: + +.. code-block:: bash + + piccolo asgi new + +See the :ref:`ASGI docs ` for more details. From 735414e004c886783c4c80cd85d7ac5974fd96cf Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 19 Oct 2021 23:17:07 +0100 Subject: [PATCH 126/727] fix minor typo in docs --- docs/src/piccolo/serialization/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/piccolo/serialization/index.rst b/docs/src/piccolo/serialization/index.rst index 7447dee99..fcb6935cd 100644 --- a/docs/src/piccolo/serialization/index.rst +++ b/docs/src/piccolo/serialization/index.rst @@ -1,8 +1,8 @@ Serialization ============= -``Piccolo`` uses `Pydantic `_ -internally to serialize and deserialize data. +Piccolo uses `Pydantic `_ internally +to serialize and deserialize data. ------------------------------------------------------------------------------- From db48428b5cccb5107cd68b291350ffad88a59bcb Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 20 Oct 2021 12:57:14 +0100 Subject: [PATCH 127/727] add examples for how to populate the Pydantic models using data from Piccolo tables (#309) --- docs/src/piccolo/serialization/index.rst | 70 ++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/docs/src/piccolo/serialization/index.rst b/docs/src/piccolo/serialization/index.rst index fcb6935cd..91095ef1e 100644 --- a/docs/src/piccolo/serialization/index.rst +++ b/docs/src/piccolo/serialization/index.rst @@ -35,6 +35,23 @@ Creating a Pydantic model is as simple as: BandModel = create_pydantic_model(Band) +We can then create model instances from data we fetch from the database: + +.. code-block:: python + + # If using objects: + model = BandModel( + **Band.objects().get(Band.name == 'Pythonistas').run_sync().to_dict() + ) + + # If using select: + model = BandModel( + **Band.select().where(Band.name == 'Pythonistas').first().run_sync() + ) + + >>> model.name + 'Pythonistas' + You have several options for configuring the model, as shown below. exclude_columns @@ -75,6 +92,59 @@ If we were to write ``BandModel`` by hand instead, it would look like this: But with ``nested=True`` we can achieve this with one line of code. +To populate a nested Pydantic model with data from the database: + +.. code-block:: python + + # If using objects: + model = BandModel( + **Band.objects(Band.manager).get(Band.name == 'Pythonistas').run_sync().to_dict() + ) + + # If using select: + model = BandModel( + **Band.select( + Band.all_columns(), + Band.manager.all_columns() + ).where( + Band.name == 'Pythonistas' + ).first().output( + nested=True + ).run_sync() + ) + + >>> model.manager.name + 'Guido' + +include_default_columns +~~~~~~~~~~~~~~~~~~~~~~~ + +Sometimes you'll want to include the Piccolo ``Table``'s primary key column in +the generated Pydantic model. For example, in a ``GET`` endpoint, we usually +want to include the ``id`` in the response: + +.. code-block:: javascript + + // GET /api/bands/1/ + // Response: + {"id": 1, "name": "Pythonistas", "popularity": 1000} + +Other times, you won't want the Pydantic model to include the primary key +column. For example, in a ``POST`` endpoint, when using a Pydantic model to +serialise the payload, we don't expect the user to pass in an ``id`` value: + +.. code-block:: javascript + + // POST /api/bands/ + // Payload: + {"name": "Pythonistas", "popularity": 1000} + +By default the primary key column isn't included - you can add it using: + +.. code-block:: python + + BandModel = create_pydantic_model(Band, include_default_columns=True) + Source ~~~~~~ From dcf48c69335506a85fd625b0eff31e61ba2da310 Mon Sep 17 00:00:00 2001 From: Yasser Tahiri Date: Thu, 21 Oct 2021 21:34:03 +0100 Subject: [PATCH 128/727] Combines multiple `isinstance` functions (#310) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐍: Refactor app Folder and Fix Code Issue * ⚡️: Fix Engine Code * 🌪: Fix Issue relate to Return & Refactor the Code * ❄️: Refactor piccolo configuration * ☀️: Refactor Utils & Fix Code Issues * 🪶: Fix Queries Code * 🐍: Refactor Multipart of Code * 🐛: Extract Some Methods and Variables * 🪄: Revert `test_apps` & `test_insert` * 📦: Use `any()` instead of for loop * Combines multiple `isinstance` functions * Convert `for` loop into dictionary comprehension --- piccolo/table.py | 4 ++-- piccolo/table_reflection.py | 7 +++---- setup.py | 7 ++----- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/piccolo/table.py b/piccolo/table.py index 10847bed3..c6c273d44 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -1036,7 +1036,6 @@ def drop_tables(*tables: t.Type[Table]) -> None: for table in sorted_table_classes ] ) - atomic.run_sync() else: atomic = engine.atomic() atomic.add( @@ -1045,7 +1044,8 @@ def drop_tables(*tables: t.Type[Table]) -> None: for table in tables ] ) - atomic.run_sync() + + atomic.run_sync() def sort_table_classes( diff --git a/piccolo/table_reflection.py b/piccolo/table_reflection.py index 78bfa98f4..13f4d5c0f 100644 --- a/piccolo/table_reflection.py +++ b/piccolo/table_reflection.py @@ -24,8 +24,7 @@ class ImmutableDict(Immutable, dict): # type: ignore clear = pop = popitem = setdefault = update = Immutable._immutable # type: ignore # noqa: E501 def __new__(cls, *args): - new = dict.__new__(cls) - return new + return dict.__new__(cls) def copy(self): raise NotImplementedError( @@ -81,7 +80,7 @@ class TableStorage(metaclass=Singleton): def __init__(self): self.tables = ImmutableDict() - self._schema_tables = dict() + self._schema_tables = {} async def reflect( self, @@ -224,7 +223,7 @@ def _get_schema_and_table_name(tablename: str) -> TableNameDetail: def _to_list(value: t.Any) -> t.List: if isinstance(value, list): return value - elif isinstance(value, tuple) or isinstance(value, set): + elif isinstance(value, (tuple, set)): return list(value) elif isinstance(value, str): return [value] diff --git a/setup.py b/setup.py index 863cdfa91..295b97ef8 100644 --- a/setup.py +++ b/setup.py @@ -36,12 +36,9 @@ def extras_require() -> t.Dict[str, t.List[str]]: """ Parse requirements in requirements/extras directory """ - extra_requirements = {} - for extra in extras: - extra_requirements[extra] = parse_requirement( + extra_requirements = {extra: parse_requirement( os.path.join("extras", extra + ".txt") - ) - + ) for extra in extras} extra_requirements["all"] = [ i for i in itertools.chain.from_iterable(extra_requirements.values()) ] From d997e899e7b24e1cfa57b25199706b817591abe6 Mon Sep 17 00:00:00 2001 From: William Michael Short <36488354+wmshort@users.noreply.github.com> Date: Fri, 22 Oct 2021 10:04:19 +0100 Subject: [PATCH 129/727] correctly reflect self-referencing FKs (#305) * correctly reflect self-referencing FKs * fix typo * fix linting * fixed incompetency in type declaration! * add unit test * fix bug with adding a ForeignKey column using Alter.add_column * add additional tests for Alter.add_column * tweaked test Co-authored-by: William Michael Short Co-authored-by: Daniel Townsend --- piccolo/apps/schema/commands/generate.py | 26 +++++--- piccolo/columns/base.py | 2 +- piccolo/columns/column_types.py | 72 ++++++++++++++++++++- piccolo/query/methods/alter.py | 4 ++ piccolo/table.py | 58 ++--------------- tests/apps/schema/commands/test_generate.py | 24 +++++++ tests/table/test_alter.py | 53 ++++++++++----- 7 files changed, 159 insertions(+), 80 deletions(-) diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index d30d4b084..67c5f4407 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -686,16 +686,24 @@ async def create_table_class_from_db( constraint_schema=fk_constraint_table.schema, ) if constraint_table.name: - referenced_output_schema = await create_table_class_from_db( - table_class=table_class, - tablename=constraint_table.name, - schema_name=constraint_table.schema, - ) - referenced_table = ( - referenced_output_schema.get_table_with_name( - tablename=constraint_table.name + referenced_table: t.Union[str, t.Optional[t.Type[Table]]] + + if constraint_table.name == tablename: + referenced_output_schema = output_schema + referenced_table = "self" + else: + referenced_output_schema = ( + await create_table_class_from_db( + table_class=table_class, + tablename=constraint_table.name, + schema_name=constraint_table.schema, + ) + ) + referenced_table = ( + referenced_output_schema.get_table_with_name( + tablename=constraint_table.name + ) ) - ) kwargs["references"] = ( referenced_table if referenced_table is not None diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 9c7f88eb1..8d3fe444e 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -91,7 +91,7 @@ def resolved_references(self) -> t.Type[Table]: return self.references else: raise ValueError( - "The references attribute is neither a Table sublclass or a " + "The references attribute is neither a Table subclass or a " "LazyTableReference instance." ) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index d7c300f3c..28e880988 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -5,6 +5,7 @@ import inspect import typing as t import uuid +from dataclasses import dataclass from datetime import date, datetime, time, timedelta from enum import Enum @@ -1010,6 +1011,11 @@ def column_type(self): ############################################################################### +@dataclass +class ForeignKeySetupResponse: + is_lazy: bool + + class ForeignKey(Column): # lgtm [py/missing-equals] """ Used to reference another table. Uses the same type as the primary key @@ -1234,11 +1240,73 @@ def __init__( super().__init__(**kwargs) - # The Table metaclass sets the actual value for `table`: + # The ``TableMetaclass``` sets the actual value for + # ``ForeignKeyMeta.references``, if the user passed in a string. self._foreign_key_meta = ForeignKeyMeta( - references=Table, on_delete=on_delete, on_update=on_update + references=Table if isinstance(references, str) else references, + on_delete=on_delete, + on_update=on_update, + ) + + def _setup(self, table_class: t.Type[Table]) -> ForeignKeySetupResponse: + """ + This is called by the ``TableMetaclass``. A ``ForeignKey`` column can + only be completely setup once it's parent ``Table`` is known. + + :param table_class: + The parent ``Table`` class for this column. + + """ + from piccolo.table import Table + + params = self._meta.params + references = params["references"] + + if isinstance(references, str): + if references == "self": + references = table_class + else: + if "." in references: + # Don't allow relative modules - this may change in + # the future. + if references.startswith("."): + raise ValueError("Relative imports aren't allowed") + + module_path, table_class_name = references.rsplit( + ".", maxsplit=1 + ) + else: + table_class_name = references + module_path = table_class.__module__ + + references = LazyTableReference( + table_class_name=table_class_name, + module_path=module_path, + ) + + is_lazy = isinstance(references, LazyTableReference) + is_table_class = inspect.isclass(references) and issubclass( + references, Table ) + if is_lazy or is_table_class: + self._foreign_key_meta.references = references + else: + raise ValueError( + "Error - ``references`` must be a ``Table`` subclass, or " + "a ``LazyTableReference`` instance." + ) + + if is_table_class: + # Record the reverse relationship on the target table. + references._meta._foreign_key_references.append(self) + + # Allow columns on the referenced table to be accessed via + # auto completion. + self.set_proxy_columns() + + return ForeignKeySetupResponse(is_lazy=is_lazy) + def copy(self) -> ForeignKey: column: ForeignKey = copy.copy(self) column._meta = self._meta.copy() diff --git a/piccolo/query/methods/alter.py b/piccolo/query/methods/alter.py index ca6fb2bf5..48aa240ad 100644 --- a/piccolo/query/methods/alter.py +++ b/piccolo/query/methods/alter.py @@ -305,6 +305,10 @@ def add_column(self, name: str, column: Column) -> Alter: column._meta._table = self.table column._meta._name = name column._meta.db_column_name = name + + if isinstance(column, ForeignKey): + column._setup(table_class=self.table) + self._add.append(AddColumn(column, name)) return self diff --git a/piccolo/table.py b/piccolo/table.py index c6c273d44..25271dd3a 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -17,10 +17,7 @@ from piccolo.columns.defaults.base import Default from piccolo.columns.indexes import IndexMethod from piccolo.columns.readable import Readable -from piccolo.columns.reference import ( - LAZY_COLUMN_REFERENCES, - LazyTableReference, -) +from piccolo.columns.reference import LAZY_COLUMN_REFERENCES from piccolo.engine import Engine, engine_finder from piccolo.query import ( Alter, @@ -240,59 +237,16 @@ def __init_subclass__( ) for foreign_key_column in foreign_key_columns: - params = foreign_key_column._meta.params - references = params["references"] - - if isinstance(references, str): - if references == "self": - references = cls - else: - if "." in references: - # Don't allow relative modules - this may change in - # the future. - if references.startswith("."): - raise ValueError("Relative imports aren't allowed") - - module_path, table_class_name = references.rsplit( - ".", maxsplit=1 - ) - else: - table_class_name = references - module_path = cls.__module__ - - references = LazyTableReference( - table_class_name=table_class_name, - module_path=module_path, - ) - - is_lazy = isinstance(references, LazyTableReference) - is_table_class = inspect.isclass(references) and issubclass( - references, Table + # ForeignKey columns require additional setup based on their + # parent Table. + foreign_key_setup_response = foreign_key_column._setup( + table_class=cls ) - - if is_lazy or is_table_class: - foreign_key_column._foreign_key_meta.references = references - else: - raise ValueError( - "Error - ``references`` must be a ``Table`` subclass, or " - "a ``LazyTableReference`` instance." - ) - - # Record the reverse relationship on the target table. - if is_table_class: - references._meta._foreign_key_references.append( - foreign_key_column - ) - elif is_lazy: + if foreign_key_setup_response.is_lazy: LAZY_COLUMN_REFERENCES.foreign_key_columns.append( foreign_key_column ) - # Allow columns on the referenced table to be accessed via - # auto completion. - if is_table_class: - foreign_key_column.set_proxy_columns() - TABLE_REGISTRY.append(cls) def __init__( diff --git a/tests/apps/schema/commands/test_generate.py b/tests/apps/schema/commands/test_generate.py index ac0f8a66c..a46dea39e 100644 --- a/tests/apps/schema/commands/test_generate.py +++ b/tests/apps/schema/commands/test_generate.py @@ -141,6 +141,30 @@ def test_exclude_table(self): SmallTable_ = output_schema.get_table_with_name("SmallTable") self._compare_table_columns(SmallTable, SmallTable_) + def test_self_referencing_fk(self): + """ + Make sure self-referencing foreign keys are handled correctly. + """ + + MegaTable.alter().add_column( + "self_referencing_fk", ForeignKey("self") + ).run_sync() + + output_schema: OutputSchema = run_sync(get_output_schema()) + + # Make sure the 'references' value of the generated column is "self". + for table in output_schema.tables: + if table.__name__ == "MegaTable": + column: ForeignKey = output_schema.tables[ + 1 + ].self_referencing_fk + + self.assertEqual( + column._foreign_key_meta.references._meta.tablename, + MegaTable._meta.tablename, + ) + self.assertEqual(column._meta.params["references"], "self") + ############################################################################### diff --git a/tests/table/test_alter.py b/tests/table/test_alter.py index 684340f58..7a60f4359 100644 --- a/tests/table/test_alter.py +++ b/tests/table/test_alter.py @@ -1,14 +1,20 @@ +from __future__ import annotations + +import typing as t from unittest import TestCase from piccolo.columns import BigInt, Integer, Numeric, Varchar +from piccolo.columns.column_types import ForeignKey, Text from piccolo.table import Table +from tests.base import DBTestCase, postgres_only from tests.example_apps.music.tables import Band, Manager -from ..base import DBTestCase, postgres_only +if t.TYPE_CHECKING: + from piccolo.columns.base import Column class TestRenameColumn(DBTestCase): - def _test_rename(self, column): + def _test_rename(self, column: Column): self.insert_row() rename_query = Band.alter().rename_column(column, "rating") @@ -64,26 +70,41 @@ def test_drop_column(self): self._test_drop("popularity") -class TestAdd(DBTestCase): - def test_add(self): - """ - This needs a lot more work. Need to set values for existing rows. - - Just write the test for now ... - """ +class TestAddColumn(DBTestCase): + def _test_add_column( + self, column: Column, column_name: str, expected_value: t.Any + ): self.insert_row() - - add_query = Band.alter().add_column( - "weight", Integer(null=True, default=None) - ) - add_query.run_sync() + Band.alter().add_column(column_name, column).run_sync() response = Band.raw("SELECT * FROM band").run_sync() column_names = response[0].keys() - self.assertTrue("weight" in column_names) + self.assertTrue(column_name in column_names) - self.assertEqual(response[0]["weight"], None) + self.assertEqual(response[0][column_name], expected_value) + + def test_add_integer(self): + self._test_add_column( + column=Integer(null=True, default=None), + column_name="members", + expected_value=None, + ) + + def test_add_foreign_key(self): + self._test_add_column( + column=ForeignKey(references=Manager), + column_name="assistant_manager", + expected_value=None, + ) + + def test_add_text(self): + bio = "An amazing band" + self._test_add_column( + column=Text(default=bio), + column_name="bio", + expected_value=bio, + ) @postgres_only From 3c6f72503064f9b003de8d243b7fd8d0939764be Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 22 Oct 2021 10:06:06 +0100 Subject: [PATCH 130/727] rename CHANGES to CHANGES.rst (#311) --- CHANGES => CHANGES.rst | 0 docs/src/piccolo/changes/index.rst | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename CHANGES => CHANGES.rst (100%) diff --git a/CHANGES b/CHANGES.rst similarity index 100% rename from CHANGES rename to CHANGES.rst diff --git a/docs/src/piccolo/changes/index.rst b/docs/src/piccolo/changes/index.rst index d2f9f30f4..55704f02a 100644 --- a/docs/src/piccolo/changes/index.rst +++ b/docs/src/piccolo/changes/index.rst @@ -1 +1 @@ -.. include:: ../../../../CHANGES +.. include:: ../../../../CHANGES.rst From 24a319182423ec0c246bc7d85c1c27f2b6c22f48 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 22 Oct 2021 10:48:46 +0100 Subject: [PATCH 131/727] make lint.sh formatters fail build (#312) --- scripts/README.md | 11 ++++++----- scripts/format.sh | 9 +++++++++ scripts/lint.sh | 18 ++++++++++++++++-- 3 files changed, 31 insertions(+), 7 deletions(-) create mode 100755 scripts/format.sh diff --git a/scripts/README.md b/scripts/README.md index 10ba8b7b6..2140f82ad 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -4,8 +4,9 @@ The scripts follow GitHub's ["Scripts to Rule Them All"](https://github.com/gith Call them from the root of the project, e.g. `./scripts/lint.sh`. -* `scripts/lint.sh` - Run the automated code linting/formatting tools. -* `scripts/piccolo.sh` - Run the Piccolo CLI on the example project in the `tests` folder. -* `scripts/release.sh` - Publish package to PyPI. -* `scripts/test-postgres.sh` - Run the test suite with Postgres. -* `scripts/test-sqlite.sh` - Run the test suite with SQLite. +- `scripts/format.sh` - Format the code to the required standards. +- `scripts/lint.sh` - Run the automated code linting/formatting tools. +- `scripts/piccolo.sh` - Run the Piccolo CLI on the example project in the `tests` folder. +- `scripts/release.sh` - Publish package to PyPI. +- `scripts/test-postgres.sh` - Run the test suite with Postgres. +- `scripts/test-sqlite.sh` - Run the test suite with SQLite. diff --git a/scripts/format.sh b/scripts/format.sh new file mode 100755 index 000000000..482e0a21d --- /dev/null +++ b/scripts/format.sh @@ -0,0 +1,9 @@ +#!/bin/bash +SOURCES="piccolo tests" + +echo "Running isort..." +isort $SOURCES +echo "-----" + +echo "Running black..." +black $SOURCES diff --git a/scripts/lint.sh b/scripts/lint.sh index 5326db07c..3908d8377 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -1,8 +1,22 @@ #!/bin/bash +set -e SOURCES="piccolo tests" -isort $SOURCES -black $SOURCES +echo "Running isort..." +isort --check $SOURCES +echo "-----" + +echo "Running black..." +black --check $SOURCES +echo "-----" + +echo "Running flake8..." flake8 $SOURCES +echo "-----" + +echo "Running mypy..." mypy $SOURCES +echo "-----" + +echo "All passed!" From 736311cd8351f859dfe0e2b526172a18bf05ae09 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 24 Oct 2021 20:16:47 +0100 Subject: [PATCH 132/727] allow additional fields to be added to the Pydantic model's schema (#313) * allow additional fields to be added to the Pydantic model's schema * fix typo * add docs for schema_extra_kwargs * add a test for schema_extra_kwargs --- piccolo/utils/pydantic.py | 18 ++++++++++++++++-- tests/utils/test_pydantic.py | 14 ++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/piccolo/utils/pydantic.py b/piccolo/utils/pydantic.py index 90c2fe17d..8767ebafc 100644 --- a/piccolo/utils/pydantic.py +++ b/piccolo/utils/pydantic.py @@ -54,6 +54,7 @@ def create_pydantic_model( all_optional: bool = False, model_name: t.Optional[str] = None, deserialize_json: bool = False, + **schema_extra_kwargs, ) -> t.Type[pydantic.BaseModel]: """ Create a Pydantic model representing a table. @@ -80,9 +81,19 @@ def create_pydantic_model( you can override it if you want multiple Pydantic models based off the same Piccolo table. :param deserialize_json: - By default, the values of any Piccolo JSON or JSONB columns are + By default, the values of any Piccolo ``JSON`` or ``JSONB`` columns are returned as strings. By setting this parameter to True, they will be returned as objects. + :param schema_extra_kwargs: + This can be used to add additional fields to the schema. This is + very useful when using Pydantic's JSON Schema features. For example: + + .. code-block:: python + + >>> my_model = create_pydantic_model(Band, my_extra_field="Hello") + >>> my_model.schema() + {..., "my_extra_field": "Hello"} + :returns: A Pydantic model. @@ -187,7 +198,10 @@ def create_pydantic_model( model_name = model_name or table.__name__ class CustomConfig(Config): - schema_extra = {"help_text": table._meta.help_text} + schema_extra = { + "help_text": table._meta.help_text, + **schema_extra_kwargs, + } return pydantic.create_model( model_name, diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index 01cfbcb77..2dd48ef6e 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -369,3 +369,17 @@ class Band(Table): model = BandModel(regrettable_column_name="test") self.assertTrue(model.name == "test") + + +class TestSchemaExtraKwargs(TestCase): + def test_schema_extra_kwargs(self): + """ + Make sure that the ``schema_extra_kwargs`` arguments are reflected in + Pydantic model's schema. + """ + + class Band(Table): + name = Varchar() + + model = create_pydantic_model(Band, visible_columns=("name",)) + self.assertEqual(model.schema()["visible_columns"], ("name",)) From 0eb325f3083bac50e6241d6df3d04fe3dc095cf6 Mon Sep 17 00:00:00 2001 From: Oscar Butler-Aldridge Date: Sun, 24 Oct 2021 12:40:35 -0700 Subject: [PATCH 133/727] Fix auto migration import conflict2 (#307) * Add integration tests for numeric and decimal (#306) Adding integration tests for the numeric and decimal column types recreates the expected behaviour where the the numeric test passes and the decimal test fails. * Add unique global names to serialisation (#306) Adds a UniqueGlobalNames class to serialisation that stores global names as class attributes. These attributes are meant to be used during serialisation to prevent conflicting global names in the generated code. As a result of runtime conflict prevention the following errors are raises while running the test suite: - `piccolo.apps.migrations.auto.serialisation.UniqueGlobalNameConflictError: Import 'Decimal' could conflict with global name` - `piccolo.apps.migrations.auto.serialisation.UniqueGlobalNameConflictError: Import 'UUID' could conflict with global name` These error match the errors found in branch /0scarB/piccolo/tree/fix-auto-migration-import-conflict * Fix serialisation for decimal.Decimal and uuid.UUID (#306) Column arguments with values of type decimal.Decimal and uuid.UUID now use absolute imports, e.g.: ```python import decimal import uuid ... async def forwards(): ... manager.add_column( ... params={ ... "default": decimal.Decimal("0"), ... }, ) manager.add_column( ... params={ ... "default": uuid.UUID("00000000-0000-0000-0000-000000000000"), ... }, ) ... ``` This avoids conflicts with `piccolo.columns.column_types.Decimal` and `piccolo.columns.column_types.UUID`. * Change conflict error to warning * remove filterwarnings for now Co-authored-by: Daniel Townsend --- piccolo/apps/migrations/auto/schema_differ.py | 31 +- piccolo/apps/migrations/auto/serialisation.py | 351 ++++++++++++++++-- .../auto/integration/test_migrations.py | 56 +++ .../migrations/auto/test_serialisation.py | 183 ++++++++- 4 files changed, 580 insertions(+), 41 deletions(-) diff --git a/piccolo/apps/migrations/auto/schema_differ.py b/piccolo/apps/migrations/auto/schema_differ.py index 2abb66b9e..859bf662b 100644 --- a/piccolo/apps/migrations/auto/schema_differ.py +++ b/piccolo/apps/migrations/auto/schema_differ.py @@ -9,7 +9,12 @@ TableDelta, ) from piccolo.apps.migrations.auto.operations import RenameColumn, RenameTable -from piccolo.apps.migrations.auto.serialisation import Import, serialise_params +from piccolo.apps.migrations.auto.serialisation import ( + Definition, + Import, + UniqueGlobalNames, + serialise_params, +) from piccolo.utils.printing import get_fixed_length_string @@ -70,7 +75,7 @@ def new_column_names(self): class AlterStatements: statements: t.List[str] extra_imports: t.List[Import] = field(default_factory=list) - extra_definitions: t.List[str] = field(default_factory=list) + extra_definitions: t.List[Definition] = field(default_factory=list) @dataclass @@ -307,7 +312,7 @@ def _get_snapshot_table( def alter_columns(self) -> AlterStatements: response: t.List[str] = [] extra_imports: t.List[Import] = [] - extra_definitions: t.List[str] = [] + extra_definitions: t.List[Definition] = [] for table in self.schema: snapshot_table = self._get_snapshot_table(table.class_name) if snapshot_table: @@ -341,6 +346,10 @@ def alter_columns(self) -> AlterStatements: Import( module=alter_column.column_class.__module__, target=alter_column.column_class.__name__, + expect_conflict_with_global_name=getattr( + UniqueGlobalNames, + f"COLUMN_{alter_column.column_class.__name__.upper()}", # noqa: E501 + ), ) ) @@ -349,6 +358,10 @@ def alter_columns(self) -> AlterStatements: Import( module=alter_column.old_column_class.__module__, target=alter_column.old_column_class.__name__, + expect_conflict_with_global_name=getattr( + UniqueGlobalNames, + f"COLUMN_{alter_column.old_column_class.__name__.upper()}", # noqa: E501 + ), ) ) @@ -388,7 +401,7 @@ def drop_columns(self) -> AlterStatements: def add_columns(self) -> AlterStatements: response: t.List[str] = [] extra_imports: t.List[Import] = [] - extra_definitions: t.List[str] = [] + extra_definitions: t.List[Definition] = [] for table in self.schema: snapshot_table = self._get_snapshot_table(table.class_name) if snapshot_table: @@ -413,6 +426,10 @@ def add_columns(self) -> AlterStatements: Import( module=column_class.__module__, target=column_class.__name__, + expect_conflict_with_global_name=getattr( + UniqueGlobalNames, + f"COLUMN_{column_class.__name__.upper()}", + ), ) ) @@ -444,7 +461,7 @@ def new_table_columns(self) -> AlterStatements: response: t.List[str] = [] extra_imports: t.List[Import] = [] - extra_definitions: t.List[str] = [] + extra_definitions: t.List[Definition] = [] for table in new_tables: if ( table.class_name @@ -464,6 +481,10 @@ def new_table_columns(self) -> AlterStatements: Import( module=column.__class__.__module__, target=column.__class__.__name__, + expect_conflict_with_global_name=getattr( + UniqueGlobalNames, + f"COLUMN_{column.__class__.__name__.upper()}", + ), ) ) diff --git a/piccolo/apps/migrations/auto/serialisation.py b/piccolo/apps/migrations/auto/serialisation.py index 087b64a30..513e7a7ec 100644 --- a/piccolo/apps/migrations/auto/serialisation.py +++ b/piccolo/apps/migrations/auto/serialisation.py @@ -1,11 +1,13 @@ from __future__ import annotations +import abc import builtins import datetime import decimal import inspect import typing as t import uuid +import warnings from copy import deepcopy from dataclasses import dataclass, field from enum import Enum @@ -21,6 +23,238 @@ ############################################################################### +class CanConflictWithGlobalNames(abc.ABC): + @abc.abstractmethod + def warn_if_is_conflicting_with_global_name(self): + ... + + +class UniqueGlobalNamesMeta(type): + """ + Metaclass for ``UniqueGlobalNames``. + + Fulfills the following functions: + + - Assure that no two class attributes have the same value. + - Add class attributes `COLUMN_` + to the class for each column type. + """ + + def __new__(mcs, name, bases, class_attributes): + class_attributes_with_columns = mcs.merge_class_attributes( + class_attributes, + mcs.get_column_class_attributes(), + ) + + return super().__new__( + mcs, + name, + bases, + mcs.merge_class_attributes( + class_attributes_with_columns, + dict( + unique_names=mcs.get_unique_class_attribute_values( + class_attributes_with_columns + ) + ), + ), + ) + + @staticmethod + def get_unique_class_attribute_values( + class_attributes: t.Dict[str, t.Any] + ) -> t.Set[t.Any]: + """ + Return class attribute values. + + Raises an error if attribute values are not unique. + """ + + unique_attribute_values = set() + for attribute_name, attribute_value in class_attributes.items(): + # Skip special attributes, i.e. "____" + if attribute_name.startswith("__") and attribute_name.endswith( + "__" + ): + continue + + if attribute_value in unique_attribute_values: + raise ValueError( + f"Duplicate unique global name {attribute_value}" + ) + unique_attribute_values.add(attribute_value) + + return unique_attribute_values + + @staticmethod + def merge_class_attributes( + class_attributes1: t.Dict[str, t.Any], + class_attributes2: t.Dict[str, t.Any], + ) -> t.Dict[str, t.Any]: + """ + Merges two class attribute dictionaries. + + Raise an error if both dictionaries have an attribute + with the same name. + """ + + for attribute_name in class_attributes2: + if attribute_name in class_attributes1: + raise ValueError(f"Duplicate class attribute {attribute_name}") + + return dict(**class_attributes1, **class_attributes2) + + @staticmethod + def get_column_class_attributes() -> t.Dict[str, str]: + """Automatically generates global names for each column type.""" + + import piccolo.columns.column_types + + class_attributes: t.Dict[str, str] = {} + for module_global in piccolo.columns.column_types.__dict__.values(): + try: + if module_global is not Column and issubclass( + module_global, Column + ): + class_attributes[ + f"COLUMN_{module_global.__name__.upper()}" + ] = module_global.__name__ + except TypeError: + pass + + return class_attributes + + +class UniqueGlobalNames(metaclass=UniqueGlobalNamesMeta): + """ + Contains global names that may be used during serialisation. + + The global names are stored as class attributes. Names that may + occur in the global namespace after serialisation should be listed here. + + This class is meant to prevent against the use of conflicting global + names. If possible imports and global definitions should use this + class to ensure that no conflicts arise during serialisation. + """ + + # Piccolo imports + TABLE = Table.__name__ + DEFAULT = Default.__name__ + # Column types are omitted because they are added by metaclass + + # Standard library imports + STD_LIB_ENUM = Enum.__name__ + STD_LIB_MODULE_DECIMAL = "decimal" + + # Third-party library imports + EXTERNAL_MODULE_UUID = "uuid" + EXTERNAL_UUID = f"{EXTERNAL_MODULE_UUID}.{uuid.UUID.__name__}" + + # This attribute is set in metaclass + unique_names: t.Set[str] + + @classmethod + def warn_if_is_conflicting_name( + cls, name: str, name_type: str = "Name" + ) -> None: + """Raise an error if ``name`` matches a class attribute value.""" + + if cls.is_conflicting_name(name): + warnings.warn( + f"{name_type} '{name}' could conflict with global name", + UniqueGlobalNameConflictWarning, + ) + + @classmethod + def is_conflicting_name(cls, name: str) -> bool: + """Check if ``name`` matches a class attribute value.""" + + return name in cls.unique_names + + @staticmethod + def warn_if_are_conflicting_objects( + objects: t.Iterable[CanConflictWithGlobalNames], + ) -> None: + """ + Call each object's ``raise_if_is_conflicting_with_global_name`` method. + """ + + for obj in objects: + obj.warn_if_is_conflicting_with_global_name() + + +class UniqueGlobalNameConflictWarning(UserWarning): + pass + + +############################################################################### + + +@dataclass +class Import(CanConflictWithGlobalNames): + module: str + target: t.Optional[str] = None + expect_conflict_with_global_name: t.Optional[str] = None + + def __post_init__(self) -> None: + if ( + self.expect_conflict_with_global_name is not None + and not UniqueGlobalNames.is_conflicting_name( + self.expect_conflict_with_global_name + ) + ): + raise ValueError( + f"`expect_conflict_with_global_name=" + f'"{self.expect_conflict_with_global_name}"` ' + f"is not listed in `{UniqueGlobalNames.__name__}`" + ) + + def __repr__(self): + if self.target is None: + return f"import {self.module}" + + return f"from {self.module} import {self.target}" + + def __hash__(self): + if self.target is None: + return hash(f"{self.module}") + + return hash(f"{self.module}-{self.target}") + + def __lt__(self, other): + return repr(self) < repr(other) + + def warn_if_is_conflicting_with_global_name(self): + if self.target is None: + name = self.module + else: + name = self.target + + if name == self.expect_conflict_with_global_name: + return + + if UniqueGlobalNames.is_conflicting_name(name): + UniqueGlobalNames.warn_if_is_conflicting_name( + name, name_type="Import" + ) + + +class Definition(CanConflictWithGlobalNames, abc.ABC): + @abc.abstractmethod + def __repr__(self): + ... + + +@dataclass +class SerialisedParams: + params: t.Dict[str, t.Any] + extra_imports: t.List[Import] + extra_definitions: t.List[Definition] = field(default_factory=list) + + +############################################################################### + + def check_equality(self, other): if getattr(other, "__hash__", None) is not None: return self.__hash__() == other.__hash__() @@ -91,7 +325,7 @@ def __repr__(self): @dataclass -class SerialisedTableType: +class SerialisedTableType(Definition): table_type: t.Type[Table] def __hash__(self): @@ -104,7 +338,6 @@ def __eq__(self, other): def __repr__(self): tablename = self.table_type._meta.tablename - class_name = self.table_type.__name__ # We have to add the primary key column definition too, so foreign # keys can be created with the correct type. @@ -116,13 +349,21 @@ def __repr__(self): ) return ( - f'class {class_name}(Table, tablename="{tablename}"): ' + f"class {self.table_class_name}" + f'({UniqueGlobalNames.TABLE}, tablename="{tablename}"): ' f"{pk_column_name} = {serialised_pk_column}" ) def __lt__(self, other): return repr(self) < repr(other) + @property + def table_class_name(self) -> str: + return self.table_type.__name__ + + def warn_if_is_conflicting_with_global_name(self) -> None: + UniqueGlobalNames.warn_if_is_conflicting_name(self.table_class_name) + @dataclass class SerialisedEnumType: @@ -137,7 +378,7 @@ def __eq__(self, other): def __repr__(self): class_name = self.enum_type.__name__ params = {i.name: i.value for i in self.enum_type} - return f"Enum('{class_name}', {params})" + return f"{UniqueGlobalNames.STD_LIB_ENUM}('{class_name}', {params})" @dataclass @@ -165,32 +406,23 @@ def __eq__(self, other): return check_equality(self, other) def __repr__(self): - return f"UUID('{str(self.instance)}')" - - -############################################################################### + return f'{UniqueGlobalNames.EXTERNAL_UUID}("{str(self.instance)}")' @dataclass -class Import: - module: str - target: str - - def __repr__(self): - return f"from {self.module} import {self.target}" +class SerialisedDecimal: + instance: decimal.Decimal def __hash__(self): - return hash(f"{self.module}-{self.target}") - - def __lt__(self, other): - return repr(self) < repr(other) + return hash(repr(self)) + def __eq__(self, other): + return check_equality(self, other) -@dataclass -class SerialisedParams: - params: t.Dict[str, t.Any] - extra_imports: t.List[Import] - extra_definitions: t.List[str] = field(default_factory=list) + def __repr__(self): + return f"{UniqueGlobalNames.STD_LIB_MODULE_DECIMAL}." + repr( + self.instance + ).replace("'", '"') ############################################################################### @@ -203,7 +435,7 @@ def serialise_params(params: t.Dict[str, t.Any]) -> SerialisedParams: """ params = deepcopy(params) extra_imports: t.List[Import] = [] - extra_definitions: t.List[t.Any] = [] + extra_definitions: t.List[Definition] = [] for key, value in params.items(): @@ -224,10 +456,15 @@ def serialise_params(params: t.Dict[str, t.Any]) -> SerialisedParams: extra_imports.extend(serialised_params.extra_imports) extra_definitions.extend(serialised_params.extra_definitions) + column_class_name = column.__class__.__name__ extra_imports.append( Import( module=column.__class__.__module__, - target=column.__class__.__name__, + target=column_class_name, + expect_conflict_with_global_name=getattr( + UniqueGlobalNames, + f"COLUMN_{column_class_name.upper()}", + ), ) ) params[key] = SerialisedColumnInstance( @@ -242,6 +479,7 @@ def serialise_params(params: t.Dict[str, t.Any]) -> SerialisedParams: Import( module=value.__class__.__module__, target=value.__class__.__name__, + expect_conflict_with_global_name=UniqueGlobalNames.DEFAULT, ) ) continue @@ -262,13 +500,27 @@ def serialise_params(params: t.Dict[str, t.Any]) -> SerialisedParams: # UUIDs if isinstance(value, uuid.UUID): params[key] = SerialisedUUID(instance=value) - extra_imports.append(Import(module="uuid", target="UUID")) + extra_imports.append( + Import( + module=UniqueGlobalNames.EXTERNAL_MODULE_UUID, + expect_conflict_with_global_name=( + UniqueGlobalNames.EXTERNAL_MODULE_UUID + ), + ) + ) continue # Decimals if isinstance(value, decimal.Decimal): - # Already has a good __repr__. - extra_imports.append(Import(module="decimal", target="Decimal")) + params[key] = SerialisedDecimal(instance=value) + extra_imports.append( + Import( + module=UniqueGlobalNames.STD_LIB_MODULE_DECIMAL, + expect_conflict_with_global_name=( + UniqueGlobalNames.STD_LIB_MODULE_DECIMAL + ), + ) + ) continue # Enum instances @@ -299,7 +551,15 @@ def serialise_params(params: t.Dict[str, t.Any]) -> SerialisedParams: # Enum types if inspect.isclass(value) and issubclass(value, Enum): params[key] = SerialisedEnumType(enum_type=value) - extra_imports.append(Import(module="enum", target="Enum")) + extra_imports.append( + Import( + module="enum", + target=UniqueGlobalNames.STD_LIB_ENUM, + expect_conflict_with_global_name=( + UniqueGlobalNames.STD_LIB_ENUM + ), + ) + ) for member in value: type_ = type(member.value) module = inspect.getmodule(type_) @@ -330,7 +590,11 @@ def serialise_params(params: t.Dict[str, t.Any]) -> SerialisedParams: SerialisedTableType(table_type=table_type) ) extra_imports.append( - Import(module=Table.__module__, target="Table") + Import( + module=Table.__module__, + target=UniqueGlobalNames.TABLE, + expect_conflict_with_global_name=UniqueGlobalNames.TABLE, + ) ) continue @@ -339,13 +603,22 @@ def serialise_params(params: t.Dict[str, t.Any]) -> SerialisedParams: params[key] = SerialisedCallable(callable_=value) extra_definitions.append(SerialisedTableType(table_type=value)) extra_imports.append( - Import(module=Table.__module__, target="Table") + Import( + module=Table.__module__, + target=UniqueGlobalNames.TABLE, + expect_conflict_with_global_name=UniqueGlobalNames.TABLE, + ) ) + primary_key_class = value._meta.primary_key.__class__ extra_imports.append( Import( - module=value._meta.primary_key.__class__.__module__, - target=value._meta.primary_key.__class__.__name__, + module=primary_key_class.__module__, + target=primary_key_class.__name__, + expect_conflict_with_global_name=getattr( + UniqueGlobalNames, + f"COLUMN_{primary_key_class.__name__.upper()}", + ), ) ) # Include the extra imports and definitions required for the @@ -368,10 +641,16 @@ def serialise_params(params: t.Dict[str, t.Any]) -> SerialisedParams: # All other types can remain as is. + unique_extra_imports = [i for i in set(extra_imports)] + UniqueGlobalNames.warn_if_are_conflicting_objects(unique_extra_imports) + + unique_extra_definitions = [i for i in set(extra_definitions)] + UniqueGlobalNames.warn_if_are_conflicting_objects(unique_extra_definitions) + return SerialisedParams( params=params, - extra_imports=[i for i in set(extra_imports)], - extra_definitions=[i for i in set(extra_definitions)], + extra_imports=unique_extra_imports, + extra_definitions=unique_extra_definitions, ) @@ -391,6 +670,8 @@ def deserialise_params(params: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]: params[key] = value.instance elif isinstance(value, SerialisedUUID): params[key] = value.instance + elif isinstance(value, SerialisedDecimal): + params[key] = value.instance elif isinstance(value, SerialisedCallable): params[key] = value.callable_ elif isinstance(value, SerialisedTableType): diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index 150c9c5c3..6be92a612 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -1,6 +1,7 @@ from __future__ import annotations import datetime +import decimal import os import shutil import tempfile @@ -25,6 +26,7 @@ BigSerial, Boolean, Date, + Decimal, DoublePrecision, Integer, Interval, @@ -80,6 +82,10 @@ def boolean_default(): return True +def numeric_default(): + return decimal.Decimal("1.2") + + def array_default_integer(): return [4, 5, 6] @@ -487,6 +493,56 @@ def test_boolean_column(self): ), ) + def test_numeric_column(self): + self._test_migrations( + table_classes=[ + self.table(column) + for column in [ + Numeric(), + Numeric(digits=(4, 2)), + Numeric(digits=None), + Numeric(default=decimal.Decimal("1.2")), + Numeric(default=numeric_default), + Numeric(null=True, default=None), + Numeric(null=False), + Numeric(index=True), + Numeric(index=False), + ] + ], + test_function=lambda x: all( + [ + x.data_type == "numeric", + x.is_nullable == "NO", + x.column_default == "0", + ] + ), + ) + + def test_decimal_column(self): + self._test_migrations( + table_classes=[ + self.table(column) + for column in [ + Decimal(), + Decimal(digits=(4, 2)), + Decimal(digits=None), + Decimal(default=decimal.Decimal("1.2")), + Decimal(default=numeric_default), + Decimal(null=True, default=None), + Decimal(null=False), + Decimal(index=True), + Decimal(index=False), + ] + ], + test_function=lambda x: all( + [ + x.data_type == "numeric", + x.is_nullable == "NO", + x.column_default == "0", + ] + ), + ) + def test_array_column_integer(self): self._test_migrations( table_classes=[ diff --git a/tests/apps/migrations/auto/test_serialisation.py b/tests/apps/migrations/auto/test_serialisation.py index b236573ac..8b4c46d9f 100644 --- a/tests/apps/migrations/auto/test_serialisation.py +++ b/tests/apps/migrations/auto/test_serialisation.py @@ -1,7 +1,19 @@ +import decimal +import uuid +import warnings from enum import Enum from unittest import TestCase -from piccolo.apps.migrations.auto.serialisation import serialise_params +import pytest + +from piccolo.apps.migrations.auto.serialisation import ( + CanConflictWithGlobalNames, + Import, + UniqueGlobalNameConflictWarning, + UniqueGlobalNames, + UniqueGlobalNamesMeta, + serialise_params, +) from piccolo.columns.base import OnDelete from piccolo.columns.choices import Choice from piccolo.columns.column_types import Varchar @@ -9,6 +21,163 @@ from piccolo.columns.reference import LazyTableReference +class TestUniqueGlobalNamesMeta: + def test_duplicate_class_attribute_values_raises_error(self): + with pytest.raises(ValueError): + + class IncorrectUniqueGlobalNames(metaclass=UniqueGlobalNamesMeta): + A = "duplicate" + B = "duplicate" + + +class TestUniqueGlobals: + def test_contains_column_types(self): + assert getattr(UniqueGlobalNames, "COLUMN_VARCHAR", "Varchar") + assert getattr(UniqueGlobalNames, "COLUMN_SECRET", "Secret") + assert getattr(UniqueGlobalNames, "COLUMN_TEXT", "Text") + assert getattr(UniqueGlobalNames, "COLUMN_UUID", "UUID") + assert getattr(UniqueGlobalNames, "COLUMN_INTEGER", "Integer") + assert getattr(UniqueGlobalNames, "COLUMN_BIGINT", "BigInt") + assert getattr(UniqueGlobalNames, "COLUMN_SMALLINT", "SmallInt") + assert getattr(UniqueGlobalNames, "COLUMN_SERIAL", "Serial") + assert getattr(UniqueGlobalNames, "COLUMN_BIGSERIAL", "BigSerial") + assert getattr(UniqueGlobalNames, "COLUMN_PRIMARYKEY", "PrimaryKey") + assert getattr(UniqueGlobalNames, "COLUMN_TIMESTAMP", "Timestamp") + assert getattr(UniqueGlobalNames, "COLUMN_TIMESTAMPZ", "Timestampz") + assert getattr(UniqueGlobalNames, "COLUMN_DATE", "Date") + assert getattr(UniqueGlobalNames, "COLUMN_TIME", "Time") + assert getattr(UniqueGlobalNames, "COLUMN_INTERVAL", "Interval") + assert getattr(UniqueGlobalNames, "COLUMN_BOOLEAN", "Boolean") + assert getattr(UniqueGlobalNames, "COLUMN_NUMERIC", "Numeric") + assert getattr(UniqueGlobalNames, "COLUMN_DECIMAL", "Decimal") + assert getattr(UniqueGlobalNames, "COLUMN_FLOAT", "Float") + assert getattr( + UniqueGlobalNames, "COLUMN_DOUBLEPERCISION", "DoublePrecision" + ) + assert getattr(UniqueGlobalNames, "COLUMN_FOREIGNKEY", "ForeignKey") + assert getattr(UniqueGlobalNames, "COLUMN_JSON", "JSON") + assert getattr(UniqueGlobalNames, "COLUMN_BYTEA", "Bytea") + assert getattr(UniqueGlobalNames, "COLUMN_BLOB", "Blob") + assert getattr(UniqueGlobalNames, "COLUMN_ARRAY", "Array") + + def test_warn_if_is_conflicting_name(self): + with pytest.warns(None) as recorded_warnings: + UniqueGlobalNames.warn_if_is_conflicting_name( + "SuperMassiveBlackHole" + ) + + if len(recorded_warnings) != 0: + pytest.fail("Unexpected warning!") + + with pytest.warns( + UniqueGlobalNameConflictWarning + ) as recorded_warnings: + UniqueGlobalNames.warn_if_is_conflicting_name("Varchar") + + if len(recorded_warnings) != 1: + pytest.fail("Expected 1 warning!") + + def test_is_conflicting_name(self): + assert ( + UniqueGlobalNames.is_conflicting_name("SuperMassiveBlackHole") + is False + ) + assert UniqueGlobalNames.is_conflicting_name("Varchar") is True + + def test_warn_if_are_conflicting_objects(self): + class ConflictingCls1(CanConflictWithGlobalNames): + def warn_if_is_conflicting_with_global_name(self): + pass + + class ConflictingCls2(CanConflictWithGlobalNames): + def warn_if_is_conflicting_with_global_name(self): + pass + + class ConflictingCls3(CanConflictWithGlobalNames): + def warn_if_is_conflicting_with_global_name(self): + warnings.warn("test", UniqueGlobalNameConflictWarning) + + with pytest.warns(None) as recorded_warnings: + UniqueGlobalNames.warn_if_are_conflicting_objects( + [ConflictingCls1(), ConflictingCls2()] + ) + + if len(recorded_warnings) != 0: + pytest.fail("Unexpected warning!") + + with pytest.warns( + UniqueGlobalNameConflictWarning + ) as recorded_warnings: + UniqueGlobalNames.warn_if_are_conflicting_objects( + [ConflictingCls2(), ConflictingCls3()] + ) + + if len(recorded_warnings) != 1: + pytest.fail("Expected 1 warning!") + + +class TestImport: + def test_with_module_only(self): + assert repr(Import(module="a.b.c")) == "import a.b.c" + + def test_with_module_and_target(self): + assert repr(Import(module="a.b", target="c")) == "from a.b import c" + + def test_warn_if_is_conflicting_with_global_name_with_module_only(self): + with pytest.warns(None) as recorded_warnings: + Import(module="a.b.c").warn_if_is_conflicting_with_global_name() + + if len(recorded_warnings) != 0: + pytest.fail("Unexpected warning!") + + with pytest.warns( + UniqueGlobalNameConflictWarning + ) as recorded_warnings: + Import(module="Varchar").warn_if_is_conflicting_with_global_name() + + if len(recorded_warnings) != 1: + pytest.fail("Expected 1 warning!") + + with pytest.warns(None) as recorded_warnings: + Import( + module="Varchar", expect_conflict_with_global_name="Varchar" + ).warn_if_is_conflicting_with_global_name() + + if len(recorded_warnings) != 0: + pytest.fail("Unexpected warning!") + + def test_warn_if_is_conflicting_with_global_name_with_module_and_target( + self, + ): + with pytest.warns(None) as recorded_warnings: + Import( + module="a.b", target="c" + ).warn_if_is_conflicting_with_global_name() + + if len(recorded_warnings) != 0: + pytest.fail("Unexpected warning!") + + with pytest.warns( + UniqueGlobalNameConflictWarning + ) as recorded_warnings: + Import( + module="a.b", target="Varchar" + ).warn_if_is_conflicting_with_global_name() + + if len(recorded_warnings) != 1: + pytest.fail("Expected 1 warning!") + + with pytest.warns(None) as recorded_warnings: + Import( + module="a.b", + target="Varchar", + expect_conflict_with_global_name="Varchar", + ).warn_if_is_conflicting_with_global_name() + + if len(recorded_warnings) != 0: + pytest.fail("Unexpected warning!") + + def example_function(): pass @@ -34,9 +203,21 @@ def test_timestamp(self): ) def test_uuid(self): + serialised = serialise_params(params={"default": uuid.UUID(int=4)}) + assert ( + repr(serialised.params["default"]) + == 'uuid.UUID("00000000-0000-0000-0000-000000000004")' + ) + serialised = serialise_params(params={"default": UUID4()}) self.assertTrue(serialised.params["default"].__repr__() == "UUID4()") + def test_decimal(self): + serialised = serialise_params( + params={"default": decimal.Decimal("1.2")} + ) + assert repr(serialised.params["default"]) == 'decimal.Decimal("1.2")' + def test_lazy_table_reference(self): # These are equivalent: references_list = [ From e0f04a40e868e9fa3c4f6bb9ebb1128f74180b07 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 24 Oct 2021 21:04:48 +0100 Subject: [PATCH 134/727] bumped version --- CHANGES.rst | 74 +++++++++++++++++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index b186ad182..44e2311ce 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,80 @@ Changes ======= +0.58.0 +------ + +Improved Pydantic docs +~~~~~~~~~~~~~~~~~~~~~~ + +The Pydantic docs used to be in the Piccolo API repo, but have been moved over +to this repo. We took this opportunity to improve them significantly with +additional examples. Courtesy @sinisaos. + +Internal code refactoring +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Some of the code has been optimised and cleaned up. Courtesy @yezz123. + +Schema generation for recursive foreign keys +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When using ``piccolo schema generate``, it would get stuck in a loop if a +table had a foreign key column which referenced itself. Thanks to @knguyen5 +for reporting this issue, and @wmshort for implementing the fix. The output +will now look like: + +.. code-block:: python + + class Employee(Table): + name = Varchar() + manager = ForeignKey("self") + +Fixing a bug with Alter.add_column +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When using the ``Alter.add_column`` API directly (not via migrations), it would +fail with foreign key columns. For example: + +.. code-block:: python + + SomeTable.alter().add_column( + name="my_fk_column", + column=ForeignKey(SomeOtherTable) + ).run_sync() + +This has now been fixed. Thanks to @wmshort for discovering this issue. + +create_pydantic_model improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Additional fields can now be added to the Pydantic schema. This is useful +when using Pydantic's JSON schema functionality: + +.. code-block:: python + + my_model = create_pydantic_model(Band, my_extra_field="Hello") + >>> my_model.schema() + {..., "my_extra_field": "Hello"} + +This feature was added to support new features in Piccolo Admin. + +Fixing a bug with import clashes in migrations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In certain situations it was possible to create a migration file with clashing +imports. For example: + +.. code-block:: python + + from uuid import UUID + from piccolo.columns.column_types import UUID + +Piccolo now tries to detect these clashes, and prevent them. If they can't be +prevented automatically, a warning is shown to the user. Courtesy @0scarB. + +------------------------------------------------------------------------------- + 0.57.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index eb9fe61d2..a4b87cf28 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.57.0" +__VERSION__ = "0.58.0" From fc2e14a1f866ff48f77a37b002c2b336591ea155 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 29 Oct 2021 19:36:38 +0100 Subject: [PATCH 135/727] improve FastAPI ASGI template (#316) --- .../templates/app/_fastapi_app.py.jinja | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/piccolo/apps/asgi/commands/templates/app/_fastapi_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_fastapi_app.py.jinja index 53b8dc20e..0015ebf79 100644 --- a/piccolo/apps/asgi/commands/templates/app/_fastapi_app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/_fastapi_app.py.jinja @@ -46,24 +46,24 @@ async def tasks(): @app.post('/tasks/', response_model=TaskModelOut) -async def create_task(task: TaskModelIn): - task = Task(**task.__dict__) +async def create_task(task_model: TaskModelIn): + task = Task(**task_model.__dict__) await task.save().run() - return TaskModelOut(**task.__dict__) + return task.to_dict() @app.put('/tasks/{task_id}/', response_model=TaskModelOut) -async def update_task(task_id: int, task: TaskModelIn): - _task = await Task.objects().where(Task.id == task_id).first().run() - if not _task: +async def update_task(task_id: int, task_model: TaskModelIn): + task = await Task.objects().where(Task.id == task_id).first().run() + if not task: return JSONResponse({}, status_code=404) - for key, value in task.__dict__.items(): - setattr(_task, key, value) + for key, value in task_model.__dict__.items(): + setattr(task, key, value) - await _task.save().run() + await task.save().run() - return TaskModelOut(**_task.__dict__) + return task.to_dict() @app.delete('/tasks/{task_id}/') From 40a45f38e902f96d0cdf952b0eeda36e1b2ceedb Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 3 Nov 2021 06:34:11 +0000 Subject: [PATCH 136/727] minor documentation tweaks (#319) * minor documentation tweaks * more tweaks * more white space between sections * more tweaks --- docs/src/piccolo/asgi/index.rst | 4 + docs/src/piccolo/contributing/index.rst | 6 + docs/src/piccolo/ecosystem/index.rst | 4 + docs/src/piccolo/engines/index.rst | 2 +- docs/src/piccolo/features/security.rst | 2 +- .../piccolo/features/supported_databases.rst | 2 + docs/src/piccolo/features/syntax.rst | 2 + .../getting_started/installing_piccolo.rst | 4 +- .../piccolo/getting_started/playground.rst | 10 +- docs/src/piccolo/migrations/create.rst | 4 +- .../projects_and_apps/piccolo_apps.rst | 7 +- .../projects_and_apps/piccolo_projects.rst | 2 +- docs/src/piccolo/query_clauses/first.rst | 2 +- docs/src/piccolo/query_clauses/freeze.rst | 3 + docs/src/piccolo/query_clauses/group_by.rst | 14 ++- docs/src/piccolo/query_clauses/order_by.rst | 17 ++- docs/src/piccolo/query_clauses/where.rst | 113 ++++++++---------- docs/src/piccolo/query_types/alter.rst | 7 ++ docs/src/piccolo/query_types/delete.rst | 4 + docs/src/piccolo/query_types/exists.rst | 2 + docs/src/piccolo/query_types/index.rst | 4 +- docs/src/piccolo/query_types/insert.rst | 2 + docs/src/piccolo/query_types/select.rst | 2 +- docs/src/piccolo/query_types/transactions.rst | 4 + docs/src/piccolo/query_types/update.rst | 2 + docs/src/piccolo/testing/index.rst | 2 +- 26 files changed, 135 insertions(+), 92 deletions(-) diff --git a/docs/src/piccolo/asgi/index.rst b/docs/src/piccolo/asgi/index.rst index da560d62d..4c00e340e 100644 --- a/docs/src/piccolo/asgi/index.rst +++ b/docs/src/piccolo/asgi/index.rst @@ -15,6 +15,8 @@ By using the ``piccolo asgi new`` command, Piccolo will scaffold an ASGI web app for you, which includes everything you need to get started. The command will ask for your preferences on which libraries to use. +------------------------------------------------------------------------------- + Routing frameworks ****************** @@ -32,6 +34,8 @@ Which to use? All are great choices. FastAPI is built on top of Starlette, so they're very similar. FastAPI is useful if you want to document a REST API. +------------------------------------------------------------------------------- + Web servers ************ diff --git a/docs/src/piccolo/contributing/index.rst b/docs/src/piccolo/contributing/index.rst index ddf897cfa..45a2ca351 100644 --- a/docs/src/piccolo/contributing/index.rst +++ b/docs/src/piccolo/contributing/index.rst @@ -6,6 +6,8 @@ Contributing If you want to dig deeper into the Piccolo internals, follow these instructions. +------------------------------------------------------------------------------- + Get the tests running --------------------- @@ -20,6 +22,8 @@ Get the tests running * Run the test suite with Postgres: ``./scripts/test-postgres.sh`` * Run the test suite with Sqlite: ``./scripts/test-sqlite.sh`` +------------------------------------------------------------------------------- + Contributing to the docs ------------------------ @@ -31,6 +35,8 @@ The docs are written using Sphinx. To get them running locally: * Serve the docs: ``python serve_docs.py`` * The docs will auto rebuild as you make changes. +------------------------------------------------------------------------------- + Code style ---------- diff --git a/docs/src/piccolo/ecosystem/index.rst b/docs/src/piccolo/ecosystem/index.rst index 2d90bb1fc..6b414dc66 100644 --- a/docs/src/piccolo/ecosystem/index.rst +++ b/docs/src/piccolo/ecosystem/index.rst @@ -17,6 +17,8 @@ Examples include: `See the docs `_ for more information. +------------------------------------------------------------------------------- + Piccolo Admin ------------- @@ -28,6 +30,8 @@ project on `Github `_. It's a modern UI built with Vue JS, which supports powerful data filtering, and CSV exports. It's the crown jewel in the Piccolo ecosystem! +------------------------------------------------------------------------------- + Piccolo Examples ---------------- diff --git a/docs/src/piccolo/engines/index.rst b/docs/src/piccolo/engines/index.rst index 6807596ce..1690dc6c6 100644 --- a/docs/src/piccolo/engines/index.rst +++ b/docs/src/piccolo/engines/index.rst @@ -58,7 +58,7 @@ Here's an example ``piccolo_conf.py`` file: DB = SQLiteEngine(path='my_db.sqlite') -.. hint:: A good place for your piccolo_conf file is at the root of your +.. hint:: A good place for your ``piccolo_conf.py`` file is at the root of your project, where the Python interpreter will be launched. .. _PICCOLO_CONF: diff --git a/docs/src/piccolo/features/security.rst b/docs/src/piccolo/features/security.rst index dddd24f2f..bef04cb6d 100644 --- a/docs/src/piccolo/features/security.rst +++ b/docs/src/piccolo/features/security.rst @@ -6,7 +6,7 @@ Security SQL Injection protection ------------------------ -If you look under the hood, Piccolo uses a custom class called `QueryString` +If you look under the hood, Piccolo uses a custom class called ``QueryString`` for composing queries. It keeps query parameters separate from the query string, so we can pass parameterised queries to the engine. This helps prevent SQL Injection attacks. diff --git a/docs/src/piccolo/features/supported_databases.rst b/docs/src/piccolo/features/supported_databases.rst index fa022d3f2..2b2ef74a4 100644 --- a/docs/src/piccolo/features/supported_databases.rst +++ b/docs/src/piccolo/features/supported_databases.rst @@ -6,6 +6,8 @@ Postgres Postgres is the primary focus for Piccolo, and is what we expect most people will be using in production. +------------------------------------------------------------------------------- + SQLite ------ SQLite support is not as complete as Postgres, but it is available - mostly diff --git a/docs/src/piccolo/features/syntax.rst b/docs/src/piccolo/features/syntax.rst index ee0fb3ba0..2bcedf89a 100644 --- a/docs/src/piccolo/features/syntax.rst +++ b/docs/src/piccolo/features/syntax.rst @@ -12,6 +12,8 @@ For example: * In other ORMs, you define models - in Piccolo you define tables. * Rather than using a filter method, you use a `where` method like in SQL. +------------------------------------------------------------------------------- + Get the SQL at any time ----------------------- diff --git a/docs/src/piccolo/getting_started/installing_piccolo.rst b/docs/src/piccolo/getting_started/installing_piccolo.rst index c6e2fce23..c4460bc7a 100644 --- a/docs/src/piccolo/getting_started/installing_piccolo.rst +++ b/docs/src/piccolo/getting_started/installing_piccolo.rst @@ -6,6 +6,8 @@ Python You need `Python 3.7 `_ or above installed on your system. +------------------------------------------------------------------------------- + Pip --- @@ -14,7 +16,7 @@ Now install piccolo, ideally inside a `virtualenv >> Band.objects().first().run_sync() -If no match is found, then `None` is returned instead. +If no match is found, then ``None`` is returned instead. diff --git a/docs/src/piccolo/query_clauses/freeze.rst b/docs/src/piccolo/query_clauses/freeze.rst index 8c61b13ea..54f7c28e5 100644 --- a/docs/src/piccolo/query_clauses/freeze.rst +++ b/docs/src/piccolo/query_clauses/freeze.rst @@ -5,6 +5,9 @@ freeze You can use the ``freeze`` clause with any query type. +Source +------ + .. currentmodule:: piccolo.query.base .. automethod:: Query.freeze diff --git a/docs/src/piccolo/query_clauses/group_by.rst b/docs/src/piccolo/query_clauses/group_by.rst index a348120ce..7cba6f9db 100644 --- a/docs/src/piccolo/query_clauses/group_by.rst +++ b/docs/src/piccolo/query_clauses/group_by.rst @@ -10,6 +10,8 @@ You can use ``group_by`` clauses with the following queries: It is used in combination with aggregate functions - ``Count`` is currently supported. +------------------------------------------------------------------------------- + Count ----- @@ -19,12 +21,11 @@ In the following query, we get a count of the number of bands per manager: >>> from piccolo.query.methods.select import Count - >>> b = Band - >>> b.select( - >>> b.manager.name, - >>> Count(b.manager) + >>> Band.select( + >>> Band.manager.name, + >>> Count(Band.manager) >>> ).group_by( - >>> b.manager + >>> Band.manager >>> ).run_sync() [ @@ -32,6 +33,9 @@ In the following query, we get a count of the number of bands per manager: {"manager.name": "Guido", "count": 1} ] +Source +~~~~~~ + .. currentmodule:: piccolo.query.methods.select .. autoclass:: Count diff --git a/docs/src/piccolo/query_clauses/order_by.rst b/docs/src/piccolo/query_clauses/order_by.rst index 17a88b7c3..2eed639ed 100644 --- a/docs/src/piccolo/query_clauses/order_by.rst +++ b/docs/src/piccolo/query_clauses/order_by.rst @@ -12,18 +12,16 @@ To order the results by a certain column (ascending): .. code-block:: python - b = Band - b.select().order_by( - b.name + Band.select().order_by( + Band.name ).run_sync() To order by descending: .. code-block:: python - b = Band - b.select().order_by( - b.name, + Band.select().order_by( + Band.name, ascending=False ).run_sync() @@ -31,8 +29,7 @@ You can order by multiple columns, and even use joins: .. code-block:: python - b = Band - b.select().order_by( - b.name, - b.manager.name + Band.select().order_by( + Band.name, + Band.manager.name ).run_sync() diff --git a/docs/src/piccolo/query_clauses/where.rst b/docs/src/piccolo/query_clauses/where.rst index 85f583f96..0c8f6d7fc 100644 --- a/docs/src/piccolo/query_clauses/where.rst +++ b/docs/src/piccolo/query_clauses/where.rst @@ -13,21 +13,21 @@ You can use ``where`` clauses with the following queries: It allows powerful filtering of your data. +------------------------------------------------------------------------------- + Equal / Not Equal ----------------- .. code-block:: python - b = Band - b.select().where( - b.name == 'Pythonistas' + Band.select().where( + Band.name == 'Pythonistas' ).run_sync() .. code-block:: python - b = Band - b.select().where( - b.name != 'Rustaceans' + Band.select().where( + Band.name != 'Rustaceans' ).run_sync() .. hint:: With ``Boolean`` columns, some linters will complain if you write @@ -45,9 +45,8 @@ You can use the ``<, >, <=, >=`` operators, which work as you expect. .. code-block:: python - b = Band - b.select().where( - b.popularity >= 100 + Band.select().where( + Band.popularity >= 100 ).run_sync() ------------------------------------------------------------------------------- @@ -59,21 +58,20 @@ The percentage operator is required to designate where the match should occur. .. code-block:: python - b = Band - b.select().where( - b.name.like('Py%') # Matches the start of the string + Band.select().where( + Band.name.like('Py%') # Matches the start of the string ).run_sync() - b.select().where( - b.name.like('%istas') # Matches the end of the string + Band.select().where( + Band.name.like('%istas') # Matches the end of the string ).run_sync() - b.select().where( - b.name.like('%is%') # Matches anywhere in string + Band.select().where( + Band.name.like('%is%') # Matches anywhere in string ).run_sync() - b.select().where( - b.name.like('Pythonistas') # Matches the entire string + Band.select().where( + Band.name.like('Pythonistas') # Matches the entire string ).run_sync() ``ilike`` is identical, except it's Postgres specific and case insensitive. @@ -87,9 +85,8 @@ Usage is the same as ``like`` excepts it excludes matching rows. .. code-block:: python - b = Band - b.select().where( - b.name.not_like('Py%') + Band.select().where( + Band.name.not_like('Py%') ).run_sync() ------------------------------------------------------------------------------- @@ -99,16 +96,14 @@ is_in / not_in .. code-block:: python - b = Band - b.select().where( - b.name.is_in(['Pythonistas']) + Band.select().where( + Band.name.is_in(['Pythonistas']) ).run_sync() .. code-block:: python - b = Band - b.select().where( - b.name.not_in(['Rustaceans']) + Band.select().where( + Band.name.not_in(['Rustaceans']) ).run_sync() ------------------------------------------------------------------------------- @@ -121,32 +116,28 @@ with None: .. code-block:: python - b = Band - # Fetch all bands with a manager - b.select().where( - b.manager != None + Band.select().where( + Band.manager != None ).run_sync() # Fetch all bands without a manager - b.select().where( - b.manager == None + Band.select().where( + Band.manager == None ).run_sync() To avoid the linter errors, you can use `is_null` and `is_not_null` instead. .. code-block:: python - b = Band - # Fetch all bands with a manager - b.select().where( - b.manager.is_not_null() + Band.select().where( + Band.manager.is_not_null() ).run_sync() # Fetch all bands without a manager - b.select().where( - b.manager.is_null() + Band.select().where( + Band.manager.is_null() ).run_sync() ------------------------------------------------------------------------------- @@ -158,13 +149,12 @@ You can make complex ``where`` queries using ``&`` for AND, and ``|`` for OR. .. code-block:: python - b = Band - b.select().where( - (b.popularity >= 100) & (b.popularity < 1000) + Band.select().where( + (Band.popularity >= 100) & (Band.popularity < 1000) ).run_sync() - b.select().where( - (b.popularity >= 100) | (b.name == 'Pythonistas') + Band.select().where( + (Band.popularity >= 100) | (Band.name == 'Pythonistas') ).run_sync() You can make really complex ``where`` clauses if you so choose - just be @@ -178,32 +168,28 @@ Using multiple ``where`` clauses is equivalent to an AND. .. code-block:: python - b = Band - # These are equivalent: - b.select().where( - (b.popularity >= 100) & (b.popularity < 1000) + Band.select().where( + (Band.popularity >= 100) & (Band.popularity < 1000) ).run_sync() - b.select().where( - b.popularity >= 100 + Band.select().where( + Band.popularity >= 100 ).where( - b.popularity < 1000 + Band.popularity < 1000 ).run_sync() Also, multiple arguments inside ``where`` clause is equivalent to an AND. .. code-block:: python - b = Band - # These are equivalent: - b.select().where( - (b.popularity >= 100) & (b.popularity < 1000) + Band.select().where( + (Band.popularity >= 100) & (Band.popularity < 1000) ).run_sync() - b.select().where( - b.popularity >= 100, b.popularity < 1000 + Band.select().where( + Band.popularity >= 100, Band.popularity < 1000 ).run_sync() Using And / Or directly @@ -216,12 +202,10 @@ Rather than using the ``|`` and ``&`` characters, you can use the ``And`` and from piccolo.columns.combination import And, Or - b = Band - - b.select().where( + Band.select().where( Or( - And(b.popularity >= 100, b.popularity < 1000), - b.name == 'Pythonistas' + And(Band.popularity >= 100, Band.popularity < 1000), + Band.name == 'Pythonistas' ) ).run_sync() @@ -259,7 +243,6 @@ untrusted source, otherwise it could lead to a SQL injection attack. from piccolo.columns.combination import WhereRaw - b = Band - b.select().where( - WhereRaw("name = 'Pythonistas'") | (b.popularity > 1000) + Band.select().where( + WhereRaw("name = 'Pythonistas'") | (Band.popularity > 1000) ).run_sync() diff --git a/docs/src/piccolo/query_types/alter.rst b/docs/src/piccolo/query_types/alter.rst index 88d485561..db1ff378a 100644 --- a/docs/src/piccolo/query_types/alter.rst +++ b/docs/src/piccolo/query_types/alter.rst @@ -7,6 +7,7 @@ This is used to modify an existing table. .. hint:: You can use migrations instead of manually altering the schema - see :ref:`Migrations`. +------------------------------------------------------------------------------- add_column ---------- @@ -17,6 +18,7 @@ Used to add a column to an existing table. Band.alter().add_column('members', Integer()).run_sync() +------------------------------------------------------------------------------- drop_column ----------- @@ -27,6 +29,7 @@ Used to drop an existing column. Band.alter().drop_column('popularity').run_sync() +------------------------------------------------------------------------------- drop_table ---------- @@ -46,6 +49,8 @@ instead. It will drop them in the correct order. drop_tables(Band, Manager) +------------------------------------------------------------------------------- + rename_column ------------- @@ -55,6 +60,7 @@ Used to rename an existing column. Band.alter().rename_column(Band.popularity, 'rating').run_sync() +------------------------------------------------------------------------------- set_null -------- @@ -69,6 +75,7 @@ Set whether a column is nullable or not. # To stop a row being nullable: Band.alter().set_null(Band.name, False).run_sync() +------------------------------------------------------------------------------- set_unique ---------- diff --git a/docs/src/piccolo/query_types/delete.rst b/docs/src/piccolo/query_types/delete.rst index 80cb42fb4..9ca85bc9d 100644 --- a/docs/src/piccolo/query_types/delete.rst +++ b/docs/src/piccolo/query_types/delete.rst @@ -10,6 +10,8 @@ This deletes any matching rows from the table. >>> Band.delete().where(Band.name == 'Rustaceans').run_sync() [] +------------------------------------------------------------------------------- + force ----- @@ -26,6 +28,8 @@ the data from a table. >>> Band.delete(force=True).run_sync() [] +------------------------------------------------------------------------------- + Query clauses ------------- diff --git a/docs/src/piccolo/query_types/exists.rst b/docs/src/piccolo/query_types/exists.rst index 8ef5ab9f2..a9325f253 100644 --- a/docs/src/piccolo/query_types/exists.rst +++ b/docs/src/piccolo/query_types/exists.rst @@ -10,6 +10,8 @@ This checks whether any rows exist which match the criteria. >>> Band.exists().where(Band.name == 'Pythonistas').run_sync() True +------------------------------------------------------------------------------- + Query clauses ------------- diff --git a/docs/src/piccolo/query_types/index.rst b/docs/src/piccolo/query_types/index.rst index ca650dec0..a07bf63e6 100644 --- a/docs/src/piccolo/query_types/index.rst +++ b/docs/src/piccolo/query_types/index.rst @@ -12,7 +12,7 @@ typical ORM. ./select ./objects - ./alter + ./alter ./create_table ./delete ./exists @@ -21,6 +21,8 @@ typical ORM. ./update ./transactions +------------------------------------------------------------------------------- + Comparisons ----------- diff --git a/docs/src/piccolo/query_types/insert.rst b/docs/src/piccolo/query_types/insert.rst index 8af1bf69c..9775f95ba 100644 --- a/docs/src/piccolo/query_types/insert.rst +++ b/docs/src/piccolo/query_types/insert.rst @@ -19,6 +19,8 @@ We can insert multiple rows in one go: Band(name="Gophers") ).run_sync() +------------------------------------------------------------------------------- + add --- diff --git a/docs/src/piccolo/query_types/select.rst b/docs/src/piccolo/query_types/select.rst index 6989bb990..9dc612937 100644 --- a/docs/src/piccolo/query_types/select.rst +++ b/docs/src/piccolo/query_types/select.rst @@ -3,7 +3,7 @@ Select ====== -.. hint:: Follow along by installing Piccolo and running `piccolo playground run` - see :ref:`Playground` +.. hint:: Follow along by installing Piccolo and running ``piccolo playground run`` - see :ref:`Playground`. To get all rows: diff --git a/docs/src/piccolo/query_types/transactions.rst b/docs/src/piccolo/query_types/transactions.rst index 919cebe30..ca0656505 100644 --- a/docs/src/piccolo/query_types/transactions.rst +++ b/docs/src/piccolo/query_types/transactions.rst @@ -8,6 +8,8 @@ Transactions allow multiple queries to be committed only once successful. This is useful for things like migrations, where you can't have it fail in an inbetween state. +------------------------------------------------------------------------------- + Atomic ------ @@ -24,6 +26,8 @@ transaction before running it. # You're also able to run this synchronously: transaction.run_sync() +------------------------------------------------------------------------------- + Transaction ----------- diff --git a/docs/src/piccolo/query_types/update.rst b/docs/src/piccolo/query_types/update.rst index 0d24e5db4..1118eb874 100644 --- a/docs/src/piccolo/query_types/update.rst +++ b/docs/src/piccolo/query_types/update.rst @@ -17,6 +17,7 @@ This is used to update any rows in the table which match the criteria. As well as replacing values with new ones, you can also modify existing values, for instance by adding to an integer. +------------------------------------------------------------------------------- Modifying values ---------------- @@ -78,6 +79,7 @@ You can concatenate values: You can currently only combine two values together at a time. +------------------------------------------------------------------------------- Query clauses ------------- diff --git a/docs/src/piccolo/testing/index.rst b/docs/src/piccolo/testing/index.rst index 41e7de824..c7d2240dd 100644 --- a/docs/src/piccolo/testing/index.rst +++ b/docs/src/piccolo/testing/index.rst @@ -36,7 +36,7 @@ You can build a random ``Band`` which will also build and save a random ``Manage band = await ModelBuilder.build(Band) # Band instance with random values persisted -.. note:: ``ModelBuilder.build(Band)`` persists record into the database by default. +.. note:: ``ModelBuilder.build(Band)`` persists the record into the database by default. You can also run it synchronously if you prefer: From 3fa34050b5044abf227f073ce3ba7bf4d161beea Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 3 Nov 2021 08:09:12 +0000 Subject: [PATCH 137/727] add readthedocs-requirements.txt (#322) --- docs/src/piccolo/contributing/index.rst | 2 +- requirements/README.md | 10 ++++++---- {docs => requirements}/doc-requirements.txt | 0 requirements/readthedocs-requirements.txt | 2 ++ 4 files changed, 9 insertions(+), 5 deletions(-) rename {docs => requirements}/doc-requirements.txt (100%) create mode 100644 requirements/readthedocs-requirements.txt diff --git a/docs/src/piccolo/contributing/index.rst b/docs/src/piccolo/contributing/index.rst index 45a2ca351..f7efbd4a7 100644 --- a/docs/src/piccolo/contributing/index.rst +++ b/docs/src/piccolo/contributing/index.rst @@ -29,8 +29,8 @@ Contributing to the docs The docs are written using Sphinx. To get them running locally: + * Install the requirements: ``pip install -r requirements/doc-requirements.txt`` * ``cd docs`` - * Install the requirements: ``pip install -r doc-requirements.txt`` * Do an initial build of the docs: ``make html`` * Serve the docs: ``python serve_docs.py`` * The docs will auto rebuild as you make changes. diff --git a/requirements/README.md b/requirements/README.md index 07faa574f..235d5e4c9 100644 --- a/requirements/README.md +++ b/requirements/README.md @@ -1,6 +1,8 @@ # Requirement files -* `extras` - Optional dependencies of `Piccolo`. -* `dev-requirements.txt` - Requirements needed to develop `Piccolo`. -* `requirements.txt` - Default requirements of `Piccolo`. -* `test-requirements.txt` - Requirements needed to run `Piccolo` tests. +- `extras` - optional dependencies of `Piccolo`. +- `dev-requirements.txt` - needed to develop `Piccolo`. +- `requirements.txt` - default requirements of `Piccolo`. +- `test-requirements.txt` - needed to run `Piccolo` tests. +- `doc-requirements.txt` - needed to run the `Piccolo` docs +- `readthedocs-requirements.txt` - just used by ReadTheDocs. diff --git a/docs/doc-requirements.txt b/requirements/doc-requirements.txt similarity index 100% rename from docs/doc-requirements.txt rename to requirements/doc-requirements.txt diff --git a/requirements/readthedocs-requirements.txt b/requirements/readthedocs-requirements.txt new file mode 100644 index 000000000..8db118e9b --- /dev/null +++ b/requirements/readthedocs-requirements.txt @@ -0,0 +1,2 @@ +-r requirements.txt +-r doc-requirements.txt From 89d0ee75582cc427c6bd143e8bb2ac99945f1b96 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 9 Nov 2021 09:36:06 +0000 Subject: [PATCH 138/727] fix engine slots (#328) --- piccolo/engine/base.py | 3 +++ piccolo/engine/postgres.py | 8 +++++++- piccolo/engine/sqlite.py | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/piccolo/engine/base.py b/piccolo/engine/base.py index e60594dcc..dbd297bc9 100644 --- a/piccolo/engine/base.py +++ b/piccolo/engine/base.py @@ -20,6 +20,9 @@ class Batch: class Engine(metaclass=ABCMeta): + + __slots__ = () + def __init__(self): run_sync(self.check_version()) run_sync(self.prep_database()) diff --git a/piccolo/engine/postgres.py b/piccolo/engine/postgres.py index 9ba9302ee..63e3e988d 100644 --- a/piccolo/engine/postgres.py +++ b/piccolo/engine/postgres.py @@ -228,7 +228,13 @@ class PostgresEngine(Engine): """ # noqa: E501 - __slots__ = ("config", "extensions", "pool", "transaction_connection") + __slots__ = ( + "config", + "extensions", + "log_queries", + "pool", + "transaction_connection", + ) engine_type = "postgres" min_version_number = 9.6 diff --git a/piccolo/engine/sqlite.py b/piccolo/engine/sqlite.py index cca29600e..aece06b6e 100644 --- a/piccolo/engine/sqlite.py +++ b/piccolo/engine/sqlite.py @@ -343,7 +343,7 @@ class SQLiteEngine(Engine): """ - __slots__ = ("connection_kwargs",) + __slots__ = ("connection_kwargs", "transaction_connection") engine_type = "sqlite" min_version_number = 3.25 From 4d35b71bd30ee5d49b3ecfe11009ad0b48d4a6de Mon Sep 17 00:00:00 2001 From: Taylor Beever Date: Tue, 9 Nov 2021 03:23:09 -0700 Subject: [PATCH 139/727] Update docs with piccolo conf envvar (#327) * Add PICCOLO_CONF to docs. * Change example path * expanded upon piccolo_conf.py docs Co-authored-by: Daniel Townsend --- .../projects_and_apps/piccolo_projects.rst | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/docs/src/piccolo/projects_and_apps/piccolo_projects.rst b/docs/src/piccolo/projects_and_apps/piccolo_projects.rst index 888b7882a..5fb784386 100644 --- a/docs/src/piccolo/projects_and_apps/piccolo_projects.rst +++ b/docs/src/piccolo/projects_and_apps/piccolo_projects.rst @@ -10,7 +10,7 @@ A Piccolo project is a collection of apps. piccolo_conf.py --------------- -A project requires a ``piccolo_conf.py`` file. To create this file, use the following command: +A project requires a ``piccolo_conf.py`` file. To create this, use the following command: .. code-block:: bash @@ -18,9 +18,40 @@ A project requires a ``piccolo_conf.py`` file. To create this file, use the foll The file serves two important purposes: - * Contains your database settings + * Contains your database settings. * Is used for registering :ref:`PiccoloApps`. +Location +~~~~~~~~ + +By convention, the ``piccolo_conf.py`` file should be at the root of your project: + +.. code-block:: + + my_project/ + piccolo_conf.py + my_app/ + piccolo_app.py + +This means that when you use the ``piccolo`` CLI from the ``my_project`` +folder it can import ``piccolo_conf.py``. + +If you prefer to keep ``piccolo_conf.py`` in a different location, or to give +it a different name, you can do so using the ``PICCOLO_CONF`` environment +variable (see :ref:`PICCOLO_CONF`). For example: + +.. code-block:: + + my_project/ + conf/ + piccolo_conf_local.py + my_app/ + piccolo_app.py + +.. code-block:: bash + + export PICCOLO_CONF=conf.piccolo_conf_local + ------------------------------------------------------------------------------- Example From 7a613163f7a6337f25e5f4835934b69d78d24bba Mon Sep 17 00:00:00 2001 From: sinisaos Date: Tue, 9 Nov 2021 12:37:02 +0100 Subject: [PATCH 140/727] added include columns (#324) * added include columns * update requirements * fix formating * moved validation to top of create_pydantic_model * make logic more readable Co-authored-by: Daniel Townsend --- piccolo/utils/pydantic.py | 67 +++++++++++++++++++++++++------ requirements/dev-requirements.txt | 2 +- tests/utils/test_pydantic.py | 28 +++++++++++++ 3 files changed, 83 insertions(+), 14 deletions(-) diff --git a/piccolo/utils/pydantic.py b/piccolo/utils/pydantic.py index 8767ebafc..16c6d80ed 100644 --- a/piccolo/utils/pydantic.py +++ b/piccolo/utils/pydantic.py @@ -44,11 +44,33 @@ def pydantic_json_validator(cls, value): return value +def is_table_column(column: Column, table: t.Type[Table]) -> bool: + """ + Verify that the given ``Column`` belongs to the given ``Table``. + """ + return column._meta.table is table + + +def validate_columns( + columns: t.Tuple[Column, ...], table: t.Type[Table] +) -> bool: + """ + Verify that each column is a ``Column``` instance, and its parent is the + given ``Table``. + """ + return all( + isinstance(column, Column) + and is_table_column(column=column, table=table) + for column in columns + ) + + @lru_cache() def create_pydantic_model( table: t.Type[Table], nested: bool = False, exclude_columns: t.Tuple[Column, ...] = (), + include_columns: t.Tuple[Column, ...] = (), include_default_columns: bool = False, include_readable: bool = False, all_optional: bool = False, @@ -66,7 +88,10 @@ def create_pydantic_model( Whether ``ForeignKey`` columns are converted to nested Pydantic models. :param exclude_columns: A tuple of ``Column`` instances that should be excluded from the - Pydantic model. + Pydantic model. Only specify ``include_column`` or ``exclude_column``. + :param include_columns: + A tuple of ``Column`` instances that should be included in the + Pydantic model. Only specify ``include_column`` or ``exclude_column``. :param include_default_columns: Whether to include columns like ``id`` in the serialiser. You will typically include these columns in GET requests, but don't require @@ -98,26 +123,42 @@ def create_pydantic_model( A Pydantic model. """ + if exclude_columns and include_columns: + raise ValueError( + "`include_columns` and `exclude_columns` can't be used at the " + "same time." + ) + + if exclude_columns: + if not validate_columns(columns=exclude_columns, table=table): + raise ValueError( + f"`exclude_columns` are invalid: ({exclude_columns!r})" + ) + + if include_columns: + if not validate_columns(columns=include_columns, table=table): + raise ValueError( + f"`include_columns` are invalid: ({include_columns!r})" + ) + + ########################################################################### + columns: t.Dict[str, t.Any] = {} validators: t.Dict[str, classmethod] = {} piccolo_columns = ( - table._meta.columns - if include_default_columns - else table._meta.non_default_columns + include_columns + if include_columns + else tuple( + table._meta.columns + if include_default_columns + else table._meta.non_default_columns + ) ) - if not all( - isinstance(column, Column) - # Make sure that every column is tied to the current Table - and column._meta.table is table - for column in exclude_columns - ): - raise ValueError(f"Exclude columns ({exclude_columns!r}) are invalid.") - for column in piccolo_columns: # normal __contains__ checks __eq__ as well which returns ``Where`` # instance which always evaluates to ``True`` - if any(column is obj for obj in exclude_columns): + if exclude_columns and any(column is obj for obj in exclude_columns): continue column_name = column._meta.name diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt index c4a733841..d4b766826 100644 --- a/requirements/dev-requirements.txt +++ b/requirements/dev-requirements.txt @@ -1,5 +1,5 @@ black>=21.7b0 -ipdb==0.12.2 +ipdb==0.12.3 ipython==7.8.0 flake8==3.8.4 isort==5.9.2 diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index 2dd48ef6e..51e4348ca 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -281,6 +281,34 @@ class Computer2(Table): create_pydantic_model(Computer, exclude_columns=(Computer2.CPU,)) +class TestIncludeColumn(TestCase): + def test_include(self): + class Computer(Table): + CPU = Varchar() + GPU = Varchar() + + pydantic_model = create_pydantic_model( + Computer, + include_columns=(Computer.CPU,), + ) + + properties = pydantic_model.schema()["properties"] + self.assertIsInstance(properties.get("CPU"), dict) + self.assertIsNone(properties.get("GPU")) + + def test_include_exclude_error(self): + class Computer(Table): + CPU = Varchar() + GPU = Varchar() + + with self.assertRaises(ValueError): + create_pydantic_model( + Computer, + exclude_columns=(Computer.CPU,), + include_columns=(Computer.CPU,), + ) + + class TestNestedModel(TestCase): def test_nested_models(self): class Country(Table): From dc16af02a6423c708ce9e8c52eb1a9a058706a39 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 9 Nov 2021 11:44:42 +0000 Subject: [PATCH 141/727] update `create_pydantic_model` docs Didn't mention the `include_columns` option. --- docs/src/piccolo/serialization/index.rst | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/src/piccolo/serialization/index.rst b/docs/src/piccolo/serialization/index.rst index 91095ef1e..a55ce333c 100644 --- a/docs/src/piccolo/serialization/index.rst +++ b/docs/src/piccolo/serialization/index.rst @@ -54,8 +54,8 @@ We can then create model instances from data we fetch from the database: You have several options for configuring the model, as shown below. -exclude_columns -~~~~~~~~~~~~~~~ +include_columns / exclude_columns +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If we want to exclude the ``popularity`` column from the ``Band`` table: @@ -63,6 +63,12 @@ If we want to exclude the ``popularity`` column from the ``Band`` table: BandModel = create_pydantic_model(Band, exclude_columns=(Band.popularity,)) +Conversely, if you only wanted the ``popularity`` column: + +.. code-block:: python + + BandModel = create_pydantic_model(Band, include_columns=(Band.popularity,)) + nested ~~~~~~ @@ -148,8 +154,9 @@ By default the primary key column isn't included - you can add it using: Source ~~~~~~ -.. automodule:: piccolo.utils.pydantic - :members: +.. currentmodule:: piccolo.utils.pydantic + +.. autofunction:: create_pydantic_model .. hint:: A good place to see ``create_pydantic_model`` in action is `PiccoloCRUD `_, as it uses ``create_pydantic_model`` extensively to create Pydantic models From 4a8ac7aab6fd5dd179abadae5f792b94dd107e04 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 9 Nov 2021 11:55:58 +0000 Subject: [PATCH 142/727] fix mypy warning about sorting (#329) * fix mypy warning about sorting * fix lgtm warning about missing `le`, `gt` and `ge` magic methods --- piccolo/apps/migrations/auto/serialisation.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/piccolo/apps/migrations/auto/serialisation.py b/piccolo/apps/migrations/auto/serialisation.py index 513e7a7ec..2217d66cf 100644 --- a/piccolo/apps/migrations/auto/serialisation.py +++ b/piccolo/apps/migrations/auto/serialisation.py @@ -244,6 +244,21 @@ class Definition(CanConflictWithGlobalNames, abc.ABC): def __repr__(self): ... + ########################################################################### + # To allow sorting: + + def __lt__(self, value): + return self.__repr__() < value.__repr__() + + def __le__(self, value): + return self.__repr__() <= value.__repr__() + + def __gt__(self, value): + return self.__repr__() > value.__repr__() + + def __ge__(self, value): + return self.__repr__() >= value.__repr__() + @dataclass class SerialisedParams: From b38505c671fc7515646106a1b81a6a0f9251b6d4 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 9 Nov 2021 13:56:41 +0000 Subject: [PATCH 143/727] Update dev-requirements.txt (#330) * Update dev-requirements.txt * fix flake8 errors * ignore mypy missing imports * ignore type errors --- piccolo/utils/graphlib/__init__.py | 2 +- pyproject.toml | 9 +++++++++ requirements/dev-requirements.txt | 14 +++++++------- tests/apps/user/commands/test_create.py | 8 ++++---- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/piccolo/utils/graphlib/__init__.py b/piccolo/utils/graphlib/__init__.py index 77a9e5774..75b339dbd 100644 --- a/piccolo/utils/graphlib/__init__.py +++ b/piccolo/utils/graphlib/__init__.py @@ -2,4 +2,4 @@ from graphlib import CycleError, TopologicalSorter # type: ignore except ImportError: # For version < Python 3.9 - from ._graphlib import CycleError, TopologicalSorter + from ._graphlib import CycleError, TopologicalSorter # type: ignore diff --git a/pyproject.toml b/pyproject.toml index b489446a7..576acccc7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,3 +5,12 @@ target-version = ['py37', 'py38', 'py39'] [tool.isort] profile = "black" line_length = 79 + +[tool.mypy] +[[tool.mypy.overrides]] +module = [ + "orjson", + "jinja2", + "dateutil" +] +ignore_missing_imports = true diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt index d4b766826..c6f25bbd5 100644 --- a/requirements/dev-requirements.txt +++ b/requirements/dev-requirements.txt @@ -1,9 +1,9 @@ black>=21.7b0 -ipdb==0.12.3 -ipython==7.8.0 -flake8==3.8.4 -isort==5.9.2 -twine==3.1.1 -mypy==0.782 +ipdb==0.13.9 +ipython==7.29.0 +flake8==4.0.1 +isort==5.10.1 +twine==3.5.0 +mypy==0.910 pip-upgrader==1.4.15 -wheel==0.36.2 +wheel==0.37.0 diff --git a/tests/apps/user/commands/test_create.py b/tests/apps/user/commands/test_create.py index 6e2854821..5c85816fd 100644 --- a/tests/apps/user/commands/test_create.py +++ b/tests/apps/user/commands/test_create.py @@ -49,8 +49,8 @@ def test_create(self, *args, **kwargs): (BaseUser.admin == True) # noqa: E712 & (BaseUser.username == "bob123") & (BaseUser.email == "bob@test.com") - & (BaseUser.superuser == True) - & (BaseUser.active == True) + & (BaseUser.superuser.eq(True)) + & (BaseUser.active.eq(True)) ) .run_sync() ) @@ -72,8 +72,8 @@ def test_create_with_arguments(self, *args, **kwargs): (BaseUser.admin == True) # noqa: E712 & (BaseUser.username == "bob123") & (BaseUser.email == "bob@test.com") - & (BaseUser.superuser == True) - & (BaseUser.active == True) + & (BaseUser.superuser.eq(True)) + & (BaseUser.active.eq(True)) ) .run_sync() ) From 19dc3930facc1e5219546cf5759bae6f19578b77 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 9 Nov 2021 15:08:37 +0000 Subject: [PATCH 144/727] add conftest.py to ASGI template, which complains if using pytest directly (#333) * add conftest.py to template, which complains if using pytest directly * Create README.md.jinja --- .../commands/templates/app/README.md.jinja | 21 +++++++++++++++++++ .../commands/templates/app/conftest.py.jinja | 17 +++++++++++++++ .../templates/app/piccolo_conf_test.py.jinja | 12 +++++++++++ piccolo/apps/tester/commands/run.py | 7 ++++++- 4 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 piccolo/apps/asgi/commands/templates/app/README.md.jinja create mode 100644 piccolo/apps/asgi/commands/templates/app/conftest.py.jinja create mode 100644 piccolo/apps/asgi/commands/templates/app/piccolo_conf_test.py.jinja diff --git a/piccolo/apps/asgi/commands/templates/app/README.md.jinja b/piccolo/apps/asgi/commands/templates/app/README.md.jinja new file mode 100644 index 000000000..b8c4e06ff --- /dev/null +++ b/piccolo/apps/asgi/commands/templates/app/README.md.jinja @@ -0,0 +1,21 @@ +# {{ project_identifier }} + +## Setup + +### Install requirements + +```bash +pip install -r requirements.txt +``` + +### Getting started guide + +```bash +python main.py +``` + +### Running tests + +```bash +piccolo tester run +``` diff --git a/piccolo/apps/asgi/commands/templates/app/conftest.py.jinja b/piccolo/apps/asgi/commands/templates/app/conftest.py.jinja new file mode 100644 index 000000000..70e2d5584 --- /dev/null +++ b/piccolo/apps/asgi/commands/templates/app/conftest.py.jinja @@ -0,0 +1,17 @@ +import os +import sys + +from piccolo.utils.warnings import colored_warning + + +def pytest_configure(*args): + if os.environ.get("PICCOLO_TEST_RUNNER") != "True": + colored_warning( + "\n\n" + "We recommend running Piccolo tests using the " + "`piccolo tester run` command, which wraps Pytest, and makes " + "sure the test database is being used. " + "To stop this warning, modify conftest.py." + "\n\n" + ) + sys.exit(1) diff --git a/piccolo/apps/asgi/commands/templates/app/piccolo_conf_test.py.jinja b/piccolo/apps/asgi/commands/templates/app/piccolo_conf_test.py.jinja new file mode 100644 index 000000000..52eaf191c --- /dev/null +++ b/piccolo/apps/asgi/commands/templates/app/piccolo_conf_test.py.jinja @@ -0,0 +1,12 @@ +from piccolo_conf import * # noqa + + +DB = PostgresEngine( + config={ + "database": "{{ project_identifier }}_test", + "user": "postgres", + "password": "", + "host": "localhost", + "port": 5432, + } +) diff --git a/piccolo/apps/tester/commands/run.py b/piccolo/apps/tester/commands/run.py index b858f587b..652911269 100644 --- a/piccolo/apps/tester/commands/run.py +++ b/piccolo/apps/tester/commands/run.py @@ -64,6 +64,9 @@ def run( """ Run your unit test suite using Pytest. + While running, it sets the ``PICCOLO_TEST_RUNNER`` environment variable to + ``'True'``, in case any other code needs to be aware of this. + :param piccolo_conf: The piccolo_conf module to use when running your tests. This will contain the database settings you want to use. For example @@ -76,4 +79,6 @@ def run( with set_env_var(var_name="PICCOLO_CONF", temp_value=piccolo_conf): refresh_db() args = pytest_args.split(" ") - sys.exit(run_pytest(args)) + + with set_env_var(var_name="PICCOLO_TEST_RUNNER", temp_value="True"): + sys.exit(run_pytest(args)) From b32ca56b31a9cceee9162488269b5e1cdbafb805 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 9 Nov 2021 16:30:14 +0000 Subject: [PATCH 145/727] added logo to docs and README (#334) --- README.md | 2 ++ docs/logo_hero.png | Bin 0 -> 18396 bytes docs/src/_static/.gitkeep | 0 docs/src/conf.py | 30 ++---------------------------- docs/src/logo.png | Bin 0 -> 11930 bytes 5 files changed, 4 insertions(+), 28 deletions(-) create mode 100644 docs/logo_hero.png delete mode 100644 docs/src/_static/.gitkeep create mode 100644 docs/src/logo.png diff --git a/README.md b/README.md index a0114b39d..bdb3bc767 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +![Logo](docs/logo_hero.png "Piccolo Logo") + # Piccolo ![Tests](https://github.com/piccolo-orm/piccolo/actions/workflows/tests.yaml/badge.svg) diff --git a/docs/logo_hero.png b/docs/logo_hero.png new file mode 100644 index 0000000000000000000000000000000000000000..fde37ca2f60f40102e3a7e4fd4bbaf6d220900cf GIT binary patch literal 18396 zcmeHvby!qU_wE=NAc}&Zv;xv0B`F}%-8q1Ccc+SgfPi!fNW;L;ozgLcfOL$2^Xa{z4wauUCUrac?n!>Vr&Qmf-5EYRv7}hB@BUFTex!_ zd?G{^pA7!-!9rA2(Meh24W*QrDECWVZf<6FmX{C+U8GCoAmdvlf-WHrQQtmu0vw!B zsXoi@h@=TRs&wiwrCZ5~R=#;2i-&X3#s${7bfvpiE&4yywHU>+jB^WvaXkk(EUBGr zk7oui=bYq^Q*pL5 zT=ml$zkdES_Qy4_fb&>e%~q;bBShTRPL6$M?~CPthyc+*UB#2Bp=dE*M;?_w4}!up zo$+Q4TR-%%%3m4Aa&f-%;Du}3N?p!$c?7e@rX|HxikWh}PznsjGl+S1>+XnH+6mjH z^T;#)-al@>D-q>%wbBg(*tc+uyA2;ca*=}N8q0;IMnxKRtf7%jNo%QFhA#Ry87QpJ zvvFbEzsn7#x6i9CJs_!GJc#%`qT}z!Y6owmtY)#$#NWQfSNtLTRwO!-lFIf786}a! zpAxu2l@}`7M)PZc(Cf%)V4k^MeW`%O!0YzMN84%V*PpyIYi)bo#R=BruDzs|69mFS zh52*MJzK~X0-=ORy%kY)|F$vf;jMe7c%t z>JJF`ea;IXB9$8Jj_o8G8 zdy;HQmL#jJF~4i6t*Xq~ETBY~BQW{=$MyH8REeL(1}i^#-Iq8^{_|A><>=gEfGCYA zDv5!!6OD&F@#0xT%wj#LRI~ zd@_@kGHcj9*5h?xSP%$@FtA5Xn%)&=8xuH@S1+E?TR}tTXci;wkAvp79$+bz#T0dG z*G?utb6k`#9dZJIDO=8ZYd_()(V@6pG@pEGY30n)JgesvM)_G=<0g34;1RI5Y_F&nmA1XmPq%mr zU7qcKCB;2^eX+gzdu_wTkCM1gJN^cE^7k9eleeiqwo!Tc5m0On)qOo>n(?8Vr|7mg zVOJ>yk3bqDD}XfE0zlXF6h8&e?2i-X$3IE@C`c?v>hLg6H2Z^TJ2U^Pviq}1(fzP_ zEP#o?(cSt4-X*%ZZ3)Fp_C31g(HqFmVgr;;(*v{)ih=N37U?nhi3dA*DQm4ZN*nV% zcdnCZLcZ%nUYR22%_@ov=l6V0kV8()+8UTe6vhMc5>8t%GLtw)t4j826P3NESsL;Y zk~+CvQG}Td1c+!gf)pog>r#oGaJcSkfPDBZiKC0JB$)o@;h0grppV!(m-62DHs&IX z_u4Yx`~hZ>Fw-y)!8jd0Px;Su4>sNwTd+;|*j@kd-l|3^DUfGj zQi5l2?R}?a6!*%{PnZw#zJQT%5@I&j-Grxc-k3o!U;%|Y{w<1h1owkXw8rXwR78sIuXkP}hCyRuH`x?C zyGcI7bnihRKbbLe8UEHS@!|~7aO>Jq{U4T4d zdb21mkB!*u{5XIvWYI?J^z^jXtaEXsV-bM=X!FKPtkiCfTEOC$;h_`iIG0}QKepdp&d1m$$ zCAtzNg$pGAKi@tKiNZ>z1< ziXPH_DotL1Q~08-ko!(}=TfO;D}~gj#j@VSAsBm?Xqv8T3xAI#xA&EJgvcW_>bI6! z0T9}ci#M3rC^tuT*;d=lHFzN6m>)lW%%IN)eXFEf>$ub#%UEqQJvKR6>vMh-ujfhs z)nNpbzONA`P_Yuv{P9h zpUbufLNk+-`JFV;#DzMtIR4aeL=y;{jY9+u#huQh>U)h2);sP<>wO-#7J@LN?AkFc zJ>&VSeWJHT7Q}9wKY^1Qk%;(cto3Q zd3BU(@>pd_F~2=mJLE~|;Akorc=_rza0dJQ_M48^y+=n!wd2_|;La=F=>2%_s`NhN z1P502@t*`ET?66bEL9j|`5veP!uLS1;;2Wz_76rBte~x=rrgqQuT%x9DhF(nCTTx$ z+)&xLJ}F3=BnAaNG*>b9?`pczYCKxub-v4EOZSCf+BuIEGI&A4zGZJYd81e2D3AYx zyNz?JUd*ET;rBhMipgX=2tenANA-Bue_YTa_7)?Ad{1`fCCELf85w0zj_j=~`+{ey zne}elV|8C;gFBayq#J5HA_Q-vl`4|%sh77{MirX+BWxLZDv5E4_9+T18Fl?G&t=eO z%{m#uYql|`+x2`J9*)+gZCDYs(X9$d^9LL^Fd=F=#vj;Eqn0XTYCas(xqY8>qQPT7 zj#-D0Sv$)`$zo)5^sqqxay@`Z@6Ij%Ry#jUc&K!slO z2JkHcNO`##9Npajmk+`Q=SQ3Tt{W3I4vX;WxjNrV&v*paQZ*`iaAmsL56xKIQ0BHh zw~t>CM)GQ4@pWOMbaxMj=sbt$yL1oDiCJi*Rc#SVa}#%S!;jY~cKRwQUPd-^sQ$K* z2Hj)xKk1wOp^i)lX;SX;UsDtUTzE;2{+KgnAdT@E&^EJmhnL4l3OmI|H$~!>W`Vrt z8mA4|3N=eJ($mwMnwoNQIt^;bGT*RFh<7&}YSHOZG&tB7a=#1<51279D^Jt)8~5v{Wsl zyz_>Rz|!t-v-;cY?6$YI!n+pp2lUH~dyFCkUxZ{Ni4BPLWbG_Owp8ngzxF%KNFIx! zOVj;bX04^M_cn9*gc%Hkas;z`MRwc;6a;qLANZZLnWgz~-p;ndT|aeob-on&(2l61z2}jik=xrF8{fKNQ`6IfDg3YcE!2U!7~tRb{{7)L z6QPWThQ<^kIll9GFh#@M%Z&)zTA1=_Uq7l>?=?Hy?}&VrCtw6S4qzQh6gA;v#Xg%> z(^WQJSEnm(8xt>0eo`SE{V)yPz{B0YKs*lv2}{ndrJyU+vFH=d`p`ZDq-Nw z?{R=~`O#`vJzZ@(TL(HBb%k?rb7K}bJKIG3G?EG7+l<^Ex7O#Cin6q_LIG<~0BuNHaPJ9Rz4iRI83AETy(MKM#s3(lyq9EGVBsQa8E^vTJ|=bHWeoSgPg z_8LEY_+YH8KiwIe!Bd)0T2y4)7KmG0TdPgPRebRWUc@!(K^Psg{waw)zRmj1TYO7( zjEf>m`Q!Tg02Wr(?W1YC=F6k$E;QGZd-7l8DbQvL3K6ZV2NfK|VMJe|i(xRs00Z&E z^9a0!%oi-C;zy~e)+;Z68R#rA_z?OHoxIi`7p3JhLms^n=naZ%oof`#mvB zu#dsThQ*_(gWBfpzdH9x1r2t9aWI!?;5)YBtOIVCe#3HGrTGe6|L z!Xvrc?kWCU?($j+ZaX5T%-YE&n5yetZ&k9cYbL7NjR7zG{NqbIHg4k2VYdwZjfSXZ zuf8+50T5bOc$`A~8sxp@>y4dRFSXuGFr2+=JE(>-l-qEI78Z4TmODeV0XJMeT`yLNMxD z!Dx|WWq#DuSzJG0jHSr>bZ-xns#(XEi^GMcq86!e&Oz63m-MOW87ra@o$9b6y5}&9M6grDZ=$J$yXSlB>d~ovr10d^59HmP;H&9(O_pNwYK+*K4z;z2^BFIn+kBm-DiQL(2&SsW;)|VEblcCn%)hGFIf0+o zPL1_Wlsf~vExAY`5*em1;=t~6iUA7;xhaAd=&!QDrM1e56{x1|9WSj_by}{WVV%b; zIsUeiH0U&NwBuDF%h9b2ej$PL^2SD@UFyn?qd|R*+6H!hL9V-Rq>60z>-K=g8VvuG zHi_I%&h!V*#A}%P9Mwq!(|2^35H@V7umyfv#^%#D);Gw*`mT7nlvX)?j$mRMaHfXO zJll&5IAq8As2T^C-9hBKd%cVEkphS&3I6&vK-x9mz>40t>K(jQd*A51B5hOFEeyT7e4x52}OC%qkg1M>bZ zIJc1BRFRr*p8bfbL@>t1PoUF~`f;0VW(N#*^hr3)a^FqM#pk8%)3g$% z2h47I+9$Ong16{Zl6{lD3@^JZk;D^S&2ljJUs|M&#y(5+Mp3zq6B_HVy9LLpO`>-u z)%cH`1uL6boj@FH@C@-=0uzr{PcTcPO{{(v_jj)|!gM2Sp{Z*JFe}OlzUZSwWPQbs zRkKF)(1A4GC(&F9(+d81vvow1{r*Bua21q%{{r~ zquSvb8n*SWN`X_mz(f0uQG4$x?|kf_)9RNlC;*YlXh@>{Y}rin$)!neKz7GfT>4s? zH##<3zs|!!dzp@YdX{q*_Eza+k%V!3QZsVMbHJ?n$<jy~@XKx=OAOX=t#XJqho^FGJX-4h;Tz&~coy zE5DK$7IpHIEcDf{$Fj{&{ODdUM4Wvr-9`v=&|mCib$h>d36 zL(gc}ks;;!QkLxqTYW=L+2c$6P=sjH=f4VW&VY$99txD}?4Pn^N75Xob7z$jlseHg zQ(%{#0Djt1XB^lU0$_Krp&O5;FqV0w4m-BtY`p;EMlbQl_zF?r0p;Lry5Eqg&%&j6B^|o zTTD?u@EBa`7*Qzx7};KWG;}=prRu@WpzEdS!rlTyY$WY_q%t1i+0 zh~~HKeA9nE?L9&-%vI1!>gU}RwbmYgrFdaK-R%XZIv;RXVV)omd#4rI_)cqUYJH~n zO;^mIhlx!KIs}2dl$-v7x&?vIV;qKyV-OaANU*k|LKXFfy{NkSc+xCw6oev{r_ZxJ z#GAguuEmWQp7h#`NEc5Hp7~wcZa0lYOFX#CJ1I!KMjm!&6OCUClo>Y*LPsYB#d1X# zY;K)8CM~GY`#l$p1G*gwj06uA6t!;c^tUi7p!+^>%Cw& z*7{BPup+~d%4m&ZCO&shXAMN3=?+jGs(&LN5_0mZgIuA@)yxRnnHnXOHv8&kHS+4H zx_NVdtpL~fe(f*#0glJ`d)Rru&r)FTMIn8T?YZc#{^pwJRhttBGf6AiV5PZF}2u*2gIgw4fI51e9-=yu#Y$WY4-wbS4eFrF3*YNBLYpoz5Z4_PI5W=p}d*T-j$CQS>2RX z=Nuw;Jaur-nE97pZ)+9r}MnW8VvBa8`Q!DJKd?mS%=IYwuX8C7c zfrTS`V7sW0x#sj&w-w)$K(%I`^FSFH0syTAP&7H7y^_e~I%FE@QFg30buKBA^;CUh^-Zu=67`p*QtJe^4ovAVLzNn;q|)7{YVp! zf!(_W3O`t2qneecKvOYZe}V@6~AdGiH>AZfS_FZW_qvLKkE6~*P)2(4M7UVr!S z^J!XC?!@oyL($8K=DGCcymM6ilC7>%o_%Yc&{_A<@?@3<{+!LR&vtFJLr2=y1G@tP zv;1l#GOG+8SUdxZ8KT9Sr!7uPl=I|jA(8iX>IzPMmc86AL$q==XPNB<(Bhon`%I7o zG_0UNA??cCLV-O|!2b7-4>xY>vW(Zcpoocy*HNhby}b&!y*LOwiAgXG>2o|=T8cBS zIr{5YMOBrh^2CfuOgo$;b~w?9Y%W2&whtQrm{CF3a{a&;m8c$D{dPuMzo;Q`HaTeU zYGmL(m#C@|!Iq6?mVK)x-4UFuKZPpN8R{@-QM}eMR%}xwRT=m59hdLcdPtWZUvv?p z-Y11Z<^czWMnB&x({m?^`&lSQu;27;gm&GIN9hYSFMf^27L!p(bCHt+#~%YA_W9{l z+g>I&pC;pwADnvgyKT+XI^|MJ$;7h&i0=>s-*s)eqHJco^!4?1{x&Z!FA#3@MAJS_ zf^Rh*&qbh16kU&(I}xwp zgOT~Q;MD6g{oA>fhR$cVZj^TE3tU#G=_Oh|N^G-ia4~5qevl%w?Q#vGY5rGffJw~) zKcYlRTJ!8f)y8O(k)-DVmm9{sr*{0K^2IkrYJA|s3ArkbMywPvZ49*$=YxWR zB$oRsEP^3YBn@}0JNUL#MN~_X~9fL zpo_r_0x`vK351`(De(dbA@XAR6&P%;051V{Wd)|m3K(({BM=BiMnNCj+l+jW;hxD5s01v4thN2WhYKGxVY;s z0b19+B%(f3C-%fI9qjdf4Rj%>@nZC^+}CRQ`e{Qr{W1WbK~5@be1-qoqg(H&-MOGf_H zaApT0g{E`8CWND$+(0rj12cb02@kRkxbmYz_ME%zsrQZoaBnXTHjkg@7ci=%*ms%G zV4UB+Ti~2AQD)Eq^X0JE8D?W+!`gh|Xk?VRCy+mx;9P51_hoR?(%igeDNaWqmEN_m ztxGDyzKaG+hg;p0KWBntE4Z3Zm4wHoD_acPXjtJeyz+kD>?-x0R=E(lciSnXdjYID zX7=1~m1@LG={MkuPqhSio+T2@SrAJ;Hn{uI9yMinc%yK7oNhM%!Qhvh8HtT5#pF*X z%1WP)embult)*dcYG(CEcijPSG$y$OC$ekNW2fad1pt8oNCgDG2|x>gBor4b#2wZv z#8FEDAl${p#mQmiEwnbH_plblz!<8W^8i1Iud!bB%d>R`U2KWtlqbsYsJl^gOpF?^ z_01Q!Fvk@wCuO@|tH9+<+cm8Ve3cZq-JHXqrdeJwW5Ol<&c$q#5t;9n#NnG8_;EK! ze!G_d`YDRKNax$7_ZheFO^8AIU+c&$B;+@eBbUr=^XTR?;xJcdXW5P8@wvIV$;rea zaRw~Y@>&?IchfcEL6lqlteTpdsZ+21RJtrdKz6`USY<6h71RhAY@n~+nKSMA#}69U zmE3*duNmp7AgCqZlrN@QQS-x;i$s}>>}ja)&+L|rlGxC1*b+z~F4Dn^i~Mxq!hA$^ zndKgh*DWbswJn~X>2sk*FQu{VHiUVaSohOYDerj6-^<1>xhbNlzKmfoAvKy903#xy ze>v%I3zjg}+FDynoWLt7D6q1&wl=)-WFk!N-xF0Lkb5!G4ml|-EbQ&=4Q_`47&|J< z`9c#Xh7LQMWq;wij@Spdz@WOuJVS4(JbGHGUWu}TS^6L@ZEMr7zDs4OQB`I?v<#=@DT)lB;yl&B$MY* z09b;(0Fouu?LX_v+12Rz0gw)~f}*@-dmVY1qM%4!+W1n`Rone9H8q$C5okE^5@c+bxkfC)C@c@xz2Z8dmE>%^vD$ zRw)0Hfp2b21s=6OO84Tv1SibWA%A#pFjMfFOD|8xgip@JaCVC0FLxNSSpMk;^9(Px zRr>esdJT(;l=`jP0jp?p3H&x9ukCImp|)3#FQuWQt^29*ucR@N-cGA>S`R~SlVgJ=eXFFYwq>nq@)D$$?LR2s0341)cvXb&Vz5ze#l*&7 zW^-P*VN9<|#1gjn!le}MUVWl1hRDBh>kXNOxU-v~Ta^jQ$Lu~`yC~f@(#XFfxv}t1ELYH^Gzx3+=hFlRC2n!slR5Fm5hsOL0(?Y&m1=Od^ftR z9xFAiuX_QHZl+r)j%F1z9y;=h;KjxA{L=bt$stv%MepYRgg7*>)^huc;e@st&*4@Y zfTfi&Dv8NCOzwt|ccv3{R!hu*!z6cR0z3UM6GLwmqmS%5`LFK^a(Fj`#PVS*E(PnV z4Z3l1*@KCbAw=scrHpcTpQ6oTO~3NOy^n4Ix$nl)0TxD;Ey}&xWM{PPM%TG2@p(#! z%x?ABGv=w{Px)nm*V163TeNfC#-NhCl}0{1({8Ss=`$jlXAvZ}EDP_{nAhWX|`Fa5+JCgmE3cO7D1`uAHr zoG$KuDi$%$0j$c*TzFPRAqdzQ`o<+yOrGPI?Kk^{(fOG5jSLmTm_0 zQ2A3VS*oRD14Gx@m{#zI+A`VwS_c!_nZD`sV=ZC5B`SGK0sZY zn3!m5q(>bi{q#|f*q z{&UY?W;2><;bJz1?x&vHvezNY(VNrNh%y#a7qcWL1Rzg-)?xQ0bz$vN?&B ztj(2eXcljV8ph8^DvpYrgBg>6k(N-B3b^lFoo^#6tu)wqHM*$jznXZg45WbBqtiu$ zDvGd(L~H#!VM_0_XHZ#ch0C}Mw=~xSxq9hS9a$2^MSHKst-DFTlQI&B%mTF4`_Q&~ znDNTKP+=k@Atl9dpuGr0we{p)u!481ZM79tgBZ9?fWC0;3MXr9Y+S_Xa&u0YjamYI z^TMjbs5m`Txb_AjN9p!hp5g7dUY~cplV2^#H2rJ-bhzlL3(Yf4t$V zIrl{bYqP+Wi-N-7ano@l#pS95$t&B!;$mdUJ5gKPqN*k6r}_DLV1Y5r!DKn@lY6Ot ztDPiv7yWj=nQHH9cvM^rEJoOhbYs;^lN{e?H&EzL(H9S>*uIm=yq4l(K+p8-m*u`R zAV{zMU5~BWl~n)R5ANT)CoAyi&P_d^!%+==AGN5gsi`SIbl27K0UljXk44H;7s`Tg zbaQjl(U}HrGjP17hlknMG|!@T zDx-%EP$<*?!0z~C&lzc%pFd)%ocqN=Ju{1~9a^dQ0G!C6zx>q2#YJwa=X#_B`6M9X zu!4-WY%MX37_Cu0sjH+kzw17ONoY877XUV#FjQ>mBUPj%@c2L+0-}(A;fR~taSYB4 zK%O-3r);}7>bZoxB|ataN|AQ7fvGvZUbiW90i#8{*pwHRC z+DI-fc_ejyRu&*P+yzoYLa=j3EXpTI?+5jpmZRhPO_P0*aZHF712&L_F^G>HaN+A` zX13z4>vFU)DMeDK45+r_SvMrqZIxlzP6;7 zyLa6tr<;Dz3r43}3B&FnN;jh})TU`ND5Nvn`wpX=US}h@t`s|5v2GaDI^;C>na>0O zo@fk)^yXQ}0(b)mKi#$g||2jm7C2(t@z`b^6Ki(+*VYRt|{t_t`q zVM?|u7fxciarw_t)NuV~U(^`<@bJ*W$D1LaLA4Hx zxsf3j==lp#jI${(ifL`_qz9Mh$LHQnlLGI6EC2R+q@=O2F*Q_-p?hFp4jc!NBM_^h z$4f-}(@5FI>;|@=oJ`2(`fbh9w`Eq8zv&(1J%3RKj|NmN`{Kfg55%#{X!R!wOskb5 zM`B_<8&p?>75klw-FOgmmqxy?9qk-2r3d0(9D z=H%oAe^8;KqFS?zN)l8e*x%o$?Q{f8KU}l)O*+ogf@!W(?MF zn5o>-{7CsLmUr|4@c(#*Hm4H=PpxRMTlbW^K7?zD4Ued5_qRZHTh)DBAP5G$r9C~}j#&ma&Y43A3)vU}9w+LeIx3zqPHjXb$`xchQ_b8{0Y zAr4|`u4F@?sA$EzBhVvjvWb)ibLQ(xz5FZ%1F`L{T8EmImE(SG>kFz9MlxatffmYkxA>i{=ZVYFG$QArvZ&GhbG)Lk`eiiSozyw>fPJZ zF|G_XAQs&5l*nl#4pq|DfmgiO03(d7@vZ}0#C+0$d0fA9c30)GbElhojc2WLo}2V+ z5lH)AehQS_k6$1dl*6=s&cS#S5EI}1$d-wVUHX;gk8xM+kKw(Z8+SzTii#K&g?_A) z-aQv}$hFxUl!>C&(~m#azdZ=k9p&{aw4zL~Y!iu}tr*yxa(hw!u3wFdSAGAFMJDiH zjE3}zhi!us?X!G1)!x(GX&rSrO}t;ycgsP}CAIZ#*Hf^se=zOHU_mG^sihowfNt5+uT%_X_Sy)B& zc=_^vx>cD83FUa(%|e)NvObU*>Az&yWXqJ zg?f6ta9bKzJya%ToMqI2u}>rXekVS?egUHu4&jOphudM@qw}O+sN49CdjX=BZrMOU zzaz*jWD-A2^##Hcz0psoceiju3FId{PMB_h=p_Lc2o~}kCo}!zL$oM1t#w^SVSDz` z=Zeh658+PN0`RYXPf3@Hr~(84JaV3^Tow3~dN=fys>$kb2Ay^N5OXlS-=7EaxM^f6 zUMv$eDH1WJ>t+X+PUA@#s-b*XBA?s;#NW628G8%z!y6OuLk`~5&h{8aHQCz)Sa!%x zoTe+QiSdrXQC#M@UTc@W0ht9eyc*0sJ?$=b(8%U2CYkw9@dS2;;+wo&p0=Fd*7G{3 zeRk`5Wp!VTn>N&o7MgGKatB7x3{jk7)@SNd6ty@bd6b>h9DZr zTU5c_Zyj^Ba2(IfC|&!JN%U%eQ=a#5Vz1R2RlN=Gmljm_&eKcL6WMy#uda4E1QwJ0cmM>3evU??v6a&07Z1{&@_Mku6fs_hlolT)&R+zu(PkptPlXSRMVS+cZTkhr==>0GNpHzME)powiiiy4RSyu}Ay=EU-daVKm zNAL4g6IT`D*V{`IUy!xcRn&ty_5)z`i^(;T40ApjE1wKPQ?O|hz6^`HF*j1nkvUBl z<=0rq1-IS>#K-O8itHtjugYYP@hgQT{$%Z3BEQ|0h>hDw{4%UR1Fhdo5+gJse%-^w zT;?;szPIfQlbEeAF5{wQ1qTUj9^O}lDzCu~{}C|~-tqv#Zym5Zq_6}KS#KFpnwz4j?ch}x z!Mi?RvsX>Jy3=?hf@HReU9OyF4je?T)_RJyG&fASJ}l_`5>*5AN(qx)|H6!QkQCmh zu{ru>sngcrZ9(K&%DMUz;*uE67w>qq3#6~YwR#m0PI3a0iLR#UBdqO@ou?Sn` z*X5-a24?#^Us6f`1Fr2O_Xma0hsAzS6ocwsD-NqU^!o7c4yqd{OGj4dw*qf%z^Acp z=c6d@Hu_l_+aY#Jn}lK4WHadgOv`0YE1;5Qqd4=kh6q93;wC-{gDS}vGfF_l6N$Yns-xSHoD1XW^ zpw#TcMIOC^GfpceH|JW|D&-)Y&3LO<;NEO4+Re_elE;g%C?CeImp zvE`K8beVAKA1v0Smv*C%J-hnW z0p*qtPRRSJw<6sq?TbWu?(=avwpz80CONEGwNw(}4F|(AFkQ>B(Dv_ii3H?+wVS2q zHe$Rg3b7z9Zq^}SGn}#4yJg}l>jF+^LrcOdJdWfnfn45kivgFUax&A*cU5l8kK)n0 z9V*BvWwUD8%e%cQ3kNYg+qQX6_Qb?*DA`di<92$t)QnhmU)rd#1W6_JvsWRfuJrsY z!Vl|4J9=}M>Y0E-y1>_nd*I$CF(nQ%QmC$0B`LXABc8r}IcU-Al;pUX6Qzr?RFEI2 zxX}bavj<33V*c#lqxqfVRJ+U06dAb?lx*0o{C6Jj+V)!QIUNeyrX5(=h(F`rHf+1n zWJ~IHN!TOJ(aohFw9wO=cZ}!B^{Bo`KS7^cqY@UDV*zR)4!mMvnW!EH3ID_{HV5p(sj@19Ly>AH@@ z6x5mK*9x6I$g^auu;nd$wguXLjXo z|L93?_L_HQN?#lEr{CaTyxc0M&Fw4c1g9BcwFgoXk}q9?x}GDiit%%An^mzt1O=By zpZOu?Qe~zEPU)IVM-R|D4={OOD0p?$)RtqKZ5h+{R3$1@|6&1TMHb0>(%kvjk~18p z^BWKg=*v*kCAqt!Mvk+RrGKlmkJmn`)T`rFLVgHE{ziF#qvZk^}H z&9=eqrrf7i+-HHJ==1y+!@wdBV{+&#sPmKkN%@>tNsuRR4dGhUaxc)bHnhn1RwDV%&h_HN}VD z<32n=d+$f}W~{L^e#@Vm{~2xP(&S{xDLRH$y#1B35? zpaKne!Qk)8%F&e(Uy;O?xJ;M3I5KbKue|k0MU4=sWhW*~XwZZkI@KS?~W|I1}b9>5k z^gsdfu^hGEj%U}_a=L;(wA5OO?}j;V;*68`Z?pRP+ZRMm%gKy))2jX|U!s&sPWM)&+p;5Vr@M%Im=4`x#~Hgx8Gp1Ch{TSA>FqA$KyOxA&WT zZojD$4=HzYzil<6UyAGt{}ux}0_>}})^XqF=IZKK#3>}@M(i;HXQI2RWUq*ZD6-0g zc}#Y%6#0TsZuf$~X0mE(s{konVYFsqYE^A~i-R1F?32FS_7$_|TltZhjTU#L9vq*b zH+pi!c;j$o99a(kFzNg$OE4up>BQgbNiEIXoNI(r*@#*sTqh4lC~Tq5m+CK3vqe^M zE5*<#Gp<2v+~vv0oyeo18@Uq5?Hw@=j>1`@FQE&6ER5NbB1;N)zf|vjadT)oJbR3# zF}6&!qG@)p*&>+%QsY_@hdNa%i*mxe_Ve-lZB>E-OBWyW|sYUP>?oN_?_P9|xB<#$I$@UAp z97-=;-JSbku5DC_lYqOhq*gR)tC2dgQI=e0{UUyLwIZ>TNJ!l9?q9Gvu(Fj(VB^X0teU> z_d$DeWd2b2yhl@VDjjxg?O2_%giCN6^bdgIZA>%!4No>xqAt@XP7uI)1$JcwO*MiT zaJRPV<0Hs>4@{;yC-=_qD|`{3&5~Otx6DN5ZIHut*mMuYS}eFFGk%9sN7V%rzg6n4 zB3^V<)+pGaRsdmN0MN&M-7^zT*pTIyGLtmg@$VB$9TCo~ zi3FX`@7iffti@6S?Z!qNf?{X{Dbn)TP4*uT*1dbO@IrdBG(d_yHi&f0`?Q<1QIFU- z2yFpA4*AZ4*_;Wr_QA$gR#2=QnKGXKQ}X{53=v}PCDCa~YnVfC<{*^ol<`id0gVAs zd`!`DP1x&8OOj~e(hR0ZSl9-mas)BOYzx0YDUV!PU0;XG|5*(rIAq|;(hLfW*tk~q zrp5fF{!`_I!LOWKn%py6w7$DMRy6EmC}S-euLG%>AoUD`Ui*B#r^V=}fO`JDy>|Wh z2@;c+ZvOGX0^~c?aa}9@86X0j_{K8^uS2!}vj}Za*Q)w! zx_N8UAL8*OuZq7FSU)7bXoJE;Nx2ws^GP|vJLpOV6sc*Hvq=VSRO(o#dcslQ-H;t2 zjK{IPF{<$tN|PCeW|6p|J`Nl6N5xtToL4 z3*b+sj{-O;$cy-h$+0v0EN*plHaHcp_saFp2qNk)- zV|y}8CIvtOAG--8K?j3@|Its7wTBd&caO<~4@kqqTtN+Hc!DD%vq?eWQbE|HiT3zk zff@e%0E#n$Ugpjus%+`{CX*h$xGoy}RpY(~{yR)z=^H)X4Y^LLH|+T?iHXlbGKQW{ zmf?<(_Qctgn8>Kdx5U}l6G*DZk9AHl4Q z`~!^)C&R(qxVKlG+8N#oQdAA(FnR5ek{1AC0Ev|NAX0=t3h(|c;2*mQ5Do~$>N=*+ ydoD0+uqBhgru%QV|BB?lyX3zI=l?+|T-%COT;U*-X9G$EA|)pOw&;z)$NvRndOX7b literal 0 HcmV?d00001 diff --git a/docs/src/_static/.gitkeep b/docs/src/_static/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/docs/src/conf.py b/docs/src/conf.py index dee72d93c..f0e40a355 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -81,32 +81,9 @@ # -- Options for HTML output ------------------------------------------------- -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# html_theme = "sphinx_rtd_theme" - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# -# html_theme_options = {} - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] - -# Custom sidebar templates, must be a dictionary that maps document names -# to template names. -# -# The default sidebars (for documents that don't match any pattern) are -# defined by theme itself. Builtin themes are using these templates by -# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', -# 'searchbox.html']``. -# -# html_sidebars = {} - +html_logo = "logo.png" +html_theme_options = {"logo_only": True} # -- Options for HTMLHelp output --------------------------------------------- @@ -120,9 +97,6 @@ # (source start file, name, description, authors, manual section). man_pages = [(master_doc, "piccolo", "Piccolo Documentation", [author], 1)] - -# -- Extension configuration ------------------------------------------------- - # -- Options for todo extension ---------------------------------------------- # If true, `todo` and `todoList` produce output, else they produce nothing. diff --git a/docs/src/logo.png b/docs/src/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..2afc9b781b86c27d25803ce25ddabc4c5cd2ddab GIT binary patch literal 11930 zcmeHt2Uk-~)NTNQR}nBOMXCk_=^)aZN|PWcp%($AH|f3W3q+9~dJ`mcL^?=|O7BH_ zi}c(PLX3n0Oy`TN;=fg`?Me=KO*B}rGxzaN^1O!5e2jB7k zkbuv=fVoo$gdosLR`#VcLh*xU6!}W z$U>F6tlJ`zMjxiiZr*?Ttatcpt`+tg9DGM8t!^z1}ZC=w3tBt9NuF|^*EgC)GomgW}_P|)?RjS?*> zaLds;uJ-SiNPkFGogZ`(`&Q>C%TJTexXtZshc6v$i$M`VvcU!~kH-6B;14&9QTIr$4$7w= z@o%^c-jnE){;@PvO zNLISUtAa|4CnqKw*RLuaj>mLO+w^vwxV}0``qh9lOfS5Xu=Rb!7orr+`v3p_*8*9N zcc~a3TW2Biep94eW4PLN#au)B?THWk^`DPa7!qTlZ`r$!6LT_wwVex2w`pEPvy#UL zHMGSMnJu}5`*ZiNI0!nhIC$3>Z1Raluqt)D&L|eh)8JIOa?Xt1z$iUd zF8R=c$_c!WiSJn0eGy8*q7?1c{nW@rHBSRCh;_o|W|n;-sv>0OitohHFv>-TF=Q2Y-7F3Tj9)%+s(YdP_*bAs%|z0mJUeheU#T ze(M{t+$Y<}TFdZ^LQ$IAY;~p%nMPkEQ}>^LdBx z9hTR{4XjG8U?Z9Dmu3>>W*8c}K$N`MU=0jVN^Q?N^zx?xk43r#YfdyGk4iLmnc)D1 zc(Xg-US3r-I6bYZuTQ_Ru~A-GS!y-NhR5TV*VgFqK6{JZa&rA8Yb)O}iJpy{gb*rf zdHOXf5J!~@0wX1>i5YsY{?2o8ap@ZyQ&LnU6*TV}%|OmHt*C>K6 z4rz#9$ecam(*BxNTbrDg#-vrK^8}G}C+0{x%J=Wz7wT1f1RZ*Lc?I)A zz~|8%^My{;C+|Ef(Ct?Q_Tv6_1ZKaZDm@&@75D;;!jqA)D{-a(d#k8Z`SS4apy2nZ zc@5g~a?(2dPK0o;mgC~6&AXGD3w6sxkLP3DJ>D*K#Jae;mYBBR{9ap22Cr5=iHD;r zooedx3`vp5M9ewMzFTsD3?aQbJXvvB3in$E;*U$&+92~GOrU{UpiwRR)qF|=o4U&d zV_gr#AHzS)g-e77kpTQry?Q0N({xK-@%=jC zR=;?|g+)Lk?TSbk(jxcv8D(+noG9VWOY_PFiktT46Pa?kRW<3P92Mh7 zncWcNSO5I0Bn3%0#ISzKS)XNA8cT<*4j1n(nUa=46e} zVOhG_aG|a|L&Hn%P#YO5($yV*j(Xxze_oX{(&n7PI(QS}vbTI0=dg&WWpKV1|N( zTb_@}?Xtr#j;u;wP{6O&AMY&*nsuNuO&$el*RN_aIJMoAZ>3!ezv6cjI`aWpFE(}| z_0$fWufKPbGW&d6{zql5ojrXXEH|fki^qYD_ z7)U4$pZUrFI>lu`iQAw$Qe9OnXn448h)l8gU~Pob?(+eLi1Ip5k(kv0EB>T)Dmb4W z0k60oG*;BXTj3laLo|K8&Xkr$J4GbH*~2 z=7+c1te&Hw<}>V)!cG1fF4A}RX!!Ew{TXg1{baMY`18MCw9QDznXuQeACaQhj<_9E z!|eN?Xnix1?wXS(_VE!dj$#d?1fd6K-ka7fPxGoNzVUQ{@!HJvwBg&D1?tO|t&-dV zRSTZl&To_+J#bD!Ns-CwhJ~S97}Cxw5%A zy>bkU>pZ9;vawx%c<8p+l_;oja-1{r56+>BHeA?Xdmx#s7cDCg&(|Fu%4<sm%u##i16S{)Q>4T?|r~d)Ixhq91rl+STBSXHG z$(vh|$932VS z4D*YM>fj7w*i_yeu8)B@b-f$9yu57OlOkr~W!0y)8S`6f+5a>ujYm2E+5-7-L7>#kS>y*|I3$_VEBxn;1 zkE@A2d}>=eHe_CK*x4EnzYjRZ>{WGEZJ0=WGpD&zSpk^(D)`Lph``Hr!7LJ#};GZ%XQ1hxx5047lia~G+g!gm%3 zA(0_%Y+IW>TnKnUaAFY~RDkM;8FL2%%b&@Bh<@$xKD${>Q7w($KSBm|O?wl*gmrq7 zjD~&h zIm824!(`X2E!CAXX8553_h9GIwh6!3@6^C9Y77Z{2t0bz8lK3Z0wmhJEEMzG{J-jA zjEH8&21SraBG_yhM)uS5s0)L^06rp|Ler^uo&RgttUtsIJF)H?)yQQ*ctHi#mTMw# z_3+QT$Gv|)Z^wEJmEkzSwS+{fVn;9|?JZc5pY9anVcSzuE8P)D zc?^rGyTYne;*#iRE|}j9=)w2LHb|sp@uX4>xoAmG$VABcX7?|sqj_q?{UHE#S%nOF zr2ehAzLi&t^tiW8X1s((uP&XV2K>pFh@t%?laRUs=_ogw7rf7=uT5TO5Z!gm_abu8 zmwRrr?WibsANQioD9tU4@yaAEbV=x_dcY8z^sE+%>R{m@`(g%Y@fAOXIj^0nJ6byi zcl1jBwXK$CnTki~38&D=HJ!8vpWGHw+%&_yV9D&48LYnv4T(DAP1AV!?^VnYbZ7cq zINt`8ea?DvZEK59{HeOCJn*1Di1> zlC=!;Y-z8k_itdEO$)l`Q%CH=N&T3hIXNgMdh@j4P~@1Ya>!!QdU<^qrpR>es4BI} z)ih>=GEdSxrifO^o_6bG0XfQt-d}lLr@#pkrIz~p%5mFY-Rnb9`9;+t(k|}qdmyq< zF`7tAWOsB z(Mv=#01G5}%BLYM3C?3dKlA~`gzniW5^Kwgs#vlQ&H)i>6H^79EVCmV==(zE(bwks z7{k0HWX?tZB9*GQJmT;S5O-Svic)B*p0>$Rqh63~3>;@$AMVr+khQVIUEij}sBxev zb5MR-%KZt5@kj(-9=T-b2zYN6 zDlP}r5RFKBG5p?-$wA7&rl`|jYg62(HZB{ZiRtG+^ZnMF*Dj^VLcLYys3#T;OOc}G zG<+g_r#kUWs6wUCk5(rEfiFWLw8P}3AslEb@I>?(li}6hyRBILSh6>HdAiHW7+m9p z@APqxdwA#c`i~z^;&@+mcrK;30vuuFk|I!CD!A6^oHV8PdovrlV~*_>dQ?v;DvWlq zf?EwvGYF#hU&2?MInXg}UM4o}VvI+6B>-Z@x=85?Ta z>8Q1L{o7si{xoX9@YOsTwaKa36bWJNnXeX{UP6D}Sr&+SrW*uvKN-yij|mBngdG#4BH~&k-(h&G)*FUP(zr zd^g!bY!nWsKzF>bd`?N^k*_{YR*wAOM8JulI~RmS8jTLg&xG5{%d2Q?S@KHn3}4*q zH0pkX?dCv3lWk)WcpOrcXgL$V*<=r^ZKFNxP)2;wS+7IS7R&@N`e1GJP}>~~P$ zvteU2H);}L)}JZQt@ACwYEWU}ql?7RhP{91e`sX(1G^g&q4evYSF&HE_aq6~Tze*R zDBq-k9Y`_PZ~GVzt4p?xL0dC@R+aZFg3S_R=$(`7NP6-Wql^mR_FL@MmR^Q%t0wUdar@&d)=Rpb<~1GkjAKe|EMTKJ|4}meDaZCNuZ76N;LDe2k73{Iw)cyP@Le~N z1Q|!It)QrAn=fvv6||~fxQU60L_v$5mw#r!P68k|0jyRG+mL%%C-O7Ec(GjC15;B{ z8hyC^{q3WF-(ZHC4S7yVk}JNzQ0&|G1%Ky zatTu(kZ#!16!U^Wf7{0?lAad=2QezbMFpqH)ev`O6A=e~;v2jZB zZINa93HL;WRGD<%GL7zCKD4+NIWLD*nnnLo|swZx3 zXQ!AOfq%!5k{BvX5GIAwjyLn_HQ#*PXIWv~>|EGC0FN?5(?8=UV+UcgM}IaKxrN>Z zHE5LdN}p_3R@H}9)o<7p>Xf`|4y7prDL-H;N))$5L`9$3+w;}e*R!O!Y?YUn^V?5L zQuF9Ft>mQlfU~Uak|n?eJ0v})KDs;BlQeW)V_e8UPLeqxeQ@1dh2NC4^i4&QE2aVr zKj&kdo+3PBEq5aTh47a<k@^3-Tk;EB`bYIcc+%E|L){{|lW3lgT z#u7BWIbqq6R9x|dq(eug7sY|~wY{318!1Wd1x4{4ri<~5k7~avu_-L34qm`Hr1~OR zd-1$OWM|Z+_XNpnA!9+FnBOVFLiQTMHxY1pmRWObZ^0w|(Ok&^2c2bQv2d|P+RG-r z9Z1;+G8B^tDd9)$6~LPS$o>iJPzvwoK2m^XqU@#t(3=+T114KuUTvo!r8(T{&$vxu zOfckqk+w#PnHgcx4%_{>)Ea2?2=*+TCaUp~m7qZ+>n;v%X<2eD65+)U^HXtSa$lml zt-rjKS`~J?jC066&v4mYp(%p{{nM>H&aL^W5nb@%;bF#4e{cP&w`PF6k&SshQm89y zWW)^CPj+rq#BRL&*NCUYPBv;3SS+|@y9!KCs?xW9tKsCK&2VAH2h3p3i{d32f9{FK zQq#H3^5~)*DdN$p-5q1BW$DzT2!cbGtJ%y1_B_=zcr&rynFIu#?X&3BU8k=^eq+eH zV&Sa*i31lC$h)-;g20wBbX2AyyVuUp9B2!=Xj8?|_PB;3Es=Uc_J#a*U_}<=pZ{K! zm0GT#qoo~v?X-cxU?^goFHb-JJ$s!hg*M$-#V=<@SQQR*AYnN9W4Uqy_W4IPO{;7b zZMA#b$!s-u+qtwz1I@zzDJ8=#O8VQ%F@SLHD?6~xy)}7eA&Zbygs&tbJ#L6Lz{_I3 z^=ZeH^u=Er7$1k**l+>H!oOebx5swzocrsNbN57~T%@SP&@iLrgUmYw-){ps|g`U_!J`G&d_ey`H6_Rl^_ z$@uZmXdXnmPoGC3@*6%r@g3TA_GZF%NLbT>#PU(8S38^J`IV!16yimRqtgPFw!iJO zk3JQ*@n7z*wFGC4?#+qVWEkyx7A)Xwz!TQ6%cmsYoBnbT){;ICw~d}I4)ktg)1w|+ z6E*NG`;mCe^9>>JSqo0gMn zfMhtsXD+n}g-pDy4P#HYjXVBi9oa3`TXku%{^-3ty7BTMDS21VW=l19N5A+zpR#lj z%QOk&o5mS@{WCMY64N^rOdi<4%rCcB@oU9P1Im0n;=y!$_N2eh=%QgphCO-EC=t`_ zO-}mHc5PN#`@Ej&&S`x4ODy&`ZTcHSJ(tKo8mTFb2gA47 zR(TS{U*KCETN&N^CQfW$J^Fvp&huOndM+t^q?ujtQ-7Wz_f{T!im*+@`FP4w@>yM= z6yEybd{*33f4aWwjdh+0{+m zc&LCFdjB~%=!J1+iW~jf>Lq|A6alnf!$Fa~<4!Y8neAAQM2ZX`A`M)JG%p)q`GJ?? z2DoVD(jkjl^f0igaQZFnh0%)uJfY(5E&>peY1xx>UvRhnImYV*RPyb4Yl%)o`K7{;h4oQu39L z4n945^}uxnvqo71VuAobntA$}Ba#s#f2m6%6Hs(=>vbA|~j9o+p<68rj$w(vGHQV8|*ij<_Vc0JjC`z`r#ML*VBBr`Ev0 zxh`T$YiNN0K8`)r<)%fdD>Zw%&+a$F0s#~Gp1mTB8m344M_2_l?PvU|I6GuSgiTe` zlh`IICLry_`Kr(baOJh%0&QK1fwR}n>eBVqmQ==h$b9qO%r<>U@!qVtY?6Q^ zpp{lOO&Grl;+9TQ89V@+~9!)HH&{NPr1VVlH;dgw7S`dD_w>E^+8c9^eNtrPcvd5}bb!(>Q>(4am4l zG;^6jCS{2k&Os%Xu}_UX(@aYr z$-O%f3gG2(in+TGEc6irAU4BmoR=QDZyHSF8gQ2m<1bRsFrN@T0x01j|I5?9E>PG3 zTx!X$(?EdR!D0QTCiX$+Kzl{Lo?f&JUgiv&on?FQd5eTMfA^H3^l4>o1|uX3mG%)S z5ip`*CzAvwl4tjvze*H6KW!uAF47K`kjA|JTKPr`TnTvf}&=4s4a;}Kr(3f;daSvRjEKO);PM)Qwq zL0T}OFyFrOK4s}eD6)oJDcX}hvdV8)GwdHQ7@(@coMVM{8UyTn04)n>?6x|j-JbJW z%+-Wj2|B2#mI<=VegX*$>#@YFU(MCvk4m+dMCrKjZ`NYc?^H)_W_dqgLl2+NGQ(!& zU0)yoAL6*{{oaVs=d+_6z?;@hrP@n#d8sV@@poT}aC$;e6Qr85*6N(AGzHUkIg>ZuVO31tCTKvnUN} z)VEc3m$(Z_8iWU0XXrW5Ic<#)aY7w~L~e;p*eu%AU37Ofs}ocg74Sy`&j4~`C5S|P zr>`yz&bENe6sQNr!ZKHHM#hQ}Z$WnBvL*jHQj$v%eY)dz11iY!#=Y|~q3o}h=h3Hy zGoO$m5yq8QO{I=KZ#fW5Mms5i@{H--VyzX8 zONs*vXyqLb>9n@Qkr;c*)d|23tv}=<6K+6@!;$_ldaktEc~kfbFIG^f?(FqrF_<(n zRQ-^R{jnSa6v`ZqG$Jg>eVsFm8Ym9%@30ysG`I!TUV0_@J96hfra0G5Go3FR=*o&S zb_g>e(C8As9Q%Aqt0(zg)5+t59hH|cs2Q0O41Y|bXBv}z@X zb{c?iOyEw0+9bd7-1G~--CLLbucm;z4+Q(~uAX}{kA~4GKSn5X#^L?zFY+Y6enehC z&gne}b;$M!1lzBb*hTybXlIFZz6X{)JOpDpurkkmX zE3Aj^D3l%(hs;0nXBrzl?^SG)!gQM^2ibLgkFp*7Cd;)YL@64wUN>ybxSg%`u;G-m zG&PYgH)A6)wsi|6cBS3eH0+qO$cObhM!1_=%|9MOW6j1z4TL@xv>WK3I-q$Bk(lj15XHv{;r`P5ybG*6(Ju|f_7!hZH5%3jB z(!tRCkL6OXbHDZo+Dz&A@)`lhX$*Ha5{rIqiLbBTrB#x`yz4-qRf(rwUHj$8;gHv9WnB&umom>x6nD<$w@n?Pb> zxoTz|6hiM6rUa*h{&o&hclhB+^@#hw!5jg&98{8-xun9KZ758 z$|Pb5@tPRQKyB*OoFvo$a5%1*_yBj2?Y_{!s%%vLhWNy91Y8D*jKxH>F%t?Q{3z?V z&@OiG!`Z`VFU1$x%Vv7DZamsXzCxERH8u5;uDj&1K}F3fZRWC(b1M%ny=dcB(}bXI7D zmAJ}et%T#O9EHrO1Bfo5IGgn!83$^nvfkeP$w5CsF%nR38MU?E%O(JE0?Sw|ydbcrZ^spRplqY(z(grG zyXobdSh2pQ1-T6TgkzOB#ZoH61~TrV!nMX$PI~>+CvH$7|E)Q4UFFL0-l{|4xlOsI z15CQ6)$S&0Kzq$`y<^8Bn&-#y+fv_K3LOWSW)8HX!nPC|B&6EvKyGuD))+Nq2j>v* z#qR^C(DLcdrEmsBX@C#i}?!fSiD?IPhAyXk#KZ0mtu5gRQ0zP|bXhQ`I% zX3xXYlqyJLuI;F%2y{o@_;oCse>!0B)9!GJu@Twduhq%_{%8E{KB3G#33ssL3@mg^ z01=i1G-h5XsS6WH>nxO|ogL$MNWvql0ieJE!XRP$uWK_)=DTR8n$z=4S<7B|IWC8r zV>gi@wzvd}bxKKj;XhsnZp>A_4Rwa~-L-9rZNY7INw$;$8)Zya_U+^CP6}-TW`vz^ zqsWA3m30>EV>hyTHkj1gc_SXX4u@(`W#pnn_58Rn&3 z%A$Z?OVhOKjgWMTF3^*4{FP6INzCQnnbL8A2z3o&?EcENxPgdB@H^3wZ2!IAH$poX zN~=2WE9GBmyHbI2$8I3Gr6Kv_|J7A3Y^RpklWoss%07#Jqy>aJx>N8<`iz#Z%?nOr z=_P=Dc#~0_t-%|gP9m-DLNU~RcT7Pc`ahvo*ZEbCy5$@Bp!jt8;%jCfD;l=1yp}oI~XS4Fly#sLL-W`7nj!HyR^of zJg0boMgdT&QB{+PeqH9KtGsvx1a$e!H-Mmz{czdl_k;LO|10Mx+mtGW2#Ek7Ap~-@ zofe=2gab(;Ne{^|O{%;Q2a!L}-BsQGJJRJv)f$6y;_ZjVp8dKSL%4t#U{rTwRwd2= zlYJrUSUCDDV2iMrOFY!=2Il^P)A8V6ji98nr%IR};3YQG!Al4|mf)=)<4z-)2Cv_! z_E*v;;k_@!hJvHZK;%`VNi}1rB|dstW8nAgQh?YFlzGqcG?M-?fSeLJSe96|#MTBz z#C0yDQ(x*Hr8>cNuw#=5PldLj#@0U^TS`L$8+qb8NB+}A0-=G47PwB5B3sF@gL8^xOEsk>}r00e~_7{b!b3s=mKW;QxaCRVO>d&2X2175`+o~%j; zav%xf1#g&#d2Xz*Fjv`&<9VnZ_|EoL=@2HVqZNEO)kwZH<^Nxu=>M0@0(B+Fh0K$0 XT|!_8KNYwD);rN@w@*6nSG7k literal 0 HcmV?d00001 From dda1c10e67e0e847f7e345f2d23dfc9136255dc2 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 9 Nov 2021 16:31:42 +0000 Subject: [PATCH 146/727] use absolute path for logo --- README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index bdb3bc767..50d8560d4 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,4 @@ -![Logo](docs/logo_hero.png "Piccolo Logo") - -# Piccolo +![Logo](https://raw.githubusercontent.com/piccolo-orm/piccolo/master/docs/logo_hero.png "Piccolo Logo") ![Tests](https://github.com/piccolo-orm/piccolo/actions/workflows/tests.yaml/badge.svg) ![Release](https://github.com/piccolo-orm/piccolo/actions/workflows/release.yaml/badge.svg) @@ -10,7 +8,7 @@ [![Total alerts](https://img.shields.io/lgtm/alerts/g/piccolo-orm/piccolo.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/piccolo-orm/piccolo/alerts/) [![codecov](https://codecov.io/gh/piccolo-orm/piccolo/branch/master/graph/badge.svg?token=V19CWH7MXX)](https://codecov.io/gh/piccolo-orm/piccolo) -A fast, user friendly ORM and query builder which supports asyncio. [Read the docs](https://piccolo-orm.readthedocs.io/en/latest/). +Piccolo is a fast, user friendly ORM and query builder which supports asyncio. [Read the docs](https://piccolo-orm.readthedocs.io/en/latest/). ## Features From 3b47e45c6f41b63e8d95b178682244d8a4e3477f Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 9 Nov 2021 16:40:28 +0000 Subject: [PATCH 147/727] bumped version --- CHANGES.rst | 16 ++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 44e2311ce..362079a14 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,22 @@ Changes ======= +0.59.0 +------ + + * When using ``piccolo asgi new`` to generate a FastAPI app, the generated code + is now cleaner. It also contains a ``conftest.py`` file, which encourages + people to use ``piccolo tester run`` rather than using ``pytest`` directly. + * Tidied up docs, and added logo. + * Clarified the use of the ``PICCOLO_CONF`` environment variable in the docs + (courtesy @theelderbeever). + * ``create_pydantic_model`` now accepts an ``include_columns`` argument, in + case you only want a few columns in your model, it's faster than using + ``exclude_columns`` (courtesy @sinisaos). + * Updated linters, and fixed new errors. + +------------------------------------------------------------------------------- + 0.58.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index a4b87cf28..94a33d464 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.58.0" +__VERSION__ = "0.59.0" From bffd3df448afd4bca123cab340f830984b42f0fb Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 12 Nov 2021 14:08:02 +0000 Subject: [PATCH 148/727] include_columns now supports related tables (#337) * include_columns now supports related tables * add extra test * make sure dict method works too --- piccolo/utils/pydantic.py | 21 +++++++++-- tests/utils/test_pydantic.py | 73 ++++++++++++++++++++++++++++-------- 2 files changed, 76 insertions(+), 18 deletions(-) diff --git a/piccolo/utils/pydantic.py b/piccolo/utils/pydantic.py index 16c6d80ed..c28ec662a 100644 --- a/piccolo/utils/pydantic.py +++ b/piccolo/utils/pydantic.py @@ -48,7 +48,15 @@ def is_table_column(column: Column, table: t.Type[Table]) -> bool: """ Verify that the given ``Column`` belongs to the given ``Table``. """ - return column._meta.table is table + if column._meta.table is table: + return True + elif ( + column._meta.call_chain + and column._meta.call_chain[0]._meta.table is table + ): + # We also allow the column if it's joined from the table. + return True + return False def validate_columns( @@ -161,7 +169,14 @@ def create_pydantic_model( if exclude_columns and any(column is obj for obj in exclude_columns): continue - column_name = column._meta.name + column_name = ( + ".".join( + [i._meta.name for i in column._meta.call_chain] + + [column._meta.name] + ) + if column._meta.call_chain + else column._meta.name + ) is_optional = True if all_optional else not column._meta.required @@ -230,7 +245,7 @@ def create_pydantic_model( elif isinstance(column, (JSON, JSONB)): field = pydantic.Field(format="json", extra=extra, **params) elif isinstance(column, Secret): - field = pydantic.Field(extra={"secret": True, **extra}) + field = pydantic.Field(extra={"secret": True, **extra}, **params) else: field = pydantic.Field(extra=extra, **params) diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index 51e4348ca..9ba00eb9f 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -4,7 +4,16 @@ import pydantic from pydantic import ValidationError -from piccolo.columns import JSON, JSONB, Array, Numeric, Secret, Text, Varchar +from piccolo.columns import ( + JSON, + JSONB, + Array, + Integer, + Numeric, + Secret, + Text, + Varchar, +) from piccolo.columns.column_types import ForeignKey from piccolo.table import Table from piccolo.utils.pydantic import create_pydantic_model @@ -281,33 +290,67 @@ class Computer2(Table): create_pydantic_model(Computer, exclude_columns=(Computer2.CPU,)) -class TestIncludeColumn(TestCase): +class TestIncludeColumns(TestCase): def test_include(self): - class Computer(Table): - CPU = Varchar() - GPU = Varchar() + class Band(Table): + name = Varchar() + popularity = Integer() pydantic_model = create_pydantic_model( - Computer, - include_columns=(Computer.CPU,), + Band, + include_columns=(Band.name,), ) properties = pydantic_model.schema()["properties"] - self.assertIsInstance(properties.get("CPU"), dict) - self.assertIsNone(properties.get("GPU")) + self.assertIsInstance(properties.get("name"), dict) + self.assertIsNone(properties.get("popularity")) def test_include_exclude_error(self): - class Computer(Table): - CPU = Varchar() - GPU = Varchar() + """ + An exception should be raised if both `include_columns` and + `exclude_columns` are provided. + """ + + class Band(Table): + name = Varchar() + popularity = Integer() with self.assertRaises(ValueError): create_pydantic_model( - Computer, - exclude_columns=(Computer.CPU,), - include_columns=(Computer.CPU,), + Band, + exclude_columns=(Band.name,), + include_columns=(Band.name,), ) + def test_join(self): + """ + Make sure that columns on related tables work. + """ + + class Manager(Table): + name = Varchar() + + class Band(Table): + name = Varchar() + manager = ForeignKey(Manager) + + pydantic_model = create_pydantic_model( + table=Band, include_columns=(Band.name, Band.manager.name) + ) + + self.assertIsNotNone(pydantic_model.__fields__.get("manager.name")) + + # Make sure it can be instantiated: + model_instance = pydantic_model( + **{"name": "Pythonistas", "manager.name": "Guido"} + ) + self.assertEqual(getattr(model_instance, "name"), "Pythonistas") + self.assertEqual(getattr(model_instance, "manager.name"), "Guido") + self.assertEqual( + model_instance.dict(), + {"name": "Pythonistas", "manager.name": "Guido"}, + ) + class TestNestedModel(TestCase): def test_nested_models(self): From 5135269109e9bf3dbda873a34b656c8e883d44b7 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Sun, 14 Nov 2021 13:21:08 +0100 Subject: [PATCH 149/727] added column secret attribute (#347) * added column secret attribute * tweak docs, and force a `Secret` column to have `_meta.secret=True` * fix linting error Co-authored-by: Daniel Townsend --- piccolo/columns/base.py | 18 +++++++++++++ piccolo/columns/column_types.py | 27 ++++--------------- piccolo/query/mixins.py | 4 +-- .../migrations/auto/test_schema_differ.py | 4 +-- .../migrations/auto/test_serialisation.py | 4 +-- .../commands/test_forwards_backwards.py | 1 + tests/columns/test_base.py | 10 +++++++ .../2021-11-13T14-01-46-114725.py | 24 +++++++++++++++++ tests/example_apps/music/tables.py | 2 +- tests/table/test_select.py | 22 ++++++++++++++- tests/table/test_str.py | 4 +-- 11 files changed, 88 insertions(+), 32 deletions(-) create mode 100644 tests/example_apps/music/piccolo_migrations/2021-11-13T14-01-46-114725.py diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 8d3fe444e..c85961d38 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -127,6 +127,7 @@ class ColumnMeta: required: bool = False help_text: t.Optional[str] = None choices: t.Optional[t.Type[Enum]] = None + secret: bool = False # Used for representing the table in migrations and the playground. params: t.Dict[str, t.Any] = field(default_factory=dict) @@ -338,6 +339,20 @@ class MyTable(Table): This is an advanced feature which you should only need in niche situations. + :param secret: + If ``secret=True`` is specified, it allows a user to automatically + omit any fields when doing a select query, to help prevent + inadvertent leakage of sensitive data. + + .. code-block:: python + + class Band(Table): + name = Varchar() + net_worth = Integer(secret=True) + + >>> Property.select(exclude_secrets=True).run_sync() + [{'name': 'Pythonistas'}] + """ value_type: t.Type = int @@ -353,6 +368,7 @@ def __init__( help_text: t.Optional[str] = None, choices: t.Optional[t.Type[Enum]] = None, db_column_name: t.Optional[str] = None, + secret: bool = False, **kwargs, ) -> None: # This is for backwards compatibility - originally there were two @@ -375,6 +391,7 @@ def __init__( "index_method": index_method, "choices": choices, "db_column_name": db_column_name, + "secret": secret, } ) @@ -398,6 +415,7 @@ def __init__( help_text=help_text, choices=choices, _db_column_name=db_column_name, + secret=secret, ) self.alias: t.Optional[str] = None diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 28e880988..edaeccb26 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -201,30 +201,13 @@ def __radd__(self, value: t.Union[str, Varchar, Text]) -> QueryString: class Secret(Varchar): """ - The database treats it the same as a ``Varchar``, but Piccolo may treat it - differently internally - for example, allowing a user to automatically - omit any secret fields when doing a select query, to help prevent - inadvertant leakage. A common use for a ``Secret`` field is a password. - - Uses the ``str`` type for values. - - **Example** - - .. code-block:: python - - class Door(Table): - code = Secret(length=100) - - # Create - >>> Door(code='123abc').save().run_sync() - - # Query - >>> Door.select(Door.code).run_sync() - {'code': '123abc'} - + This is just an alias to ``Varchar(secret=True)``. It's here for backwards + compatibility. """ - pass + def __init__(self, *args, **kwargs): + kwargs["secret"] = True + super().__init__(*args, **kwargs) class Text(Column): diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index ced0b0e77..338dc5620 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -3,7 +3,7 @@ import typing as t from dataclasses import dataclass, field -from piccolo.columns import And, Column, Or, Secret, Where +from piccolo.columns import And, Column, Or, Where from piccolo.columns.column_types import ForeignKey from piccolo.custom_types import Combinable from piccolo.querystring import QueryString @@ -288,7 +288,7 @@ def columns(self, *columns: t.Union[Selectable, t.List[Selectable]]): def remove_secret_columns(self): self.selected_columns = [ - i for i in self.selected_columns if not isinstance(i, Secret) + i for i in self.selected_columns if not i._meta.secret ] diff --git a/tests/apps/migrations/auto/test_schema_differ.py b/tests/apps/migrations/auto/test_schema_differ.py index a0b0d93e2..35e4a6a1f 100644 --- a/tests/apps/migrations/auto/test_schema_differ.py +++ b/tests/apps/migrations/auto/test_schema_differ.py @@ -38,7 +38,7 @@ def test_add_table(self): self.assertTrue(len(new_table_columns.statements) == 1) self.assertEqual( new_table_columns.statements[0], - "manager.add_column(table_class_name='Band', tablename='band', column_name='name', db_column_name='name', column_class_name='Varchar', column_class=Varchar, params={'length': 255, 'default': '', 'null': False, 'primary_key': False, 'unique': False, 'index': False, 'index_method': IndexMethod.btree, 'choices': None, 'db_column_name': None})", # noqa + "manager.add_column(table_class_name='Band', tablename='band', column_name='name', db_column_name='name', column_class_name='Varchar', column_class=Varchar, params={'length': 255, 'default': '', 'null': False, 'primary_key': False, 'unique': False, 'index': False, 'index_method': IndexMethod.btree, 'choices': None, 'db_column_name': None, 'secret': False})", # noqa ) def test_drop_table(self): @@ -122,7 +122,7 @@ def test_add_column(self): self.assertTrue(len(schema_differ.add_columns.statements) == 1) self.assertEqual( schema_differ.add_columns.statements[0], - "manager.add_column(table_class_name='Band', tablename='band', column_name='genre', db_column_name='genre', column_class_name='Varchar', column_class=Varchar, params={'length': 255, 'default': '', 'null': False, 'primary_key': False, 'unique': False, 'index': False, 'index_method': IndexMethod.btree, 'choices': None, 'db_column_name': None})", # noqa + "manager.add_column(table_class_name='Band', tablename='band', column_name='genre', db_column_name='genre', column_class_name='Varchar', column_class=Varchar, params={'length': 255, 'default': '', 'null': False, 'primary_key': False, 'unique': False, 'index': False, 'index_method': IndexMethod.btree, 'choices': None, 'db_column_name': None, 'secret': False})", # noqa ) def test_drop_column(self): diff --git a/tests/apps/migrations/auto/test_serialisation.py b/tests/apps/migrations/auto/test_serialisation.py index 8b4c46d9f..1b813ba66 100644 --- a/tests/apps/migrations/auto/test_serialisation.py +++ b/tests/apps/migrations/auto/test_serialisation.py @@ -247,7 +247,7 @@ def test_lazy_table_reference(self): 'class Manager(Table, tablename="manager"): ' "id = Serial(null=False, primary_key=True, unique=False, " "index=False, index_method=IndexMethod.btree, " - "choices=None, db_column_name='id')" + "choices=None, db_column_name='id', secret=False)" ), ) @@ -298,7 +298,7 @@ def test_column_instance(self): self.assertEqual( serialised.params["base_column"].__repr__(), - "Varchar(length=255, default='', null=False, primary_key=False, unique=False, index=False, index_method=IndexMethod.btree, choices=None, db_column_name=None)", # noqa: E501 + "Varchar(length=255, default='', null=False, primary_key=False, unique=False, index=False, index_method=IndexMethod.btree, choices=None, db_column_name=None, secret=False)", # noqa: E501 ) self.assertEqual( diff --git a/tests/apps/migrations/commands/test_forwards_backwards.py b/tests/apps/migrations/commands/test_forwards_backwards.py index 701bf5916..3bfe7ebb6 100644 --- a/tests/apps/migrations/commands/test_forwards_backwards.py +++ b/tests/apps/migrations/commands/test_forwards_backwards.py @@ -176,6 +176,7 @@ def test_forwards_fake(self): "2020-12-17T18:44:44", "2021-07-25T22:38:48:009306", "2021-09-06T13:58:23:024723", + "2021-11-13T14:01:46:114725", ], ) diff --git a/tests/columns/test_base.py b/tests/columns/test_base.py index 091e1b764..cef974c0c 100644 --- a/tests/columns/test_base.py +++ b/tests/columns/test_base.py @@ -34,6 +34,16 @@ def test_help_text(self): self.assertTrue(column._meta.help_text == help_text) +class TestSecretParameter(TestCase): + def test_secret_parameter(self): + """ + Test adding secret parameter to a column. + """ + secret = False + column = Varchar(secret=secret) + self.assertTrue(column._meta.secret == secret) + + class TestChoices(TestCase): def test_choices(self): """ diff --git a/tests/example_apps/music/piccolo_migrations/2021-11-13T14-01-46-114725.py b/tests/example_apps/music/piccolo_migrations/2021-11-13T14-01-46-114725.py new file mode 100644 index 000000000..e7db0f3a9 --- /dev/null +++ b/tests/example_apps/music/piccolo_migrations/2021-11-13T14-01-46-114725.py @@ -0,0 +1,24 @@ +from piccolo.apps.migrations.auto import MigrationManager +from piccolo.columns.column_types import Integer + +ID = "2021-11-13T14:01:46:114725" +VERSION = "0.59.0" +DESCRIPTION = "" + + +async def forwards(): + manager = MigrationManager( + migration_id=ID, app_name="music", description=DESCRIPTION + ) + + manager.alter_column( + table_class_name="Venue", + tablename="venue", + column_name="capacity", + params={"secret": True}, + old_params={"secret": False}, + column_class=Integer, + old_column_class=Integer, + ) + + return manager diff --git a/tests/example_apps/music/tables.py b/tests/example_apps/music/tables.py index 331fab081..774718cae 100644 --- a/tests/example_apps/music/tables.py +++ b/tests/example_apps/music/tables.py @@ -36,7 +36,7 @@ class Band(Table): class Venue(Table): name = Varchar(length=100) - capacity = Integer(default=0) + capacity = Integer(default=0, secret=True) class Concert(Table): diff --git a/tests/table/test_select.py b/tests/table/test_select.py index 92ac15ab2..5d7d774e0 100644 --- a/tests/table/test_select.py +++ b/tests/table/test_select.py @@ -3,7 +3,7 @@ from piccolo.apps.user.tables import BaseUser from piccolo.columns.combination import WhereRaw from piccolo.query.methods.select import Avg, Count, Max, Min, Sum -from tests.example_apps.music.tables import Band, Concert, Manager +from tests.example_apps.music.tables import Band, Concert, Manager, Venue from ..base import DBTestCase, postgres_only, sqlite_only @@ -965,3 +965,23 @@ def test_secret(self): user_dict = BaseUser.select(exclude_secrets=True).first().run_sync() self.assertTrue("password" not in user_dict.keys()) + + +class TestSelectSecretParameter(TestCase): + def setUp(self): + Venue.create_table().run_sync() + + def tearDown(self): + Venue.alter().drop_table().run_sync() + + def test_secret_parameter(self): + """ + Make sure that fields with parameter ``secret=True`` are omitted + from the response when requested. + """ + venue = Venue(name="The Garage", capacity=1000) + venue.save().run_sync() + + venue_dict = Venue.select(exclude_secrets=True).first().run_sync() + self.assertTrue(venue_dict, {"id": 1, "name": "The Garage"}) + self.assertTrue("capacity" not in venue_dict.keys()) diff --git a/tests/table/test_str.py b/tests/table/test_str.py index f7f296f20..ac3861700 100644 --- a/tests/table/test_str.py +++ b/tests/table/test_str.py @@ -9,8 +9,8 @@ def test_str(self): Manager._table_str(), ( "class Manager(Table, tablename='manager'):\n" - " id = Serial(null=False, primary_key=True, unique=False, index=False, index_method=IndexMethod.btree, choices=None, db_column_name='id')\n" # noqa: E501 - " name = Varchar(length=50, default='', null=False, primary_key=False, unique=False, index=False, index_method=IndexMethod.btree, choices=None, db_column_name=None)\n" # noqa: E501 + " id = Serial(null=False, primary_key=True, unique=False, index=False, index_method=IndexMethod.btree, choices=None, db_column_name='id', secret=False)\n" # noqa: E501 + " name = Varchar(length=50, default='', null=False, primary_key=False, unique=False, index=False, index_method=IndexMethod.btree, choices=None, db_column_name=None, secret=False)\n" # noqa: E501 ), ) From 925eeca1b78842772d5785388fe21869caf1c956 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 14 Nov 2021 15:09:12 +0000 Subject: [PATCH 150/727] add language to code blocks (#348) --- piccolo/columns/base.py | 10 ++++++---- piccolo/table.py | 16 +++++++++------- piccolo/utils/dictionary.py | 2 +- piccolo/utils/objects.py | 2 +- tests/columns/test_db_column_name.py | 2 +- 5 files changed, 18 insertions(+), 14 deletions(-) diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index c85961d38..b36c7c9d8 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -557,14 +557,14 @@ def __hash__(self): def is_null(self) -> Where: """ - Can be used instead of `MyTable.column != None`, because some linters + Can be used instead of ``MyTable.column != None``, because some linters don't like a comparison to None. """ return Where(column=self, operator=IsNull) def is_not_null(self) -> Where: """ - Can be used instead of `MyTable.column == None`, because some linters + Can be used instead of ``MyTable.column == None``, because some linters don't like a comparison to None. """ return Where(column=self, operator=IsNotNull) @@ -575,8 +575,10 @@ def as_alias(self, name: str) -> Column: For example: - >>> await Band.select(Band.name.as_alias('title')).run() - {'title': 'Pythonistas'} + .. code-block:: python + + >>> await Band.select(Band.name.as_alias('title')).run() + {'title': 'Pythonistas'} """ column = copy.deepcopy(self) diff --git a/piccolo/table.py b/piccolo/table.py index 25271dd3a..e27adfd06 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -341,13 +341,15 @@ def get_related(self, foreign_key: t.Union[ForeignKey, str]) -> Objects: """ Used to fetch a Table instance, for the target of a foreign key. - band = await Band.objects().first().run() - manager = await band.get_related(Band.manager).run() - >>> print(manager.name) - 'Guido' + .. code-block:: python + + band = await Band.objects().first().run() + manager = await band.get_related(Band.manager).run() + >>> print(manager.name) + 'Guido' It can only follow foreign keys one level currently. - i.e. Band.manager, but not Band.manager.x.y.z + i.e. ``Band.manager``, but not ``Band.manager.x.y.z``. """ if isinstance(foreign_key, str): @@ -383,7 +385,7 @@ def to_dict(self, *columns: Column) -> t.Dict[str, t.Any]: A convenience method which returns a dictionary, mapping column names to values for this table instance. - .. code-block:: + .. code-block:: python instance = await Manager.objects().get( Manager.name == 'Guido' @@ -395,7 +397,7 @@ def to_dict(self, *columns: Column) -> t.Dict[str, t.Any]: If the columns argument is provided, only those columns are included in the output. It also works with column aliases. - .. code-block:: + .. code-block:: python >>> instance.to_dict(Manager.id, Manager.name.as_alias('title')) {'id': 1, 'title': 'Guido'} diff --git a/piccolo/utils/dictionary.py b/piccolo/utils/dictionary.py index 6259e395e..e5b26aed8 100644 --- a/piccolo/utils/dictionary.py +++ b/piccolo/utils/dictionary.py @@ -10,7 +10,7 @@ def make_nested(dictionary: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]: This function puts any values from a related table into a sub dictionary. - .. code-block:: + .. code-block:: python response = Band.select(Band.name, Band.manager.name).run_sync() >>> print(response) diff --git a/piccolo/utils/objects.py b/piccolo/utils/objects.py index 312bcd362..1faf5b41b 100644 --- a/piccolo/utils/objects.py +++ b/piccolo/utils/objects.py @@ -27,7 +27,7 @@ def make_nested_object( For example: - .. code-block:: + .. code-block:: python band = make_nested(row, Band) >>> band diff --git a/tests/columns/test_db_column_name.py b/tests/columns/test_db_column_name.py index cf1bceaf7..1355186c6 100644 --- a/tests/columns/test_db_column_name.py +++ b/tests/columns/test_db_column_name.py @@ -13,7 +13,7 @@ class TestDBColumnName(DBTestCase): By using the ``db_column_name`` arg, the user can map a ``Column`` to a database column with a different name. For example: - .. code-block:: + .. code-block:: python class MyTable(Table): class_ = Varchar(db_column_name='class') From 019cf6c4edad1e71471391d3fd8cf5607552ed47 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 14 Nov 2021 20:55:55 +0000 Subject: [PATCH 151/727] added `Column._equals` method (#349) * added `Column._equals` method * added an extra test * add more tests --- piccolo/columns/base.py | 58 ++++++++++++++++++++++++++++++++++++++ tests/columns/test_base.py | 33 ++++++++++++++++++++++ 2 files changed, 91 insertions(+) diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index b36c7c9d8..bed24f9da 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -540,6 +540,64 @@ def __gt__(self, value) -> Where: def __ge__(self, value) -> Where: return Where(column=self, value=value, operator=GreaterEqualThan) + def _equals(self, column: Column, including_joins: bool = False) -> bool: + """ + We override ``__eq__``, in order to do queries such as: + + .. code-block:: python + + Band.select().where(Band.name == 'Pythonistas').run_sync() + + But this means that comparisons such as this can give unexpected + results: + + .. code-block:: python + + # We would expect the answer to be `True`, but we get `Where` + # instead: + >>> MyTable.some_column == MyTable.some_column + + + Also, column comparison is sometimes more complex than it appears. This + is why we have this custom method for comparing columns. + + Take this example: + + .. code-block:: python + + Band.manager.name == Manager.name + + They both refer to the ``name`` column on the ``Manager`` table, except + one has joins and the other doesn't. + + :param including_joins: + If ``True``, then we check if the columns are the same, as well as + their joins, i.e. ``Band.manager.name`` != ``Manager.name``. + + """ + if isinstance(column, Column): + if ( + self._meta.name == column._meta.name + and self._meta.table._meta.tablename + == column._meta.table._meta.tablename + ): + if including_joins: + if len(column._meta.call_chain) == len( + self._meta.call_chain + ): + return all( + column_a._equals(column_b, including_joins=False) + for column_a, column_b in zip( + column._meta.call_chain, + self._meta.call_chain, + ) + ) + + else: + return True + + return False + def __eq__(self, value) -> Where: # type: ignore if value is None: return Where(column=self, operator=IsNull) diff --git a/tests/columns/test_base.py b/tests/columns/test_base.py index cef974c0c..309e558b8 100644 --- a/tests/columns/test_base.py +++ b/tests/columns/test_base.py @@ -4,6 +4,7 @@ from piccolo.columns.choices import Choice from piccolo.columns.column_types import Integer, Varchar from piccolo.table import Table +from tests.example_apps.music.tables import Band, Manager class MyTable(Table): @@ -112,3 +113,35 @@ class Title(Enum): "mrs": {"display_name": "Mrs.", "value": 2}, }, ) + + +class TestEquals(TestCase): + def test_non_column(self): + """ + Make sure non-column values don't match. + """ + for value in (1, "abc", None): + self.assertFalse(Manager.name._equals(value)) + + def test_equals(self): + """ + Test basic usage. + """ + self.assertTrue(Manager.name._equals(Manager.name)) + + def test_same_name(self): + """ + Make sure that columns with the same name, but on different tables, + don't match. + """ + self.assertFalse(Manager.name._equals(Band.name)) + + def test_including_joins(self): + """ + Make sure `including_joins` arg works correctly. + """ + self.assertTrue(Band.manager.name._equals(Manager.name)) + + self.assertFalse( + Band.manager.name._equals(Manager.name, including_joins=True) + ) From 2f27337d102f3a1c6dafe55f8deacaac6ebe3fbc Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 15 Nov 2021 08:29:38 +0000 Subject: [PATCH 152/727] `create_pydantic_model` can now accept a tuple for the `nested` arg (#350) * `create_pydantic_model` can now accept a tuple for the `nested` arg * fix typo in test * more tests --- piccolo/utils/pydantic.py | 18 ++++- tests/utils/test_pydantic.py | 146 ++++++++++++++++++++++++++++------- 2 files changed, 133 insertions(+), 31 deletions(-) diff --git a/piccolo/utils/pydantic.py b/piccolo/utils/pydantic.py index c28ec662a..d204ba8ee 100644 --- a/piccolo/utils/pydantic.py +++ b/piccolo/utils/pydantic.py @@ -1,5 +1,6 @@ from __future__ import annotations +import itertools import json import typing as t import uuid @@ -76,7 +77,7 @@ def validate_columns( @lru_cache() def create_pydantic_model( table: t.Type[Table], - nested: bool = False, + nested: t.Union[bool, t.Tuple[ForeignKey, ...]] = False, exclude_columns: t.Tuple[Column, ...] = (), include_columns: t.Tuple[Column, ...] = (), include_default_columns: bool = False, @@ -94,6 +95,9 @@ def create_pydantic_model( for. :param nested: Whether ``ForeignKey`` columns are converted to nested Pydantic models. + If ``False``, none are converted. If ``True``, they all are converted. + If a tuple of ``ForeignKey`` columns is passed in, then only those are + converted. :param exclude_columns: A tuple of ``Column`` instances that should be excluded from the Pydantic model. Only specify ``include_column`` or ``exclude_column``. @@ -221,10 +225,18 @@ def create_pydantic_model( } if isinstance(column, ForeignKey): - if nested: + if (nested is True) or ( + isinstance(nested, tuple) + and any( + column._equals(i) + for i in itertools.chain( + nested, *[i._meta.call_chain for i in nested] + ) + ) + ): _type = create_pydantic_model( table=column._foreign_key_meta.resolved_references, - nested=True, + nested=nested, include_default_columns=include_default_columns, include_readable=include_readable, all_optional=all_optional, diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index 9ba00eb9f..3552eb7a3 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -353,37 +353,128 @@ class Band(Table): class TestNestedModel(TestCase): - def test_nested_models(self): + def test_true(self): + """ + Make sure all foreign key columns are convered to nested models, when + `nested=True`. + """ + class Country(Table): name = Varchar(length=10) - class Director(Table): + class Manager(Table): name = Varchar(length=10) country = ForeignKey(Country) - class Movie(Table): + class Band(Table): name = Varchar(length=10) - director = ForeignKey(Director) + manager = ForeignKey(Manager) + + BandModel = create_pydantic_model(table=Band, nested=True) + + ####################################################################### + + ManagerModel = BandModel.__fields__["manager"].type_ + self.assertTrue(issubclass(ManagerModel, pydantic.BaseModel)) + self.assertEqual( + [i for i in ManagerModel.__fields__.keys()], ["name", "country"] + ) + + ####################################################################### + + CountryModel = ManagerModel.__fields__["country"].type_ + self.assertTrue(issubclass(CountryModel, pydantic.BaseModel)) + self.assertEqual([i for i in CountryModel.__fields__.keys()], ["name"]) + + def test_tuple(self): + """ + Make sure only the specified foreign key columns are converted to + nested models. + """ + + class Country(Table): + name = Varchar() + + class Manager(Table): + name = Varchar() + country = ForeignKey(Country) + + class Band(Table): + name = Varchar() + manager = ForeignKey(Manager) + assistant_manager = ForeignKey(Manager) + + class Venue(Table): + name = Varchar() - MovieModel = create_pydantic_model(table=Movie, nested=True) + class Concert(Table): + band_1 = ForeignKey(Band) + band_2 = ForeignKey(Band) + venue = ForeignKey(Venue) ####################################################################### + # Test one level deep - DirectorModel = MovieModel.__fields__["director"].type_ + BandModel = create_pydantic_model(table=Band, nested=(Band.manager,)) - self.assertTrue(issubclass(DirectorModel, pydantic.BaseModel)) + ManagerModel = BandModel.__fields__["manager"].type_ + self.assertTrue(issubclass(ManagerModel, pydantic.BaseModel)) + self.assertEqual( + [i for i in ManagerModel.__fields__.keys()], ["name", "country"] + ) - director_model_keys = [i for i in DirectorModel.__fields__.keys()] - self.assertEqual(director_model_keys, ["name", "country"]) + AssistantManagerType = BandModel.__fields__["assistant_manager"].type_ + self.assertTrue(AssistantManagerType is int) ####################################################################### + # Test two levels deep - CountryModel = DirectorModel.__fields__["country"].type_ + BandModel = create_pydantic_model( + table=Band, nested=(Band.manager.country,) + ) + ManagerModel = BandModel.__fields__["manager"].type_ + self.assertTrue(issubclass(ManagerModel, pydantic.BaseModel)) + self.assertEqual( + [i for i in ManagerModel.__fields__.keys()], ["name", "country"] + ) + + AssistantManagerType = BandModel.__fields__["assistant_manager"].type_ + self.assertTrue(AssistantManagerType is int) + + CountryModel = ManagerModel.__fields__["country"].type_ self.assertTrue(issubclass(CountryModel, pydantic.BaseModel)) + self.assertEqual([i for i in CountryModel.__fields__.keys()], ["name"]) - country_model_keys = [i for i in CountryModel.__fields__.keys()] - self.assertEqual(country_model_keys, ["name"]) + ####################################################################### + # Test three levels deep + + ConcertModel = create_pydantic_model( + Concert, nested=(Concert.band_1.manager,) + ) + + VenueModel = ConcertModel.__fields__["venue"].type_ + self.assertTrue(VenueModel is int) + + BandModel = ConcertModel.__fields__["band_1"].type_ + self.assertTrue(issubclass(BandModel, pydantic.BaseModel)) + self.assertEqual( + [i for i in BandModel.__fields__.keys()], + ["name", "manager", "assistant_manager"], + ) + + ManagerModel = BandModel.__fields__["manager"].type_ + self.assertTrue(issubclass(ManagerModel, pydantic.BaseModel)) + self.assertEqual( + [i for i in ManagerModel.__fields__.keys()], + ["name", "country"], + ) + + AssistantManagerType = BandModel.__fields__["assistant_manager"].type_ + self.assertTrue(AssistantManagerType is int) + + CountryModel = ManagerModel.__fields__["country"].type_ + self.assertTrue(CountryModel is int) def test_cascaded_args(self): """ @@ -394,35 +485,34 @@ def test_cascaded_args(self): class Country(Table): name = Varchar(length=10) - class Director(Table): + class Manager(Table): name = Varchar(length=10) country = ForeignKey(Country) - class Movie(Table): + class Band(Table): name = Varchar(length=10) - director = ForeignKey(Director) + manager = ForeignKey(Manager) - MovieModel = create_pydantic_model( - table=Movie, nested=True, include_default_columns=True + BandModel = create_pydantic_model( + table=Band, nested=True, include_default_columns=True ) ####################################################################### - DirectorModel = MovieModel.__fields__["director"].type_ - - self.assertTrue(issubclass(DirectorModel, pydantic.BaseModel)) - - director_model_keys = [i for i in DirectorModel.__fields__.keys()] - self.assertEqual(director_model_keys, ["id", "name", "country"]) + ManagerModel = BandModel.__fields__["manager"].type_ + self.assertTrue(issubclass(ManagerModel, pydantic.BaseModel)) + self.assertEqual( + [i for i in ManagerModel.__fields__.keys()], + ["id", "name", "country"], + ) ####################################################################### - CountryModel = DirectorModel.__fields__["country"].type_ - + CountryModel = ManagerModel.__fields__["country"].type_ self.assertTrue(issubclass(CountryModel, pydantic.BaseModel)) - - country_model_keys = [i for i in CountryModel.__fields__.keys()] - self.assertEqual(country_model_keys, ["id", "name"]) + self.assertEqual( + [i for i in CountryModel.__fields__.keys()], ["id", "name"] + ) class TestDBColumnName(TestCase): From 95d52e5f417609e32e805cd1b9d45a9214848c96 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 15 Nov 2021 10:26:03 +0000 Subject: [PATCH 153/727] added max recursion depth, and can include/exclude columns from related tables (#351) --- piccolo/utils/pydantic.py | 97 ++++++++++++++++++++++-------------- tests/utils/test_pydantic.py | 94 ++++++++++++++++++++++++++++------ 2 files changed, 139 insertions(+), 52 deletions(-) diff --git a/piccolo/utils/pydantic.py b/piccolo/utils/pydantic.py index d204ba8ee..b5ddda329 100644 --- a/piccolo/utils/pydantic.py +++ b/piccolo/utils/pydantic.py @@ -85,6 +85,8 @@ def create_pydantic_model( all_optional: bool = False, model_name: t.Optional[str] = None, deserialize_json: bool = False, + recursion_depth: int = 0, + max_recursion_depth: int = 5, **schema_extra_kwargs, ) -> t.Type[pydantic.BaseModel]: """ @@ -121,6 +123,10 @@ def create_pydantic_model( By default, the values of any Piccolo ``JSON`` or ``JSONB`` columns are returned as strings. By setting this parameter to True, they will be returned as objects. + :recursion_depth: + Not to be set by the user - used internally to track recursion. + :max_recursion_depth: + If using nested models, this specifies the max amount of recursion. :param schema_extra_kwargs: This can be used to add additional fields to the schema. This is very useful when using Pydantic's JSON Schema features. For example: @@ -141,51 +147,61 @@ def create_pydantic_model( "same time." ) - if exclude_columns: - if not validate_columns(columns=exclude_columns, table=table): - raise ValueError( - f"`exclude_columns` are invalid: ({exclude_columns!r})" - ) + if recursion_depth == 0: + if exclude_columns: + if not validate_columns(columns=exclude_columns, table=table): + raise ValueError( + f"`exclude_columns` are invalid: {exclude_columns!r}" + ) - if include_columns: - if not validate_columns(columns=include_columns, table=table): - raise ValueError( - f"`include_columns` are invalid: ({include_columns!r})" - ) + if include_columns: + if not validate_columns(columns=include_columns, table=table): + raise ValueError( + f"`include_columns` are invalid: {include_columns!r}" + ) ########################################################################### columns: t.Dict[str, t.Any] = {} validators: t.Dict[str, classmethod] = {} - piccolo_columns = ( - include_columns - if include_columns - else tuple( - table._meta.columns - if include_default_columns - else table._meta.non_default_columns - ) + + piccolo_columns = tuple( + table._meta.columns + if include_default_columns + else table._meta.non_default_columns ) - for column in piccolo_columns: - # normal __contains__ checks __eq__ as well which returns ``Where`` - # instance which always evaluates to ``True`` - if exclude_columns and any(column is obj for obj in exclude_columns): - continue - - column_name = ( - ".".join( - [i._meta.name for i in column._meta.call_chain] - + [column._meta.name] + if include_columns: + include_columns_plus_ancestors = [ + i + for i in itertools.chain( + include_columns, *[i._meta.call_chain for i in include_columns] + ) + ] + piccolo_columns = tuple( + i + for i in piccolo_columns + if any( + i._equals(include_column) + for include_column in include_columns_plus_ancestors ) - if column._meta.call_chain - else column._meta.name ) + if exclude_columns: + piccolo_columns = tuple( + i + for i in piccolo_columns + if not any( + i._equals(exclude_column) for exclude_column in exclude_columns + ) + ) + + for column in piccolo_columns: + column_name = column._meta.name + is_optional = True if all_optional else not column._meta.required ####################################################################### - # Work out the column type if isinstance(column, (Decimal, Numeric)): @@ -225,22 +241,29 @@ def create_pydantic_model( } if isinstance(column, ForeignKey): - if (nested is True) or ( - isinstance(nested, tuple) - and any( - column._equals(i) - for i in itertools.chain( - nested, *[i._meta.call_chain for i in nested] + if recursion_depth < max_recursion_depth and ( + (nested is True) + or ( + isinstance(nested, tuple) + and any( + column._equals(i) + for i in itertools.chain( + nested, *[i._meta.call_chain for i in nested] + ) ) ) ): _type = create_pydantic_model( table=column._foreign_key_meta.resolved_references, nested=nested, + include_columns=include_columns, + exclude_columns=exclude_columns, include_default_columns=include_default_columns, include_readable=include_readable, all_optional=all_optional, deserialize_json=deserialize_json, + recursion_depth=recursion_depth + 1, + max_recursion_depth=max_recursion_depth, ) tablename = ( diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index 3552eb7a3..ab23187e7 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -206,7 +206,7 @@ class Movie(Table): ) -class TestExcludeColumn(TestCase): +class TestExcludeColumns(TestCase): def test_all(self): class Computer(Table): CPU = Varchar() @@ -289,6 +289,33 @@ class Computer2(Table): with self.assertRaises(ValueError): create_pydantic_model(Computer, exclude_columns=(Computer2.CPU,)) + def test_exclude_nested(self): + class Manager(Table): + name = Varchar() + phone_number = Integer() + + class Band(Table): + name = Varchar() + manager = ForeignKey(Manager) + popularity = Integer() + + pydantic_model = create_pydantic_model( + table=Band, + exclude_columns=( + Band.popularity, + Band.manager.phone_number, + ), + nested=(Band.manager,), + ) + + model_instance = pydantic_model( + name="Pythonistas", manager={"name": "Guido"} + ) + self.assertEqual( + model_instance.dict(), + {"name": "Pythonistas", "manager": {"name": "Guido"}}, + ) + class TestIncludeColumns(TestCase): def test_include(self): @@ -322,40 +349,42 @@ class Band(Table): include_columns=(Band.name,), ) - def test_join(self): + def test_nested(self): """ Make sure that columns on related tables work. """ class Manager(Table): name = Varchar() + phone_number = Integer() class Band(Table): name = Varchar() manager = ForeignKey(Manager) + popularity = Integer() pydantic_model = create_pydantic_model( - table=Band, include_columns=(Band.name, Band.manager.name) + table=Band, + include_columns=( + Band.name, + Band.manager.name, + ), + nested=(Band.manager,), ) - self.assertIsNotNone(pydantic_model.__fields__.get("manager.name")) - - # Make sure it can be instantiated: model_instance = pydantic_model( - **{"name": "Pythonistas", "manager.name": "Guido"} + name="Pythonistas", manager={"name": "Guido"} ) - self.assertEqual(getattr(model_instance, "name"), "Pythonistas") - self.assertEqual(getattr(model_instance, "manager.name"), "Guido") self.assertEqual( model_instance.dict(), - {"name": "Pythonistas", "manager.name": "Guido"}, + {"name": "Pythonistas", "manager": {"name": "Guido"}}, ) class TestNestedModel(TestCase): def test_true(self): """ - Make sure all foreign key columns are convered to nested models, when + Make sure all foreign key columns are converted to nested models, when `nested=True`. """ @@ -497,8 +526,6 @@ class Band(Table): table=Band, nested=True, include_default_columns=True ) - ####################################################################### - ManagerModel = BandModel.__fields__["manager"].type_ self.assertTrue(issubclass(ManagerModel, pydantic.BaseModel)) self.assertEqual( @@ -506,8 +533,6 @@ class Band(Table): ["id", "name", "country"], ) - ####################################################################### - CountryModel = ManagerModel.__fields__["country"].type_ self.assertTrue(issubclass(CountryModel, pydantic.BaseModel)) self.assertEqual( @@ -515,6 +540,45 @@ class Band(Table): ) +class TestRecursionDepth(TestCase): + def test_max(self): + class Country(Table): + name = Varchar() + + class Manager(Table): + name = Varchar() + country = ForeignKey(Country) + + class Band(Table): + name = Varchar() + manager = ForeignKey(Manager) + assistant_manager = ForeignKey(Manager) + + class Venue(Table): + name = Varchar() + + class Concert(Table): + band = ForeignKey(Band) + venue = ForeignKey(Venue) + + ConcertModel = create_pydantic_model( + table=Concert, nested=True, max_recursion_depth=2 + ) + + VenueModel = ConcertModel.__fields__["venue"].type_ + self.assertTrue(issubclass(VenueModel, pydantic.BaseModel)) + + BandModel = ConcertModel.__fields__["band"].type_ + self.assertTrue(issubclass(BandModel, pydantic.BaseModel)) + + ManagerModel = BandModel.__fields__["manager"].type_ + self.assertTrue(issubclass(ManagerModel, pydantic.BaseModel)) + + # We should have hit the recursion depth: + CountryModel = ManagerModel.__fields__["country"].type_ + self.assertTrue(CountryModel is int) + + class TestDBColumnName(TestCase): def test_db_column_name(self): """ From 4fe12a898daf0f51d27527ee737ec50a80f194a4 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 15 Nov 2021 10:43:37 +0000 Subject: [PATCH 154/727] fix bug - add type check for column (#352) --- piccolo/query/mixins.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index 338dc5620..d7d235d88 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -287,9 +287,12 @@ def columns(self, *columns: t.Union[Selectable, t.List[Selectable]]): self.selected_columns = combined def remove_secret_columns(self): - self.selected_columns = [ - i for i in self.selected_columns if not i._meta.secret - ] + non_secret = [] + for i in self.selected_columns: + if isinstance(i, Column) and i._meta.secret: + continue + non_secret.append(i) + self.selected_columns = non_secret @dataclass From c569d1419e2c197c3954d457501055a2d83a5287 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 15 Nov 2021 11:06:17 +0000 Subject: [PATCH 155/727] bumped version --- CHANGES.rst | 69 +++++++++++++++++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 362079a14..4c09401f9 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,75 @@ Changes ======= +0.60.0 +------ + +Secret columns +~~~~~~~~~~~~~~ + +All column types can now be secret, rather than being limited to the +``Secret`` column type which is a ``Varchar`` under the hood (courtesy +@sinisaos). + +.. code-block:: python + + class Manager(Table): + name = Varchar() + net_worth = Integer(secret=True) + +The reason this is useful is you can do queries such as: + +.. code-block:: python + + >>> Manager.select(exclude_secrets=True).run_sync() + [{'id': 1, 'name': 'Guido'}] + +In the Piccolo API project we have ``PiccoloCRUD`` which is an incredibly +powerful way of building an API with very little code. ``PiccoloCRUD`` has an +``exclude_secrets`` option which lets you safely expose your data without +leaking sensitive information. + +Pydantic improvements +~~~~~~~~~~~~~~~~~~~~~ + +max_recursion_depth +******************* + +``create_pydantic_model`` now has a ``max_recursion_depth`` argument, which is +useful when using ``nested=True`` on large database schemas. + +.. code-block:: python + + >>> create_pydantic_model(MyTable, nested=True, max_recursion_depth=3) + +Nested tuple +************ + +You can now pass a tuple of columns as the argument to ``nested``: + +.. code-block:: python + + >>> create_pydantic_model(Band, nested=(Band.manager,)) + +This gives you more control than just using ``nested=True``. + +include_columns / exclude_columns +********************************* + +You can now include / exclude columns from related tables. For example: + +.. code-block:: python + + >>> create_pydantic_model(Band, nested=(Band.manager,), exclude_columns=(Band.manager.country)) + +Similarly: + +.. code-block:: python + + >>> create_pydantic_model(Band, nested=(Band.manager,), include_columns=(Band.name, Band.manager.name)) + +------------------------------------------------------------------------------- + 0.59.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 94a33d464..084ceb867 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.59.0" +__VERSION__ = "0.60.0" From a2e56ef2ec7fe7ffbabb3e4a52cb60741bcfdfc1 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 24 Nov 2021 23:00:54 +0000 Subject: [PATCH 156/727] fix import path for MigrationManager (#357) --- piccolo/apps/migrations/commands/templates/migration.py.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piccolo/apps/migrations/commands/templates/migration.py.jinja b/piccolo/apps/migrations/commands/templates/migration.py.jinja index 90331d9b4..70cb3b793 100644 --- a/piccolo/apps/migrations/commands/templates/migration.py.jinja +++ b/piccolo/apps/migrations/commands/templates/migration.py.jinja @@ -1,4 +1,4 @@ -from piccolo.apps.migrations.auto import MigrationManager +from piccolo.apps.migrations.auto.migration_manager import MigrationManager {% for extra_import in extra_imports -%} {{ extra_import }} {% endfor %} From ac265ad8c92bcc9f65c17bc5809df6640a5afe3f Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 24 Nov 2021 23:36:23 +0000 Subject: [PATCH 157/727] modify remaining MigrationManager imports --- docs/src/piccolo/migrations/create.rst | 2 +- piccolo/apps/migrations/commands/backwards.py | 2 +- piccolo/apps/migrations/commands/forwards.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/src/piccolo/migrations/create.rst b/docs/src/piccolo/migrations/create.rst index 75329001f..74119873f 100644 --- a/docs/src/piccolo/migrations/create.rst +++ b/docs/src/piccolo/migrations/create.rst @@ -23,7 +23,7 @@ The contents of an empty migration file looks like this: .. code-block:: python - from piccolo.apps.migrations.auto import MigrationManager + from piccolo.apps.migrations.auto.migration_manager import MigrationManager ID = '2021-08-06T16:22:51:415781' diff --git a/piccolo/apps/migrations/commands/backwards.py b/piccolo/apps/migrations/commands/backwards.py index df29486b9..e8ca28c08 100644 --- a/piccolo/apps/migrations/commands/backwards.py +++ b/piccolo/apps/migrations/commands/backwards.py @@ -3,7 +3,7 @@ import os import sys -from piccolo.apps.migrations.auto import MigrationManager +from piccolo.apps.migrations.auto.migration_manager import MigrationManager from piccolo.apps.migrations.commands.base import ( BaseMigrationManager, MigrationResult, diff --git a/piccolo/apps/migrations/commands/forwards.py b/piccolo/apps/migrations/commands/forwards.py index 99929ea67..19ee6366e 100644 --- a/piccolo/apps/migrations/commands/forwards.py +++ b/piccolo/apps/migrations/commands/forwards.py @@ -3,7 +3,7 @@ import sys import typing as t -from piccolo.apps.migrations.auto import MigrationManager +from piccolo.apps.migrations.auto.migration_manager import MigrationManager from piccolo.apps.migrations.commands.base import ( BaseMigrationManager, MigrationResult, From 535fca7d20a84673f4f8985221773d2276321215 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 24 Nov 2021 23:44:36 +0000 Subject: [PATCH 158/727] bumped version --- CHANGES.rst | 9 +++++++++ piccolo/__init__.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 4c09401f9..c0604e0cb 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,15 @@ Changes ======= +0.60.1 +------ + +Modified the import path for ``MigrationManager`` in migration files. It was +confusing Pylance (VSCode's type checker). Thanks to @gmos for reporting and +investigating this issue. + +------------------------------------------------------------------------------- + 0.60.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 084ceb867..9964c39eb 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.60.0" +__VERSION__ = "0.60.1" From bcbc257a7067a277a22a06bb48d73770e520d9cb Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 25 Nov 2021 12:13:11 +0000 Subject: [PATCH 159/727] asyncio.gather wasn't working with some queries (#359) --- piccolo/query/methods/exists.py | 2 -- piccolo/query/methods/insert.py | 2 -- piccolo/query/methods/objects.py | 1 - piccolo/query/methods/raw.py | 2 -- piccolo/query/methods/update.py | 2 -- tests/query/test_await.py | 3 +-- tests/query/test_gather.py | 24 ++++++++++++++++++++++++ 7 files changed, 25 insertions(+), 11 deletions(-) create mode 100644 tests/query/test_gather.py diff --git a/piccolo/query/methods/exists.py b/piccolo/query/methods/exists.py index 7b5b4b3a7..01e0ddb7e 100644 --- a/piccolo/query/methods/exists.py +++ b/piccolo/query/methods/exists.py @@ -1,7 +1,6 @@ from __future__ import annotations import typing as t -from dataclasses import dataclass from piccolo.custom_types import Combinable from piccolo.query.base import Query @@ -13,7 +12,6 @@ from piccolo.table import Table -@dataclass class Exists(Query): __slots__ = ("where_delegate",) diff --git a/piccolo/query/methods/insert.py b/piccolo/query/methods/insert.py index 84e3ad1ee..b1b839630 100644 --- a/piccolo/query/methods/insert.py +++ b/piccolo/query/methods/insert.py @@ -1,7 +1,6 @@ from __future__ import annotations import typing as t -from dataclasses import dataclass from piccolo.query.base import Query from piccolo.query.mixins import AddDelegate @@ -11,7 +10,6 @@ from piccolo.table import Table -@dataclass class Insert(Query): __slots__ = ("add_delegate",) diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index db9469c39..e814da129 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -112,7 +112,6 @@ def run_sync(self): return run_sync(self.run()) -@dataclass class Objects(Query): """ Almost identical to select, except you have to select all fields, and diff --git a/piccolo/query/methods/raw.py b/piccolo/query/methods/raw.py index 2f6176b18..260448485 100644 --- a/piccolo/query/methods/raw.py +++ b/piccolo/query/methods/raw.py @@ -1,7 +1,6 @@ from __future__ import annotations import typing as t -from dataclasses import dataclass from piccolo.query.base import Query from piccolo.querystring import QueryString @@ -10,7 +9,6 @@ from piccolo.table import Table -@dataclass class Raw(Query): __slots__ = ("querystring",) diff --git a/piccolo/query/methods/update.py b/piccolo/query/methods/update.py index db9017e08..129599e39 100644 --- a/piccolo/query/methods/update.py +++ b/piccolo/query/methods/update.py @@ -1,7 +1,6 @@ from __future__ import annotations import typing as t -from dataclasses import dataclass from piccolo.custom_types import Combinable from piccolo.query.base import Query @@ -13,7 +12,6 @@ from piccolo.table import Table -@dataclass class Update(Query): __slots__ = ("values_delegate", "where_delegate") diff --git a/tests/query/test_await.py b/tests/query/test_await.py index fe011b52f..e34abaef7 100644 --- a/tests/query/test_await.py +++ b/tests/query/test_await.py @@ -1,9 +1,8 @@ import asyncio +from tests.base import DBTestCase from tests.example_apps.music.tables import Band -from ..base import DBTestCase - class TestAwait(DBTestCase): def test_await(self): diff --git a/tests/query/test_gather.py b/tests/query/test_gather.py new file mode 100644 index 000000000..548efa9b5 --- /dev/null +++ b/tests/query/test_gather.py @@ -0,0 +1,24 @@ +import asyncio + +from tests.base import DBTestCase +from tests.example_apps.music.tables import Manager + + +class TestAwait(DBTestCase): + def test_await(self): + """ + Make sure that asyncio.gather works with the main query types. + """ + + async def run_queries(): + return await asyncio.gather( + Manager.select(), + Manager.insert(Manager(name="Golangs")), + Manager.delete().where(Manager.name != "Golangs"), + Manager.objects(), + Manager.count(), + Manager.raw("SELECT * FROM manager"), + ) + + # No exceptions should be raised. + self.assertIsInstance(asyncio.run(run_queries()), list) From 0840fe5141c55d93f2dd82d0e3285132e131d8a9 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 25 Nov 2021 12:15:21 +0000 Subject: [PATCH 160/727] bumped version --- CHANGES.rst | 9 +++++++++ piccolo/__init__.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index c0604e0cb..a05557f98 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,15 @@ Changes ======= +0.60.2 +------ + +Fixed a bug with ``asyncio.gather`` not working with some query types. It was +due to them being dataclasses, and they couldn't be hashed properly. Thanks to +@brnosouza for reporting this issue. + +------------------------------------------------------------------------------- + 0.60.1 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 9964c39eb..641c5ca11 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.60.1" +__VERSION__ = "0.60.2" From 2c2053e73991c9b6694fb4385aaed47f3495520d Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 26 Nov 2021 10:30:34 +0000 Subject: [PATCH 161/727] prototype for adding `columns` arg to `save` method (#345) * prototype for adding `columns` arg to `save` method * add a test for saving specific columns * wip * remove changed_columns for now --- docs/src/piccolo/query_types/objects.rst | 5 ++ piccolo/table.py | 41 ++++++++++++--- tests/table/instance/test_save.py | 66 ++++++++++++++++++++++-- 3 files changed, 102 insertions(+), 10 deletions(-) diff --git a/docs/src/piccolo/query_types/objects.rst b/docs/src/piccolo/query_types/objects.rst index 03946d700..851df01a8 100644 --- a/docs/src/piccolo/query_types/objects.rst +++ b/docs/src/piccolo/query_types/objects.rst @@ -77,8 +77,13 @@ Objects have a ``save`` method, which is convenient for updating values: ).first().run_sync() band.popularity = 100000 + + # This saves all values back to the database. band.save().run_sync() + # Or specify specific columns to save: + band.save([Band.popularity]).run_sync() + ------------------------------------------------------------------------------- Deleting objects diff --git a/piccolo/table.py b/piccolo/table.py index e27adfd06..aab5beec4 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -298,24 +298,51 @@ def _create_serial_primary_key(cls) -> Serial: ########################################################################### - def save(self) -> t.Union[Insert, Update]: + def save( + self, columns: t.Optional[t.List[t.Union[Column, str]]] = None + ) -> t.Union[Insert, Update]: """ A proxy to an insert or update query. + + :param columns: + Only the specified columns will be synced back to the database + when doing an update. For example: + + .. code-block:: python + + band = Band.objects().first().run_sync() + band.popularity = 2000 + band.save(columns=[Band.popularity]).run_sync() + + If ``columns=None`` (the default) then all columns will be synced + back to the database. + """ cls = self.__class__ if not self._exists_in_db: return cls.insert().add(self) - # pre-existing row - kwargs: t.Dict[Column, t.Any] = { - i: getattr(self, i._meta.name, None) - for i in cls._meta.columns - if i._meta.name != self._meta.primary_key._meta.name + # Pre-existing row - update + if columns is None: + column_instances = [ + i + for i in cls._meta.columns + if i._meta.name != self._meta.primary_key._meta.name + ] + else: + column_instances = [ + self._meta.get_column_by_name(i) if isinstance(i, str) else i + for i in columns + ] + + values: t.Dict[Column, t.Any] = { + i: getattr(self, i._meta.name, None) for i in column_instances } + return ( cls.update() - .values(kwargs) # type: ignore + .values(values) # type: ignore .where( cls._meta.primary_key == getattr(self, self._meta.primary_key._meta.name) diff --git a/tests/table/instance/test_save.py b/tests/table/instance/test_save.py index ba6bee5d4..e1bc49d1d 100644 --- a/tests/table/instance/test_save.py +++ b/tests/table/instance/test_save.py @@ -1,14 +1,15 @@ from unittest import TestCase -from tests.example_apps.music.tables import Manager +from piccolo.table import create_tables, drop_tables +from tests.example_apps.music.tables import Band, Manager class TestSave(TestCase): def setUp(self): - Manager.create_table().run_sync() + create_tables(Manager, Band) def tearDown(self): - Manager.alter().drop_table().run_sync() + drop_tables(Manager, Band) def test_save_new(self): """ @@ -34,3 +35,62 @@ def test_save_new(self): names = [i["name"] for i in Manager.select(Manager.name).run_sync()] self.assertTrue("Maz2" in names) self.assertTrue("Maz" not in names) + + def test_save_specific_columns(self): + """ + Make sure that we can save a subset of columns. + """ + manager = Manager(name="Guido") + manager.save().run_sync() + + band = Band(name="Pythonistas", popularity=1000, manager=manager) + band.save().run_sync() + + self.assertEqual( + Band.select().run_sync(), + [ + { + "id": 1, + "name": "Pythonistas", + "manager": 1, + "popularity": 1000, + } + ], + ) + + band.name = "Pythonistas 2" + band.popularity = 2000 + band.save(columns=[Band.name]).run_sync() + + # Only the name should update, and not the popularity: + self.assertEqual( + Band.select().run_sync(), + [ + { + "id": 1, + "name": "Pythonistas 2", + "manager": 1, + "popularity": 1000, + } + ], + ) + + ####################################################################### + + # Also test it using strings to identify columns + band.name = "Pythonistas 3" + band.popularity = 3000 + band.save(columns=["popularity"]).run_sync() + + # Only the popularity should update, and not the name: + self.assertEqual( + Band.select().run_sync(), + [ + { + "id": 1, + "name": "Pythonistas 2", + "manager": 1, + "popularity": 3000, + } + ], + ) From 2580d1eabeb2f5a9affa5a60509e8d76e1b35206 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 26 Nov 2021 10:57:18 +0000 Subject: [PATCH 162/727] bumped version --- CHANGES.rst | 19 +++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index a05557f98..5328b8d44 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,25 @@ Changes ======= +0.61.0 +------ + +The ``save`` method now supports a ``columns`` argument, so when updating a +row you can specify which values to sync back. For example: + +.. code-block:: python + + band = await Band.objects().get(Band.name == "Pythonistas") + band.name = "Super Pythonistas" + await band.save([Band.name]) + + # Alternatively, strings are also supported: + await band.save(['name']) + +Thanks to @trondhindenes for suggesting this feature. + +------------------------------------------------------------------------------- + 0.60.2 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 641c5ca11..995c80643 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.60.2" +__VERSION__ = "0.61.0" From 9a3434b9ce9f01cc3418bdcf6a01aa3110f2fa30 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 29 Nov 2021 11:58:08 +0000 Subject: [PATCH 163/727] remove redundant line (#360) --- piccolo/conf/apps.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/piccolo/conf/apps.py b/piccolo/conf/apps.py index 3d75f679d..67ab3d76d 100644 --- a/piccolo/conf/apps.py +++ b/piccolo/conf/apps.py @@ -369,8 +369,7 @@ def get_engine( self, module_name: t.Optional[str] = None ) -> t.Optional[Engine]: piccolo_conf = self.get_piccolo_conf_module(module_name=module_name) - engine: t.Optional[Engine] = None - engine = getattr(piccolo_conf, ENGINE_VAR, None) + engine: t.Optional[Engine] = getattr(piccolo_conf, ENGINE_VAR, None) if not engine: colored_warning( From 406de58867ed2131e9fec4e00bf7a52df2b00147 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 29 Nov 2021 12:37:13 +0000 Subject: [PATCH 164/727] the docs around supported Postgres versions were massively out of date (#361) --- docs/src/piccolo/getting_started/index.rst | 1 + docs/src/piccolo/getting_started/setup_postgres.rst | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/src/piccolo/getting_started/index.rst b/docs/src/piccolo/getting_started/index.rst index 3ce421721..9403952d4 100644 --- a/docs/src/piccolo/getting_started/index.rst +++ b/docs/src/piccolo/getting_started/index.rst @@ -3,6 +3,7 @@ Getting Started .. toctree:: :caption: Contents: + :maxdepth: 1 ./what_is_piccolo ./database_support diff --git a/docs/src/piccolo/getting_started/setup_postgres.rst b/docs/src/piccolo/getting_started/setup_postgres.rst index 6ef181334..9092ea40f 100644 --- a/docs/src/piccolo/getting_started/setup_postgres.rst +++ b/docs/src/piccolo/getting_started/setup_postgres.rst @@ -79,9 +79,7 @@ DEB packages are available for `Ubuntu `_. +Piccolo is tested on most major Postgres versions (see the `GitHub Actions file `_). ------------------------------------------------------------------------------- From 5876978f1dd171001ff3453debff1230a87524f0 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 29 Nov 2021 13:20:13 +0000 Subject: [PATCH 165/727] improve ASGI home template (#362) --- .../app/home/templates/home.html.jinja_raw | 4 ++-- .../asgi/commands/templates/app/static/main.css | 17 +++++++++++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw b/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw index 35159a5e0..726bcc612 100644 --- a/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw +++ b/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw @@ -8,7 +8,7 @@

Postgres

Make sure you create the database. See the docs for guidance.

-

See piccolo_conf.py for the database settings.

+

See piccolo_conf.py for the database settings.

@@ -26,7 +26,7 @@

Custom Tables

-

An example table called Task exists in tables.py.

+

An example table called Task exists in tables.py.

When you're ready, create a migration, and run it to add the table to the database:

piccolo migrations new home --auto diff --git a/piccolo/apps/asgi/commands/templates/app/static/main.css b/piccolo/apps/asgi/commands/templates/app/static/main.css index b846e1327..7aff46dbc 100644 --- a/piccolo/apps/asgi/commands/templates/app/static/main.css +++ b/piccolo/apps/asgi/commands/templates/app/static/main.css @@ -4,7 +4,8 @@ body, html { } body { - background-color: #f0f0f0; + background-color: #f0f7fd; + color: #2b475f; font-family: 'Open Sans', sans-serif; } @@ -16,6 +17,7 @@ div.hero { a { color: #4C89C8; + text-decoration: none; } div.hero h1 { @@ -36,20 +38,27 @@ div.content { max-width: 50rem; padding: 2rem; transform: translateY(-4rem); + box-shadow: 0px 1px 1px 1px rgb(0,0,0,0.05); } -div.content h2 { +div.content h2, div.content h3 { font-weight: normal; - border-bottom: 4px solid #f0f0f0; +} + +div.content code { + padding: 2px 4px; + background-color: #f0f7fd; + border-radius: 0.2rem; } p.code { - background-color: #2b2b2b; + background-color: #233d58; color: white; font-family: monospace; padding: 1rem; margin: 0; display: block; + border-radius: 0.2rem; } p.code span { From 212724e3c6624c487c4bbe9f9e435ee65f59bb1b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 29 Nov 2021 20:17:38 +0000 Subject: [PATCH 166/727] handle unrecognised column types when parsing default values (#365) --- piccolo/apps/schema/commands/generate.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index 67c5f4407..035a95e61 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -347,9 +347,11 @@ def __add__(self, value: OutputSchema) -> OutputSchema: def get_column_default( column_type: t.Type[Column], column_default: str ) -> t.Any: - pat = COLUMN_DEFAULT_PARSER[column_type] + pat = COLUMN_DEFAULT_PARSER.get(column_type) - if pat is not None: + if pat is None: + return None + else: match = re.match(pat, column_default) if match is not None: value = match.groupdict() From 144faba91f8ca3b049a0918ad808eedcfe0c3d9f Mon Sep 17 00:00:00 2001 From: Gijs Mos <1526544+gmos@users.noreply.github.com> Date: Wed, 1 Dec 2021 17:20:52 +0100 Subject: [PATCH 167/727] add sqlite connection pool warning to Engine (#366) * add sqlite connection pool warning to Engine * use temp directory for sqlite file, and add assertion Co-authored-by: Daniel Townsend --- piccolo/engine/base.py | 19 +++++++++++++++++++ tests/engine/test_pool.py | 39 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/piccolo/engine/base.py b/piccolo/engine/base.py index dbd297bc9..4ae7b7201 100644 --- a/piccolo/engine/base.py +++ b/piccolo/engine/base.py @@ -88,3 +88,22 @@ async def check_version(self): "Piccolo docs." ) colored_warning(message, stacklevel=3) + + def _connection_pool_warning(self): + message = ( + f"Connection pooling is not supported for {self.engine_type}." + ) + logger.warning(message) + colored_warning(message, stacklevel=3) + + async def start_connection_pool(self): + """ + The database driver doesn't implement connection pooling. + """ + self._connection_pool_warning() + + async def close_connection_pool(self): + """ + The database driver doesn't implement connection pooling. + """ + self._connection_pool_warning() diff --git a/tests/engine/test_pool.py b/tests/engine/test_pool.py index b91ade933..f39be20a8 100644 --- a/tests/engine/test_pool.py +++ b/tests/engine/test_pool.py @@ -1,9 +1,14 @@ import asyncio +import os +import tempfile +from unittest import TestCase +from unittest.mock import MagicMock, call, patch from piccolo.engine.postgres import PostgresEngine +from piccolo.engine.sqlite import SQLiteEngine from tests.example_apps.music.tables import Manager -from ..base import DBTestCase, postgres_only +from ..base import DBTestCase, postgres_only, sqlite_only @postgres_only @@ -78,3 +83,35 @@ def test_proxy_methods(self): work, to ensure backwards compatibility. """ asyncio.run(self._create_pool()) + + +@sqlite_only +class TestConnectionPoolWarning(TestCase): + async def _create_pool(self): + sqlite_file = os.path.join(tempfile.gettempdir(), "engine.sqlite") + engine = SQLiteEngine(path=sqlite_file) + await engine.start_connection_pool() + await engine.close_connection_pool() + + @patch("piccolo.engine.base.colored_warning") + def test_warnings(self, colored_warning: MagicMock): + """ + Make sure that when trying to start and close a connection pool with + SQLite, a warning is printed out, as connection pools aren't currently + supported. + """ + asyncio.run(self._create_pool()) + + self.assertEqual( + colored_warning.call_args_list, + [ + call( + "Connection pooling is not supported for sqlite.", + stacklevel=3, + ), + call( + "Connection pooling is not supported for sqlite.", + stacklevel=3, + ), + ], + ) From e800ec581fe59ef460c876c55849d730a5e60c20 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 1 Dec 2021 16:55:54 +0000 Subject: [PATCH 168/727] bumped version --- CHANGES.rst | 24 ++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 5328b8d44..cf42255eb 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,30 @@ Changes ======= +0.61.1 +------ + +Nicer ASGI template +~~~~~~~~~~~~~~~~~~~ + +When using ``piccolo asgi new`` to generate a web app, it now has a nicer home +page template, with improved styles. + +Improved schema generation +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Fixed a bug with ``piccolo schema generate`` where it would crash if the column +type was unrecognised, due to failing to parse the column's default value. +Thanks to @gmos for reporting this issue, and figuring out the fix. + +Fix Pylance error +~~~~~~~~~~~~~~~~~ + +Added ``start_connection_pool`` and ``close_connection_pool`` methods to the +base ``Engine`` class (courtesy @gmos). + +------------------------------------------------------------------------------- + 0.61.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 995c80643..60c015826 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.61.0" +__VERSION__ = "0.61.1" From f8340e689cf1fc0221caf5797d70bcd0cc1e72db Mon Sep 17 00:00:00 2001 From: ali sayyah Date: Thu, 2 Dec 2021 12:38:09 +0330 Subject: [PATCH 169/727] Better tests for asgi new (#340) * Improve tests for piccolo asgi new #317 * lint * lint * tests will check every combination of routers and servers * add dummy server * use postgres engine instead * add missing install * add database to Github Actions * create test db * add postgres env vars * use `python -m virtualenv` * fix typo for virtualenv module * remove coverage Co-authored-by: Daniel Townsend --- .github/workflows/tests.yaml | 60 +++++++++-- pyproject.toml | 6 ++ scripts/test-integration.sh | 10 ++ scripts/test-postgres.sh | 8 +- scripts/test-sqlite.sh | 8 +- .../apps/asgi/commands/files/dummy_server.py | 52 +++++++++ tests/apps/asgi/commands/test_new.py | 101 ++++++++++++++---- tests/base.py | 6 ++ 8 files changed, 224 insertions(+), 27 deletions(-) create mode 100755 scripts/test-integration.sh create mode 100644 tests/apps/asgi/commands/files/dummy_server.py diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index a22673060..c85c11f51 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -1,17 +1,17 @@ name: Test Suite on: - push: - branches: ["master"] - pull_request: - branches: ["master"] + push: + branches: ["master"] + pull_request: + branches: ["master"] jobs: linters: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10'] + python-version: ["3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 @@ -28,11 +28,57 @@ jobs: - name: Lint run: ./scripts/lint.sh + integration: + runs-on: ubuntu-latest + strategy: + matrix: + # These tests are slow, so we only run on the latest Python + # version. + python-version: ["3.10"] + postgres-version: [14] + services: + postgres: + image: postgres:${{ matrix.postgres-version }} + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements/requirements.txt + pip install -r requirements/test-requirements.txt + pip install -r requirements/extras/postgres.txt + - name: Setup postgres + run: | + export PGPASSWORD=postgres + psql -h localhost -c 'CREATE DATABASE piccolo;' -U postgres + psql -h localhost -c "CREATE USER piccolo PASSWORD 'piccolo';" -U postgres + psql -h localhost -c "GRANT ALL PRIVILEGES ON DATABASE piccolo TO piccolo;" -U postgres + psql -h localhost -c "CREATE EXTENSION IF NOT EXISTS \"uuid-ossp\";" -d piccolo -U postgres + - name: Run integration tests + run: ./scripts/test-integration.sh + env: + PG_HOST: localhost + PG_DATABASE: piccolo + PG_PASSWORD: postgres + postgres: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10'] + python-version: ["3.7", "3.8", "3.9", "3.10"] postgres-version: [9.6, 10, 11, 12, 13, 14] # Service containers to run with `container-job` @@ -87,7 +133,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10'] + python-version: ["3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 diff --git a/pyproject.toml b/pyproject.toml index 576acccc7..f46892289 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,3 +14,9 @@ module = [ "dateutil" ] ignore_missing_imports = true + + +[tool.pytest.ini_options] +markers = [ + "integration", +] diff --git a/scripts/test-integration.sh b/scripts/test-integration.sh new file mode 100755 index 000000000..41afc1823 --- /dev/null +++ b/scripts/test-integration.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# To run all in a folder tests/ +# To run all in a file tests/test_foo.py +# To run all in a class tests/test_foo.py::TestFoo +# To run a single test tests/test_foo.py::TestFoo::test_foo + +export PICCOLO_CONF="tests.postgres_conf" +python -m pytest \ + -m integration \ + -s $@ diff --git a/scripts/test-postgres.sh b/scripts/test-postgres.sh index 0c59964f0..9f853b734 100755 --- a/scripts/test-postgres.sh +++ b/scripts/test-postgres.sh @@ -5,4 +5,10 @@ # To run a single test tests/test_foo.py::TestFoo::test_foo export PICCOLO_CONF="tests.postgres_conf" -python -m pytest --cov=piccolo --cov-report xml --cov-report html --cov-fail-under 85 -s $@ +python -m pytest \ + --cov=piccolo \ + --cov-report=xml \ + --cov-report=html \ + --cov-fail-under=85 \ + -m "not integration" \ + -s $@ diff --git a/scripts/test-sqlite.sh b/scripts/test-sqlite.sh index e14ba4aa4..fa53de8bc 100755 --- a/scripts/test-sqlite.sh +++ b/scripts/test-sqlite.sh @@ -5,4 +5,10 @@ # To run a single test tests/test_foo.py::TestFoo::test_foo export PICCOLO_CONF="tests.sqlite_conf" -python -m pytest --cov=piccolo --cov-report xml --cov-report html --cov-fail-under 75 -s $@ +python -m pytest \ + --cov=piccolo \ + --cov-report=xml \ + --cov-report=html \ + --cov-fail-under=75 \ + -m "not integration" \ + -s $@ diff --git a/tests/apps/asgi/commands/files/dummy_server.py b/tests/apps/asgi/commands/files/dummy_server.py new file mode 100644 index 000000000..3c3250776 --- /dev/null +++ b/tests/apps/asgi/commands/files/dummy_server.py @@ -0,0 +1,52 @@ +import asyncio +import importlib +import sys +import typing as t + + +def dummy_server(app: t.Union[str, t.Callable] = "app:app"): + """ + A very simplistic ASGI server. It's used to run the generated ASGI + applications in unit tests. + + :param app: + Either an ASGI app, or a string representing the path to an ASGI app. + For example, ``module_1.app:app`` which would import an ASGI app called + ``app`` from ``module_1.app``. + + """ + print("Running dummy server ...") + + if isinstance(app, str): + path, app_name = app.rsplit(":") + module = importlib.import_module(path) + app = getattr(module, app_name) + + async def send(message): + if message["type"] == "http.response.start": + if message["status"] == 200: + print("Received 200") + else: + sys.exit("200 not received from app") + + async def receive(): + pass + + scope = { + "scheme": "http", + "type": "http", + "path": "/", + "raw_path": b"/", + "method": "GET", + "query_string": b"", + "headers": [], + } + if callable(app): + asyncio.run(app(scope, receive, send)) + print("Exiting dummy server ...") + else: + sys.exit("The app isn't callable!") + + +if __name__ == "__main__": + dummy_server() diff --git a/tests/apps/asgi/commands/test_new.py b/tests/apps/asgi/commands/test_new.py index 68dcc84b0..7ec147bd0 100644 --- a/tests/apps/asgi/commands/test_new.py +++ b/tests/apps/asgi/commands/test_new.py @@ -1,28 +1,93 @@ +import ast import os import shutil +import subprocess import tempfile +from pathlib import Path from unittest import TestCase from unittest.mock import patch +import pytest + from piccolo.apps.asgi.commands.new import ROUTERS, SERVERS, new +from tests.base import unix_only class TestNewApp(TestCase): - @patch( - "piccolo.apps.asgi.commands.new.get_routing_framework", - return_value=ROUTERS[0], - ) - @patch( - "piccolo.apps.asgi.commands.new.get_server", - return_value=SERVERS[0], - ) - def test_new(self, *args, **kwargs): - root = os.path.join(tempfile.gettempdir(), "asgi_app") - - if os.path.exists(root): - shutil.rmtree(root) - - os.mkdir(root) - new(root=root) - - self.assertTrue(os.path.exists(os.path.join(root, "app.py"))) + def test_new(self): + """ + Test that the created files have the correct content. List all .py + files inside the root directory and check if they are valid python code + with ast.parse. + """ + for router in ROUTERS: + for server in SERVERS: + with patch( + "piccolo.apps.asgi.commands.new.get_routing_framework", + return_value=router, + ), patch( + "piccolo.apps.asgi.commands.new.get_server", + return_value=server, + ): + root = os.path.join(tempfile.gettempdir(), "asgi_app") + + if os.path.exists(root): + shutil.rmtree(root) + + os.mkdir(root) + new(root=root) + + # Make sure the files were created + self.assertTrue( + os.path.exists(os.path.join(root, "app.py")) + ) + + # Make sure the Python code is valid. + for file in list(Path(root).rglob("*.py")): + with open(os.path.join(root, file), "r") as f: + ast.parse(f.read()) + f.close() + + +class TestNewAppRuns(TestCase): + @unix_only + @pytest.mark.integration + def test_new(self): + """ + Test that the ASGI app actually runs. + """ + router = "fastapi" + + with patch( + "piccolo.apps.asgi.commands.new.get_routing_framework", + return_value=router, + ), patch( + "piccolo.apps.asgi.commands.new.get_server", + return_value=SERVERS[0], + ): + root = os.path.join(tempfile.gettempdir(), "asgi_app") + + if os.path.exists(root): + shutil.rmtree(root) + + os.mkdir(root) + new(root=root) + + # Copy a dummy ASGI server, so we can test that the server works. + shutil.copyfile( + os.path.join( + os.path.dirname(__file__), + "files", + "dummy_server.py", + ), + os.path.join(root, "dummy_server.py"), + ) + + response = subprocess.run( + f"cd {root} && " + "python -m venv venv && " + "./venv/bin/pip install -r requirements.txt && " + "./venv/bin/python dummy_server.py", + shell=True, + ) + self.assertEqual(response.returncode, 0, msg=f"{router} failed") diff --git a/tests/base.py b/tests/base.py index 240151073..3d9c03851 100644 --- a/tests/base.py +++ b/tests/base.py @@ -1,5 +1,6 @@ from __future__ import annotations +import sys import typing as t from unittest import TestCase from unittest.mock import MagicMock @@ -25,6 +26,11 @@ ) +unix_only = pytest.mark.skipif( + sys.platform.startswith("win"), reason="Only running on a Unix system" +) + + def set_mock_return_value(magic_mock: MagicMock, return_value: t.Any): """ Python 3.8 has good support for mocking coroutines. For older versions, From 823be3e35f42858eeaf79e9188649cc43302a329 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 4 Dec 2021 09:24:04 +0000 Subject: [PATCH 170/727] fix docstring typos --- piccolo/utils/pydantic.py | 6 ++++-- piccolo/utils/sync.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/piccolo/utils/pydantic.py b/piccolo/utils/pydantic.py index b5ddda329..192c0641a 100644 --- a/piccolo/utils/pydantic.py +++ b/piccolo/utils/pydantic.py @@ -102,10 +102,12 @@ def create_pydantic_model( converted. :param exclude_columns: A tuple of ``Column`` instances that should be excluded from the - Pydantic model. Only specify ``include_column`` or ``exclude_column``. + Pydantic model. Only specify ``include_columns`` or + ``exclude_columns``. :param include_columns: A tuple of ``Column`` instances that should be included in the - Pydantic model. Only specify ``include_column`` or ``exclude_column``. + Pydantic model. Only specify ``include_columns`` or + ``exclude_columns``. :param include_default_columns: Whether to include columns like ``id`` in the serialiser. You will typically include these columns in GET requests, but don't require diff --git a/piccolo/utils/sync.py b/piccolo/utils/sync.py index 42317af01..020b5899e 100644 --- a/piccolo/utils/sync.py +++ b/piccolo/utils/sync.py @@ -11,7 +11,7 @@ def run_sync(coroutine: t.Coroutine): as possible. 1. When called within a coroutine. - 2. When called from `python -m asyncio`, or iPython with %autoawait + 2. When called from ``python -m asyncio``, or iPython with %autoawait enabled, which means an event loop may already be running in the current thread. From 0164bd98406d014e0f390adb47c78febd302fc4e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 4 Dec 2021 09:38:21 +0000 Subject: [PATCH 171/727] more docstring improvements --- piccolo/columns/column_types.py | 8 ++-- piccolo/columns/defaults/base.py | 6 ++- piccolo/querystring.py | 4 +- piccolo/table.py | 66 ++++++++++++++++++++++++-------- 4 files changed, 59 insertions(+), 25 deletions(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index edaeccb26..b815f9b02 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -871,13 +871,13 @@ class Ticket(Table): >>> Ticket.select(Ticket.price).run_sync() {'price': Decimal('50.0')} - :arg digits: + :param digits: When creating the column, you specify how many digits are allowed - using a tuple. The first value is the `precision`, which is the - total number of digits allowed. The second value is the `range`, + using a tuple. The first value is the ``precision``, which is the + total number of digits allowed. The second value is the ``range``, which specifies how many of those digits are after the decimal point. For example, to store monetary values up to £999.99, the - digits argument is `(5,2)`. + digits argument is ``(5,2)``. """ diff --git a/piccolo/columns/defaults/base.py b/piccolo/columns/defaults/base.py index 1346a2964..eceb6f0f1 100644 --- a/piccolo/columns/defaults/base.py +++ b/piccolo/columns/defaults/base.py @@ -24,9 +24,10 @@ def get_postgres_interval_string(self, attributes: t.List[str]) -> str: Returns a string usable as an interval argument in Postgres e.g. "1 day 2 hour". - :arg attributes: + :param attributes: Date / time attributes to extract from the instance. e.g. ['hours', 'minutes'] + """ interval_components = [] for attr_name in attributes: @@ -41,9 +42,10 @@ def get_sqlite_interval_string(self, attributes: t.List[str]) -> str: Returns a string usable as an interval argument in SQLite e.g. "'-2 hours', '1 days'". - :arg attributes: + :param attributes: Date / time attributes to extract from the instance. e.g. ['hours', 'minutes'] + """ interval_components = [] for attr_name in attributes: diff --git a/piccolo/querystring.py b/piccolo/querystring.py index 5b273db71..2d68dca70 100644 --- a/piccolo/querystring.py +++ b/piccolo/querystring.py @@ -73,8 +73,8 @@ def __init__( def __str__(self): """ - The SQL returned by the __str__ method isn't used directly in queries - - it's just a usability feature. + The SQL returned by the ``__str__`` method isn't used directly in + queries - it's just a usability feature. """ _, bundled, combined_args = self.bundle( start_index=1, bundled=[], combined_args=[] diff --git a/piccolo/table.py b/piccolo/table.py index aab5beec4..6d963a0bb 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -696,13 +696,17 @@ def select( These are all equivalent: - await Band.select().columns(Band.name).run() - await Band.select(Band.name).run() - await Band.select('name').run() + .. code-block:: python + + await Band.select().columns(Band.name).run() + await Band.select(Band.name).run() + await Band.select('name').run() + + :param exclude_secrets: + If ``True``, any password fields are omitted from the response. + Even though passwords are hashed, you still don't want them being + passed over the network if avoidable. - :param exclude_secrets: If True, any password fields are omitted from - the response. Even though passwords are hashed, you still don't want - them being passed over the network if avoidable. """ _columns = cls._process_column_args(*columns) return Select( @@ -714,10 +718,14 @@ def delete(cls, force=False) -> Delete: """ Delete rows from the table. - await Band.delete().where(Band.name == 'Pythonistas').run() + .. code-block:: python + + await Band.delete().where(Band.name == 'Pythonistas').run() + + :param force: + Unless set to ``True``, deletions aren't allowed without a + ``where`` clause, to prevent accidental mass deletions. - Unless 'force' is set to True, deletions aren't allowed without a - 'where' clause, to prevent accidental mass deletions. """ return Delete(table=cls, force=force) @@ -728,7 +736,10 @@ def create_table( """ Create table, along with all columns. - await Band.create_table().run() + .. code-block:: python + + await Band.create_table().run() + """ return Create( table=cls, @@ -741,7 +752,10 @@ def alter(cls) -> Alter: """ Used to modify existing tables and columns. - await Band.alter().rename_column(Band.popularity, 'rating').run() + .. code-block:: python + + await Band.alter().rename_column(Band.popularity, 'rating').run() + """ return Alter(table=cls) @@ -790,7 +804,10 @@ def count(cls) -> Count: """ Count the number of matching rows. - await Band.count().where(Band.popularity > 1000).run() + .. code-block:: python + + await Band.count().where(Band.popularity > 1000).run() + """ return Count(table=cls) @@ -799,7 +816,10 @@ def exists(cls) -> Exists: """ Use it to check if a row exists, not if the table exists. - await Band.exists().where(Band.name == 'Pythonistas').run() + .. code-block:: python + + await Band.exists().where(Band.name == 'Pythonistas').run() + """ return Exists(table=cls) @@ -808,7 +828,10 @@ def table_exists(cls) -> TableExists: """ Check if the table exists in the database. - await Band.table_exists().run() + .. code-block:: python + + await Band.table_exists().run() + """ return TableExists(table=cls) @@ -850,7 +873,10 @@ def indexes(cls) -> Indexes: """ Returns a list of the indexes for this tables. - await Band.indexes().run() + .. code-block:: python + + await Band.indexes().run() + """ return Indexes(table=cls) @@ -865,7 +891,10 @@ def create_index( Create a table index. If multiple columns are specified, this refers to a multicolumn index, rather than multiple single column indexes. - await Band.create_index([Band.name]).run() + .. code-block:: python + + await Band.create_index([Band.name]).run() + """ return CreateIndex( table=cls, @@ -882,7 +911,10 @@ def drop_index( Drop a table index. If multiple columns are specified, this refers to a multicolumn index, rather than multiple single column indexes. - await Band.drop_index([Band.name]).run() + .. code-block:: python + + await Band.drop_index([Band.name]).run() + """ return DropIndex(table=cls, columns=columns, if_exists=if_exists) From b0e0fe3d2ef96ab4dfd1e1c5b49709fbd61b2274 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 14 Dec 2021 21:33:16 +0000 Subject: [PATCH 172/727] Fixes for problematic column names (#374) * use absolute imports * modify some DDL statements to handle columns with problematic names * add missing codeblocks * fix linting errors * more tests for problematic column names --- piccolo/columns/base.py | 50 ++++++++- piccolo/columns/column_types.py | 8 +- piccolo/query/methods/alter.py | 28 ++--- piccolo/query/methods/insert.py | 4 +- piccolo/query/methods/select.py | 116 ++++++++++++++++---- piccolo/query/methods/update.py | 2 +- tests/columns/test_combination.py | 12 +- tests/columns/test_reserved_column_names.py | 60 ++++++++++ tests/engine/test_nested_transaction.py | 3 +- tests/engine/test_pool.py | 3 +- tests/table/test_alter.py | 57 ++++++++-- tests/table/test_batch.py | 3 +- tests/table/test_count.py | 3 +- tests/table/test_delete.py | 3 +- tests/table/test_exists.py | 3 +- tests/table/test_indexes.py | 3 +- tests/table/test_insert.py | 3 +- tests/table/test_raw.py | 3 +- tests/table/test_repr.py | 3 +- tests/table/test_select.py | 3 +- tests/table/test_update.py | 3 +- 21 files changed, 285 insertions(+), 88 deletions(-) create mode 100644 tests/columns/test_reserved_column_names.py diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index bed24f9da..b339a782d 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -208,14 +208,46 @@ def get_choices_dict(self) -> t.Optional[t.Dict[str, t.Any]]: return output - def get_full_name(self, just_alias=False) -> str: + def get_full_name( + self, just_alias: bool = False, include_quotes: bool = False + ) -> str: """ Returns the full column name, taking into account joins. + + :param just_alias: + Examples: + + .. code-block python:: + + >>> Band.manager.name._meta.get_full_name(just_alias=True) + 'band$manager.name' + + >>> Band.manager.name._meta.get_full_name(just_alias=False) + 'band$manager.name AS "manager$name"' + + :param include_quotes: + If you're using the name in a SQL query, each component needs to be + surrounded by double quotes, in case the table or column name + clashes with a reserved SQL keyword (for example, a column called + ``order``). + + .. code-block python:: + + >>> column._meta.get_full_name(include_quotes=True) + '"my_table_name"."my_column_name"' + + >>> column._meta.get_full_name(include_quotes=False) + 'my_table_name.my_column_name' + + """ column_name = self.db_column_name if not self.call_chain: - return f"{self.table._meta.tablename}.{column_name}" + if include_quotes: + return f'"{self.table._meta.tablename}"."{column_name}"' + else: + return f"{self.table._meta.tablename}.{column_name}" column_name = ( "$".join( @@ -224,7 +256,11 @@ def get_full_name(self, just_alias=False) -> str: + f"${column_name}" ) - alias = f"{self.call_chain[-1]._meta.table_alias}.{self.name}" + if include_quotes: + alias = f'"{self.call_chain[-1]._meta.table_alias}"."{self.name}"' + else: + alias = f"{self.call_chain[-1]._meta.table_alias}.{self.name}" + if just_alias: return alias else: @@ -660,8 +696,12 @@ def get_select_string(self, engine_type: str, just_alias=False) -> str: How to refer to this column in a SQL query. """ if self.alias is None: - return self._meta.get_full_name(just_alias=just_alias) - original_name = self._meta.get_full_name(just_alias=True) + return self._meta.get_full_name( + just_alias=just_alias, include_quotes=True + ) + original_name = self._meta.get_full_name( + just_alias=True, include_quotes=True + ) return f"{original_name} AS {self.alias}" def get_where_string(self, engine_type: str) -> str: diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index b815f9b02..ba3d4bbc6 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -1552,7 +1552,9 @@ def arrow(self, key: str) -> JSONB: return instance def get_select_string(self, engine_type: str, just_alias=False) -> str: - select_string = self._meta.get_full_name(just_alias=just_alias) + select_string = self._meta.get_full_name( + just_alias=just_alias, include_quotes=True + ) if self.json_operator is None: return select_string else: @@ -1720,7 +1722,9 @@ def __getitem__(self, value: int) -> Array: raise ValueError("Only integers can be used for indexing.") def get_select_string(self, engine_type: str, just_alias=False) -> str: - select_string = self._meta.get_full_name(just_alias=just_alias) + select_string = self._meta.get_full_name( + just_alias=just_alias, include_quotes=True + ) if isinstance(self.index, int): return f"{select_string}[{self.index}]" diff --git a/piccolo/query/methods/alter.py b/piccolo/query/methods/alter.py index 48aa240ad..0768c27c0 100644 --- a/piccolo/query/methods/alter.py +++ b/piccolo/query/methods/alter.py @@ -60,14 +60,14 @@ class RenameColumn(AlterColumnStatement): @property def ddl(self) -> str: - return f"RENAME COLUMN {self.column_name} TO {self.new_name}" + return f'RENAME COLUMN "{self.column_name}" TO "{self.new_name}"' @dataclass class DropColumn(AlterColumnStatement): @property def ddl(self) -> str: - return f"DROP COLUMN {self.column_name}" + return f'DROP COLUMN "{self.column_name}"' @dataclass @@ -87,7 +87,7 @@ def ddl(self) -> str: class DropDefault(AlterColumnStatement): @property def ddl(self) -> str: - return f"ALTER COLUMN {self.column_name} DROP DEFAULT" + return f'ALTER COLUMN "{self.column_name}" DROP DEFAULT' @dataclass @@ -111,7 +111,7 @@ def ddl(self) -> str: column_name = self.old_column._meta.db_column_name query = ( - f"ALTER COLUMN {column_name} TYPE {self.new_column.column_type}" + f'ALTER COLUMN "{column_name}" TYPE {self.new_column.column_type}' ) if self.using_expression is not None: query += f" USING {self.using_expression}" @@ -128,7 +128,7 @@ class SetDefault(AlterColumnStatement): @property def ddl(self) -> str: sql_value = self.column.get_sql_value(self.value) - return f"ALTER COLUMN {self.column_name} SET DEFAULT {sql_value}" + return f'ALTER COLUMN "{self.column_name}" SET DEFAULT {sql_value}' @dataclass @@ -140,7 +140,7 @@ class SetUnique(AlterColumnStatement): @property def ddl(self) -> str: if self.boolean: - return f"ADD UNIQUE ({self.column_name})" + return f'ADD UNIQUE ("{self.column_name}")' if isinstance(self.column, str): raise ValueError( "Removing a unique constraint requires a Column instance " @@ -161,9 +161,9 @@ class SetNull(AlterColumnStatement): @property def ddl(self) -> str: if self.boolean: - return f"ALTER COLUMN {self.column_name} DROP NOT NULL" + return f'ALTER COLUMN "{self.column_name}" DROP NOT NULL' else: - return f"ALTER COLUMN {self.column_name} SET NOT NULL" + return f'ALTER COLUMN "{self.column_name}" SET NOT NULL' @dataclass @@ -175,7 +175,7 @@ class SetLength(AlterColumnStatement): @property def ddl(self) -> str: - return f"ALTER COLUMN {self.column_name} TYPE VARCHAR({self.length})" + return f'ALTER COLUMN "{self.column_name}" TYPE VARCHAR({self.length})' @dataclass @@ -209,9 +209,9 @@ class AddForeignKeyConstraint(AlterStatement): @property def ddl(self) -> str: query = ( - f"ADD CONSTRAINT {self.constraint_name} FOREIGN KEY " - f"({self.foreign_key_column_name}) REFERENCES " - f"{self.referenced_table_name} ({self.referenced_column_name})" + f'ADD CONSTRAINT "{self.constraint_name}" FOREIGN KEY ' + f'("{self.foreign_key_column_name}") REFERENCES ' + f'"{self.referenced_table_name}" ("{self.referenced_column_name}")' ) if self.on_delete: query += f" ON DELETE {self.on_delete.value}" @@ -231,12 +231,12 @@ class SetDigits(AlterColumnStatement): @property def ddl(self) -> str: if self.digits is None: - return f"ALTER COLUMN {self.column_name} TYPE {self.column_type}" + return f'ALTER COLUMN "{self.column_name}" TYPE {self.column_type}' precision = self.digits[0] scale = self.digits[1] return ( - f"ALTER COLUMN {self.column_name} TYPE " + f'ALTER COLUMN "{self.column_name}" TYPE ' f"{self.column_type}({precision}, {scale})" ) diff --git a/piccolo/query/methods/insert.py b/piccolo/query/methods/insert.py index b1b839630..313c7e1c1 100644 --- a/piccolo/query/methods/insert.py +++ b/piccolo/query/methods/insert.py @@ -39,7 +39,7 @@ def run_callback(self, results): def sqlite_querystrings(self) -> t.Sequence[QueryString]: base = f"INSERT INTO {self.table._meta.tablename}" columns = ",".join( - i._meta.db_column_name for i in self.table._meta.columns + f'"{i._meta.db_column_name}"' for i in self.table._meta.columns ) values = ",".join("{}" for _ in self.add_delegate._add) query = f"{base} ({columns}) VALUES {values}" @@ -58,7 +58,7 @@ def postgres_querystrings(self) -> t.Sequence[QueryString]: columns = ",".join( f'"{i._meta.db_column_name}"' for i in self.table._meta.columns ) - values = ",".join("{}" for i in self.add_delegate._add) + values = ",".join("{}" for _ in self.add_delegate._add) primary_key_name = self.table._meta.primary_key._meta.name query = ( f"{base} ({columns}) VALUES {values} RETURNING {primary_key_name}" diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 6260ad917..3395d6dce 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -32,11 +32,22 @@ def is_numeric_column(column: Column) -> bool: class Avg(Selectable): """ - AVG() SQL function. Column type must be numeric to run the query. + ``AVG()`` SQL function. Column type must be numeric to run the query. + + .. code-block:: python + + await Band.select(Avg(Band.popularity)).run() + + # We can use an alias. These two are equivalent: + + await Band.select( + Avg(Band.popularity, alias="popularity_avg") + ).run() + + await Band.select( + Avg(Band.popularity).as_alias("popularity_avg") + ).run() - await Band.select(Avg(Band.popularity)).run() or with aliases - await Band.select(Avg(Band.popularity, alias="popularity_avg")).run() - await Band.select(Avg(Band.popularity).as_alias("popularity_avg")).run() """ def __init__(self, column: Column, alias: str = "avg"): @@ -47,7 +58,9 @@ def __init__(self, column: Column, alias: str = "avg"): self.alias = alias def get_select_string(self, engine_type: str, just_alias=False) -> str: - column_name = self.column._meta.get_full_name(just_alias=just_alias) + column_name = self.column._meta.get_full_name( + just_alias=just_alias, include_quotes=True + ) return f"AVG({column_name}) AS {self.alias}" @@ -59,9 +72,21 @@ class Count(Selectable): column. If no column is specified, the count is for all rows, whether they have null values or not. - Band.select(Band.name, Count()).group_by(Band.name).run() - Band.select(Band.name, Count(alias="total")).group_by(Band.name).run() - Band.select(Band.name, Count().as_alias("total")).group_by(Band.name).run() + .. code-block:: python + + Band.select(Band.name, Count()).group_by(Band.name).run() + + # We can use an alias. These two are equivalent: + + Band.select( + Band.name, Count(alias="total") + ).group_by(Band.name).run() + + Band.select( + Band.name, + Count().as_alias("total") + ).group_by(Band.name).run() + """ def __init__( @@ -75,18 +100,31 @@ def get_select_string(self, engine_type: str, just_alias=False) -> str: column_name = "*" else: column_name = self.column._meta.get_full_name( - just_alias=just_alias + just_alias=just_alias, include_quotes=True ) return f"COUNT({column_name}) AS {self.alias}" class Max(Selectable): """ - MAX() SQL function. + ``MAX()`` SQL function. + + .. code-block:: python + + await Band.select( + Max(Band.popularity) + ).run() + + # We can use an alias. These two are equivalent: + + await Band.select( + Max(Band.popularity, alias="popularity_max") + ).run() + + await Band.select( + Max(Band.popularity).as_alias("popularity_max") + ).run() - await Band.select(Max(Band.popularity)).run() or with aliases - await Band.select(Max(Band.popularity, alias="popularity_max")).run() - await Band.select(Max(Band.popularity).as_alias("popularity_max")).run() """ def __init__(self, column: Column, alias: str = "max"): @@ -94,17 +132,30 @@ def __init__(self, column: Column, alias: str = "max"): self.alias = alias def get_select_string(self, engine_type: str, just_alias=False) -> str: - column_name = self.column._meta.get_full_name(just_alias=just_alias) + column_name = self.column._meta.get_full_name( + just_alias=just_alias, include_quotes=True + ) return f"MAX({column_name}) AS {self.alias}" class Min(Selectable): """ - MIN() SQL function. + ``MIN()`` SQL function. + + .. code-block:: python + + await Band.select(Min(Band.popularity)).run() + + # We can use an alias. These two are equivalent: + + await Band.select( + Min(Band.popularity, alias="popularity_min") + ).run() + + await Band.select( + Min(Band.popularity).as_alias("popularity_min") + ).run() - await Band.select(Min(Band.popularity)).run() - await Band.select(Min(Band.popularity, alias="popularity_min")).run() - await Band.select(Min(Band.popularity).as_alias("popularity_min")).run() """ def __init__(self, column: Column, alias: str = "min"): @@ -112,17 +163,32 @@ def __init__(self, column: Column, alias: str = "min"): self.alias = alias def get_select_string(self, engine_type: str, just_alias=False) -> str: - column_name = self.column._meta.get_full_name(just_alias=just_alias) + column_name = self.column._meta.get_full_name( + just_alias=just_alias, include_quotes=True + ) return f"MIN({column_name}) AS {self.alias}" class Sum(Selectable): """ - SUM() SQL function. Column type must be numeric to run the query. + ``SUM()`` SQL function. Column type must be numeric to run the query. + + .. code-block:: python + + await Band.select( + Sum(Band.popularity) + ).run() + + # We can use an alias. These two are equivalent: + + await Band.select( + Sum(Band.popularity, alias="popularity_sum") + ).run() + + await Band.select( + Sum(Band.popularity).as_alias("popularity_sum") + ).run() - await Band.select(Sum(Band.popularity)).run() - await Band.select(Sum(Band.popularity, alias="popularity_sum")).run() - await Band.select(Sum(Band.popularity).as_alias("popularity_sum")).run() """ def __init__(self, column: Column, alias: str = "sum"): @@ -133,7 +199,9 @@ def __init__(self, column: Column, alias: str = "sum"): self.alias = alias def get_select_string(self, engine_type: str, just_alias=False) -> str: - column_name = self.column._meta.get_full_name(just_alias=just_alias) + column_name = self.column._meta.get_full_name( + just_alias=just_alias, include_quotes=True + ) return f"SUM({column_name}) AS {self.alias}" diff --git a/piccolo/query/methods/update.py b/piccolo/query/methods/update.py index 129599e39..d2f029998 100644 --- a/piccolo/query/methods/update.py +++ b/piccolo/query/methods/update.py @@ -49,7 +49,7 @@ def default_querystrings(self) -> t.Sequence[QueryString]: self.validate() columns_str = ", ".join( - f"{col._meta.db_column_name} = {{}}" + f'"{col._meta.db_column_name}" = {{}}' for col, _ in self.values_delegate._values.items() ) diff --git a/tests/columns/test_combination.py b/tests/columns/test_combination.py index 4f756192e..032830d6e 100644 --- a/tests/columns/test_combination.py +++ b/tests/columns/test_combination.py @@ -7,22 +7,24 @@ class TestWhere(unittest.TestCase): def test_equals(self): _where = Band.name == "Pythonistas" sql = _where.__str__() - self.assertEqual(sql, "band.name = 'Pythonistas'") + self.assertEqual(sql, '"band"."name" = \'Pythonistas\'') def test_not_equal(self): _where = Band.name != "CSharps" sql = _where.__str__() - self.assertEqual(sql, "band.name != 'CSharps'") + self.assertEqual(sql, '"band"."name" != \'CSharps\'') def test_like(self): _where = Band.name.like("Python%") sql = _where.__str__() - self.assertEqual(sql, "band.name LIKE 'Python%'") + self.assertEqual(sql, '"band"."name" LIKE \'Python%\'') def test_is_in(self): _where = Band.name.is_in(["Pythonistas", "Rustaceans"]) sql = _where.__str__() - self.assertEqual(sql, "band.name IN ('Pythonistas', 'Rustaceans')") + self.assertEqual( + sql, "\"band\".\"name\" IN ('Pythonistas', 'Rustaceans')" + ) with self.assertRaises(ValueError): Band.name.is_in([]) @@ -30,7 +32,7 @@ def test_is_in(self): def test_not_in(self): _where = Band.name.not_in(["CSharps"]) sql = _where.__str__() - self.assertEqual(sql, "band.name NOT IN ('CSharps')") + self.assertEqual(sql, '"band"."name" NOT IN (\'CSharps\')') with self.assertRaises(ValueError): Band.name.not_in([]) diff --git a/tests/columns/test_reserved_column_names.py b/tests/columns/test_reserved_column_names.py new file mode 100644 index 000000000..77a56b9ae --- /dev/null +++ b/tests/columns/test_reserved_column_names.py @@ -0,0 +1,60 @@ +from unittest import TestCase + +from piccolo.columns.column_types import Integer, Varchar +from piccolo.table import Table + + +class Concert(Table): + """ + ``order`` is a problematic name, as it clashes with a reserved SQL keyword: + + https://www.postgresql.org/docs/current/sql-keywords-appendix.html + + """ + + name = Varchar() + order = Integer() + + +class TestReservedColumnNames(TestCase): + """ + Make sure the table works as expected, even though it has a problematic + column name. + """ + + def setUp(self): + Concert.create_table().run_sync() + + def tearDown(self): + Concert.alter().drop_table().run_sync() + + def test_common_operations(self): + # Save / Insert + concert = Concert(name="Royal Albert Hall", order=1) + concert.save().run_sync() + self.assertEqual( + Concert.select(Concert.order).run_sync(), + [{"order": 1}], + ) + + # Save / Update + concert.order = 2 + concert.save().run_sync() + self.assertEqual( + Concert.select(Concert.order).run_sync(), + [{"order": 2}], + ) + + # Update + Concert.update({Concert.order: 3}).run_sync() + self.assertEqual( + Concert.select(Concert.order).run_sync(), + [{"order": 3}], + ) + + # Delete + Concert.delete().where(Concert.order == 3).run_sync() + self.assertEqual( + Concert.select(Concert.order).run_sync(), + [], + ) diff --git a/tests/engine/test_nested_transaction.py b/tests/engine/test_nested_transaction.py index 999870ed7..cfc08e35b 100644 --- a/tests/engine/test_nested_transaction.py +++ b/tests/engine/test_nested_transaction.py @@ -5,10 +5,9 @@ from piccolo.engine.exceptions import TransactionError from piccolo.engine.sqlite import SQLiteEngine from piccolo.table import Table +from tests.base import DBTestCase, sqlite_only from tests.example_apps.music.tables import Manager -from ..base import DBTestCase, sqlite_only - ENGINE_1 = SQLiteEngine(path="engine1.sqlite") ENGINE_2 = SQLiteEngine(path="engine2.sqlite") diff --git a/tests/engine/test_pool.py b/tests/engine/test_pool.py index f39be20a8..b061ef042 100644 --- a/tests/engine/test_pool.py +++ b/tests/engine/test_pool.py @@ -6,10 +6,9 @@ from piccolo.engine.postgres import PostgresEngine from piccolo.engine.sqlite import SQLiteEngine +from tests.base import DBTestCase, postgres_only, sqlite_only from tests.example_apps.music.tables import Manager -from ..base import DBTestCase, postgres_only, sqlite_only - @postgres_only class TestPool(DBTestCase): diff --git a/tests/table/test_alter.py b/tests/table/test_alter.py index 7a60f4359..125b387f0 100644 --- a/tests/table/test_alter.py +++ b/tests/table/test_alter.py @@ -4,36 +4,61 @@ from unittest import TestCase from piccolo.columns import BigInt, Integer, Numeric, Varchar +from piccolo.columns.base import Column from piccolo.columns.column_types import ForeignKey, Text from piccolo.table import Table from tests.base import DBTestCase, postgres_only from tests.example_apps.music.tables import Band, Manager -if t.TYPE_CHECKING: - from piccolo.columns.base import Column - class TestRenameColumn(DBTestCase): - def _test_rename(self, column: Column): + def _test_rename( + self, + existing_column: t.Union[Column, str], + new_column_name: str = "rating", + ): self.insert_row() - rename_query = Band.alter().rename_column(column, "rating") + rename_query = Band.alter().rename_column( + existing_column, new_column_name + ) rename_query.run_sync() select_query = Band.raw("SELECT * FROM band") response = select_query.run_sync() column_names = response[0].keys() + existing_column_name = ( + existing_column._meta.name + if isinstance(existing_column, Column) + else existing_column + ) self.assertTrue( - ("rating" in column_names) and ("popularity" not in column_names) + (new_column_name in column_names) + and (existing_column_name not in column_names) ) - def test_rename_string(self): + def test_column(self): + """ + Make sure a ``Column`` argument works. + """ self._test_rename(Band.popularity) - def test_rename_column(self): + def test_string(self): + """ + Make sure a string argument works. + """ self._test_rename("popularity") + def test_problematic_name(self): + """ + Make sure we can rename columns with names which clash with SQL + keywords. + """ + self._test_rename( + existing_column=Band.popularity, new_column_name="order" + ) + class TestRenameTable(DBTestCase): def test_rename(self): @@ -84,21 +109,21 @@ def _test_add_column( self.assertEqual(response[0][column_name], expected_value) - def test_add_integer(self): + def test_integer(self): self._test_add_column( column=Integer(null=True, default=None), column_name="members", expected_value=None, ) - def test_add_foreign_key(self): + def test_foreign_key(self): self._test_add_column( column=ForeignKey(references=Manager), column_name="assistant_manager", expected_value=None, ) - def test_add_text(self): + def test_text(self): bio = "An amazing band" self._test_add_column( column=Text(default=bio), @@ -106,6 +131,16 @@ def test_add_text(self): expected_value=bio, ) + def test_problematic_name(self): + """ + Make sure we can add columns with names which clash with SQL keywords. + """ + self._test_add_column( + column=Text(default="asc"), + column_name="order", + expected_value="asc", + ) + @postgres_only class TestUnique(DBTestCase): diff --git a/tests/table/test_batch.py b/tests/table/test_batch.py index b06238880..114c683fa 100644 --- a/tests/table/test_batch.py +++ b/tests/table/test_batch.py @@ -1,10 +1,9 @@ import asyncio import math +from tests.base import DBTestCase from tests.example_apps.music.tables import Manager -from ..base import DBTestCase - class TestBatchSelect(DBTestCase): def _check_results(self, batch): diff --git a/tests/table/test_count.py b/tests/table/test_count.py index 45f33f2f5..a5943a51d 100644 --- a/tests/table/test_count.py +++ b/tests/table/test_count.py @@ -1,7 +1,6 @@ +from tests.base import DBTestCase from tests.example_apps.music.tables import Band -from ..base import DBTestCase - class TestCount(DBTestCase): def test_exists(self): diff --git a/tests/table/test_delete.py b/tests/table/test_delete.py index 15d3026b9..58838bd94 100644 --- a/tests/table/test_delete.py +++ b/tests/table/test_delete.py @@ -1,8 +1,7 @@ from piccolo.query.methods.delete import DeletionError +from tests.base import DBTestCase from tests.example_apps.music.tables import Band -from ..base import DBTestCase - class TestDelete(DBTestCase): def test_delete(self): diff --git a/tests/table/test_exists.py b/tests/table/test_exists.py index 59f7c1382..7650ad85f 100644 --- a/tests/table/test_exists.py +++ b/tests/table/test_exists.py @@ -1,7 +1,6 @@ +from tests.base import DBTestCase from tests.example_apps.music.tables import Band -from ..base import DBTestCase - class TestExists(DBTestCase): def test_exists(self): diff --git a/tests/table/test_indexes.py b/tests/table/test_indexes.py index ba6cb9aa1..95275a4f5 100644 --- a/tests/table/test_indexes.py +++ b/tests/table/test_indexes.py @@ -1,9 +1,8 @@ from unittest import TestCase +from tests.base import DBTestCase from tests.example_apps.music.tables import Manager -from ..base import DBTestCase - class TestIndexes(DBTestCase): def test_create_index(self): diff --git a/tests/table/test_insert.py b/tests/table/test_insert.py index 66e90350d..b2f9bf33a 100644 --- a/tests/table/test_insert.py +++ b/tests/table/test_insert.py @@ -1,7 +1,6 @@ +from tests.base import DBTestCase from tests.example_apps.music.tables import Band, Manager -from ..base import DBTestCase - class TestInsert(DBTestCase): def test_insert(self): diff --git a/tests/table/test_raw.py b/tests/table/test_raw.py index 9da3b919c..3e6ebdb41 100644 --- a/tests/table/test_raw.py +++ b/tests/table/test_raw.py @@ -1,7 +1,6 @@ +from tests.base import DBTestCase from tests.example_apps.music.tables import Band -from ..base import DBTestCase - class TestRaw(DBTestCase): def test_raw_without_args(self): diff --git a/tests/table/test_repr.py b/tests/table/test_repr.py index 240bfc53d..08f0ff783 100644 --- a/tests/table/test_repr.py +++ b/tests/table/test_repr.py @@ -1,7 +1,6 @@ +from tests.base import DBTestCase, postgres_only, sqlite_only from tests.example_apps.music.tables import Manager -from ..base import DBTestCase, postgres_only, sqlite_only - class TestTableRepr(DBTestCase): @postgres_only diff --git a/tests/table/test_select.py b/tests/table/test_select.py index 5d7d774e0..f00421762 100644 --- a/tests/table/test_select.py +++ b/tests/table/test_select.py @@ -3,10 +3,9 @@ from piccolo.apps.user.tables import BaseUser from piccolo.columns.combination import WhereRaw from piccolo.query.methods.select import Avg, Count, Max, Min, Sum +from tests.base import DBTestCase, postgres_only, sqlite_only from tests.example_apps.music.tables import Band, Concert, Manager, Venue -from ..base import DBTestCase, postgres_only, sqlite_only - class TestSelect(DBTestCase): def test_query_all_columns(self): diff --git a/tests/table/test_update.py b/tests/table/test_update.py index d3d27715a..d2b2e3280 100644 --- a/tests/table/test_update.py +++ b/tests/table/test_update.py @@ -1,7 +1,6 @@ +from tests.base import DBTestCase from tests.example_apps.music.tables import Band, Poster -from ..base import DBTestCase - class TestUpdate(DBTestCase): def check_response(self): From 1c6654f52c64ed989829913d50aa88eb7314cd4e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 14 Dec 2021 21:40:48 +0000 Subject: [PATCH 173/727] bumped version --- CHANGES.rst | 14 ++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index cf42255eb..74d2c59e1 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,20 @@ Changes ======= +0.61.2 +------ + +Fixed some edge cases where migrations would fail if a column name clashed with +a reserved Postgres keyword (for example ``order`` or ``select``). + +We now have more robust tests for ``piccolo asgi new`` - as part of our CI we +actually run the generated ASGI app to make sure it works (thanks to @AliSayyah +and @yezz123 for their help with this). + +We also improved docstrings across the project. + +------------------------------------------------------------------------------- + 0.61.1 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 60c015826..bbbe9d079 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.61.1" +__VERSION__ = "0.61.2" From 2301183f01f11aca56fe460922244907546f0525 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 17 Dec 2021 11:01:24 +0000 Subject: [PATCH 174/727] m2m prototype --- piccolo/columns/__init__.py | 1 + piccolo/columns/m2m.py | 140 ++++++++++++++++++++++++++++++++++++ piccolo/table.py | 5 ++ tests/columns/test_m2m.py | 69 ++++++++++++++++++ 4 files changed, 215 insertions(+) create mode 100644 piccolo/columns/m2m.py create mode 100644 tests/columns/test_m2m.py diff --git a/piccolo/columns/__init__.py b/piccolo/columns/__init__.py index 644b685ac..4f8c968e0 100644 --- a/piccolo/columns/__init__.py +++ b/piccolo/columns/__init__.py @@ -27,4 +27,5 @@ Varchar, ) from .combination import And, Or, Where +from .m2m import M2M from .reference import LazyTableReference diff --git a/piccolo/columns/m2m.py b/piccolo/columns/m2m.py new file mode 100644 index 000000000..9299ce2f7 --- /dev/null +++ b/piccolo/columns/m2m.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +import inspect +import typing as t +from dataclasses import dataclass + +from piccolo.columns.base import Selectable +from piccolo.columns.column_types import Column, ForeignKey, LazyTableReference + +if t.TYPE_CHECKING: + from piccolo.table import Table + + +class M2MSelect(Selectable): + """ + This is a subquery used within a select to fetch data via an M2M table. + """ + + def __init__(self, column: Column, m2m: M2M): + """ + :param column: + Which column to include from the related table. + + """ + self.column = column + self.m2m = m2m + + def get_select_string(self, engine_type: str, just_alias=False) -> str: + m2m_table_name = self.m2m._meta.resolved_joining_table._meta.tablename + m2m_relationship_name = self.m2m._meta.name + + fk_1 = self.m2m._meta.foreign_key_columns[0] + fk_1_name = fk_1._meta.db_column_name + table_1 = fk_1._foreign_key_meta.resolved_references + table_1_name = table_1._meta.tablename + table_1_pk_name = table_1._meta.primary_key._meta.db_column_name + + fk_2 = self.m2m._meta.foreign_key_columns[1] + fk_2_name = fk_2._meta.db_column_name + table_2 = fk_2._foreign_key_meta.resolved_references + table_2_name = table_2._meta.tablename + table_2_pk_name = table_2._meta.primary_key._meta.db_column_name + + column_name = self.column._meta.db_column_name + + return f""" + ARRAY( + SELECT + inner_{table_2_name}.{column_name} + from {m2m_table_name} + join {table_1_name} inner_{table_1_name} on ( + {m2m_table_name}.{fk_1_name} = inner_{table_1_name}.{table_1_pk_name} + ) + join {table_2_name} inner_{table_2_name} on ( + {m2m_table_name}.{fk_2_name} = inner_{table_2_name}.{table_2_pk_name} + ) + where {m2m_table_name}.{fk_1_name} = {table_1_name}.{table_1_pk_name} + ) as {m2m_relationship_name} + """ # noqa: E501 + + +@dataclass +class M2MMeta: + joining_table: t.Union[t.Type[Table], LazyTableReference] + _foreign_key_columns: t.Optional[t.List[ForeignKey]] = None + + # Set by the Table Metaclass: + _name: t.Optional[str] = None + _table: t.Optional[t.Type[Table]] = None + + @property + def name(self) -> str: + if not self._name: + raise ValueError( + "`_name` isn't defined - the Table Metaclass should set it." + ) + return self._name + + @property + def resolved_joining_table(self) -> t.Type[Table]: + """ + Evaluates the ``joining_table`` attribute if it's a + ``LazyTableReference``, raising a ``ValueError`` if it fails, otherwise + returns a ``Table`` subclass. + """ + from piccolo.table import Table + + if isinstance(self.joining_table, LazyTableReference): + return self.joining_table.resolve() + elif inspect.isclass(self.joining_table) and issubclass( + self.joining_table, Table + ): + return self.joining_table + else: + raise ValueError( + "The joining_table attribute is neither a Table subclass or a " + "LazyTableReference instance." + ) + + @property + def foreign_key_columns(self) -> t.List[ForeignKey]: + if not self._foreign_key_columns: + self._foreign_key_columns = ( + self.resolved_joining_table._meta.foreign_key_columns[0:2] + ) + return self._foreign_key_columns + + +class M2M: + def __init__( + self, + joining_table: t.Union[t.Type[Table], LazyTableReference], + foreign_key_columns: t.Optional[t.List[ForeignKey]] = None, + ): + """ + :param joining_table: + A ``Table`` containing two ``ForeignKey`` columns. + :param foreign_key_columns: + If for some reason your joining table has more than two foreign key + columns, you can explicitly specify which two are relevant. + + """ + if foreign_key_columns: + if len(foreign_key_columns) != 2 or not all( + isinstance(i, ForeignKey) for i in foreign_key_columns + ): + raise ValueError("You must specify two ForeignKey columns.") + + self._meta = M2MMeta( + joining_table=joining_table, + _foreign_key_columns=foreign_key_columns, + ) + + def __call__(self, column: Column) -> Selectable: + """ + :param column: + Which column to include from the related table. + + """ + return M2MSelect(column, m2m=self) diff --git a/piccolo/table.py b/piccolo/table.py index 6d963a0bb..5849cbc7e 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -16,6 +16,7 @@ ) from piccolo.columns.defaults.base import Default from piccolo.columns.indexes import IndexMethod +from piccolo.columns.m2m import M2M from piccolo.columns.readable import Readable from piccolo.columns.reference import LAZY_COLUMN_REFERENCES from piccolo.engine import Engine, engine_finder @@ -215,6 +216,10 @@ def __init_subclass__( if isinstance(column, (JSON, JSONB)): json_columns.append(column) + if isinstance(attribute, M2M): + attribute._meta._name = attribute_name + attribute._meta._table = cls + if not primary_key: primary_key = cls._create_serial_primary_key() setattr(cls, "id", primary_key) diff --git a/tests/columns/test_m2m.py b/tests/columns/test_m2m.py new file mode 100644 index 000000000..2d94a1ef5 --- /dev/null +++ b/tests/columns/test_m2m.py @@ -0,0 +1,69 @@ +from unittest import TestCase + +from piccolo.columns.column_types import ( + ForeignKey, + LazyTableReference, + Varchar, +) +from piccolo.columns.m2m import M2M +from piccolo.table import Table, create_tables, drop_tables +from tests.base import postgres_only + + +class Band(Table): + name = Varchar() + genres = M2M( + LazyTableReference("GenreToBand", module_path="tests.columns.test_m2m") + ) + + +class Genre(Table): + name = Varchar() + + +class GenreToBand(Table): + band = ForeignKey(Band) + genre = ForeignKey(Genre) + + +TABLES = [Band, Genre, GenreToBand] + + +@postgres_only +class TestM2M(TestCase): + def setUp(self): + create_tables(*TABLES, if_not_exists=True) + + Band.insert( + Band(name="Pythonistas"), + Band(name="Rustaceans"), + Band(name="C-Sharps"), + ).run_sync() + + Genre.insert( + Genre(name="Rock"), + Genre(name="Folk"), + Genre(name="Classical"), + ).run_sync() + + GenreToBand.insert( + GenreToBand(genre=1, band=1), + GenreToBand(genre=2, band=1), + GenreToBand(genre=2, band=2), + GenreToBand(genre=1, band=3), + GenreToBand(genre=3, band=3), + ).run_sync() + + def tearDown(self): + drop_tables(*TABLES) + + def test_m2m(self): + response = Band.select(Band.name, Band.genres(Genre.name)).run_sync() + self.assertEqual( + response, + [ + {"name": "Pythonistas", "genres": ["Rock", "Folk"]}, + {"name": "Rustaceans", "genres": ["Folk"]}, + {"name": "C-Sharps", "genres": ["Rock", "Classical"]}, + ], + ) From 9437d851af961fc178d2afae02a19f8eefa7b47c Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 17 Dec 2021 11:11:14 +0000 Subject: [PATCH 175/727] use __name__ instead of hardcoding module --- tests/columns/test_m2m.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/columns/test_m2m.py b/tests/columns/test_m2m.py index 2d94a1ef5..f835ca343 100644 --- a/tests/columns/test_m2m.py +++ b/tests/columns/test_m2m.py @@ -12,9 +12,7 @@ class Band(Table): name = Varchar() - genres = M2M( - LazyTableReference("GenreToBand", module_path="tests.columns.test_m2m") - ) + genres = M2M(LazyTableReference("GenreToBand", module_path=__name__)) class Genre(Table): From 7a19c26ce89745597fed190b578df4176ac0457b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 17 Dec 2021 12:42:56 +0000 Subject: [PATCH 176/727] started adding m2m docs --- docs/src/piccolo/schema/index.rst | 1 + docs/src/piccolo/schema/m2m.rst | 82 +++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 docs/src/piccolo/schema/m2m.rst diff --git a/docs/src/piccolo/schema/index.rst b/docs/src/piccolo/schema/index.rst index e1450745f..36ea8b807 100644 --- a/docs/src/piccolo/schema/index.rst +++ b/docs/src/piccolo/schema/index.rst @@ -8,4 +8,5 @@ The schema is how you define your database tables, columns and relationships. ./defining ./column_types + ./m2m ./advanced diff --git a/docs/src/piccolo/schema/m2m.rst b/docs/src/piccolo/schema/m2m.rst new file mode 100644 index 000000000..09d2db052 --- /dev/null +++ b/docs/src/piccolo/schema/m2m.rst @@ -0,0 +1,82 @@ +.. currentmodule:: piccolo.columns.m2m + +### +M2M +### + +Sometimes in database design you need `many-to-many (M2M) `_ +relationships. + +For example, we might have our ``Band`` table, and want to describe what genre +each band belongs to. Each band may have multiple genres, so a ``ForeignKey`` +alone won't suffice. Our options are using an ``Array`` / ``JSON`` / ``JSONB`` +column, or using an ``M2M`` relationship. + +Postgres and SQLite don't natively support ``M2M`` relationships - we create +them using a joining table which has foreign keys to each of the related tables +(in our example, ``Genre`` and ``Band``). + +Take this schema as an example: + +.. code-block:: python + + from piccolo.columns.column_types import ( + ForeignKey, + LazyTableReference, + Varchar + ) + from piccolo.columns.m2m import M2M + + + class Band(Table): + name = Varchar() + genres = M2M(LazyTableReference("GenreToBand", module_path=__name__)) + + + class Genre(Table): + name = Varchar() + + + # This is our joining table: + class GenreToBand(Table): + band = ForeignKey(Band) + genre = ForeignKey(Genre) + + +.. note:: + We use ``LazyTableReference`` because when Python evaluates ``Band``, + the ``GenreToBand`` class doesn't exist yet. + +By using ``M2M`` it unlocks some powerful and convenient features. + +------------------------------------------------------------------------------- + +Select queries +============== + +If we want to select each band, along with a list of genres, we can do this: + +.. code-block:: python + + >>> await Band.select(Band.name, Band.genres(Genre.name)) + [ + {"name": "Pythonistas", "genres": ["Rock", "Folk"]}, + {"name": "Rustaceans", "genres": ["Folk"]}, + {"name": "C-Sharps", "genres": ["Rock", "Classical"]}, + ] + +------------------------------------------------------------------------------- + +Objects queries +=============== + +add_related +----------- + +... + + +get_related +----------- + +... From 6dad3273f2a1e884447d679482d8c864440f1551 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 17 Dec 2021 12:44:38 +0000 Subject: [PATCH 177/727] improved wording in the docs --- docs/src/piccolo/schema/m2m.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/piccolo/schema/m2m.rst b/docs/src/piccolo/schema/m2m.rst index 09d2db052..d519df058 100644 --- a/docs/src/piccolo/schema/m2m.rst +++ b/docs/src/piccolo/schema/m2m.rst @@ -7,8 +7,8 @@ M2M Sometimes in database design you need `many-to-many (M2M) `_ relationships. -For example, we might have our ``Band`` table, and want to describe what genre -each band belongs to. Each band may have multiple genres, so a ``ForeignKey`` +For example, we might have our ``Band`` table, and want to describe which genres +each band belongs to. As each band can have multiple genres, a ``ForeignKey`` alone won't suffice. Our options are using an ``Array`` / ``JSON`` / ``JSONB`` column, or using an ``M2M`` relationship. From df9e15694f9b23b96c99d4182dbdfceb0e00e2e7 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 17 Dec 2021 14:07:19 +0000 Subject: [PATCH 178/727] added `add_m2m` and `get_m2m` methods to `Table` --- piccolo/columns/m2m.py | 138 ++++++++++++++++++++++++++++++++++++++ piccolo/table.py | 49 +++++++++++++- tests/columns/test_m2m.py | 35 +++++++++- 3 files changed, 219 insertions(+), 3 deletions(-) diff --git a/piccolo/columns/m2m.py b/piccolo/columns/m2m.py index 9299ce2f7..2a67574a9 100644 --- a/piccolo/columns/m2m.py +++ b/piccolo/columns/m2m.py @@ -6,6 +6,7 @@ from piccolo.columns.base import Selectable from piccolo.columns.column_types import Column, ForeignKey, LazyTableReference +from piccolo.utils.sync import run_sync if t.TYPE_CHECKING: from piccolo.table import Table @@ -76,6 +77,14 @@ def name(self) -> str: ) return self._name + @property + def table(self) -> t.Type[Table]: + if not self._table: + raise ValueError( + "`_table` isn't defined - the Table Metaclass should set it." + ) + return self._table + @property def resolved_joining_table(self) -> t.Type[Table]: """ @@ -105,6 +114,135 @@ def foreign_key_columns(self) -> t.List[ForeignKey]: ) return self._foreign_key_columns + @property + def primary_foreign_key(self): + """ + The joining table has two foreign keys. We need a way to distinguish + between them. The primary is the one which points to the table with + ``M2M`` defined on it. In this example the primary foreign key is the + one which points to ``Band``: + + .. code-block:: python + + class Band(Table): + name = Varchar() + genres = M2M( + LazyTableReference("GenreToBand", module_path=__name__) + ) + + class Genre(Table): + name = Varchar() + + class GenreToBand(Table): + band = ForeignKey(Band) # primary + genre = ForeignKey(Genre) # secondary + + The secondary foreign key is the one which points to ``Genre``. + + """ + for fk_column in self.foreign_key_columns: + if fk_column._foreign_key_meta.resolved_references == self.table: + return fk_column + + raise ValueError("No matching foreign key column found!") + + @property + def secondary_foreign_key(self): + """ + See ``primary_foreign_key``. + """ + for fk_column in self.foreign_key_columns: + if fk_column._foreign_key_meta.resolved_references != self.table: + return fk_column + + raise ValueError("No matching foreign key column found!") + + +@dataclass +class M2MAddRelated: + + target_row: Table + m2m: M2M + rows: t.Sequence[Table] + extra_column_values: t.Dict[t.Union[Column, str], t.Any] + + async def run(self): + rows = self.rows + unsaved = [i for i in rows if not i._exists_in_db] + + async with rows[0]._meta.db.transaction(): + await rows[0].__class__.insert(*unsaved) + + joining_table = self.m2m._meta.resolved_joining_table + + joining_table_rows = [] + + for row in rows: + joining_table_row = joining_table(**self.extra_column_values) + setattr( + joining_table_row, + self.m2m._meta.primary_foreign_key._meta.name, + getattr( + self.target_row, + self.target_row._meta.primary_key._meta.name, + ), + ) + setattr( + joining_table_row, + self.m2m._meta.secondary_foreign_key._meta.name, + getattr( + row, + row._meta.primary_key._meta.name, + ), + ) + joining_table_rows.append(joining_table_row) + + return await joining_table.insert(*joining_table_rows) + + def run_sync(self): + return run_sync(self.run()) + + def __await__(self): + return self.run().__await__() + + +@dataclass +class M2MGetRelated: + + row: Table + m2m: M2M + + async def run(self): + joining_table = self.m2m._meta.resolved_joining_table + + async with self.row._meta.db.transaction(): + secondary_table = ( + self.m2m._meta.secondary_foreign_key._foreign_key_meta.resolved_references # noqa: E501 + ) + + ids = ( + await joining_table.select( + getattr( + self.m2m._meta.secondary_foreign_key, + secondary_table._meta.primary_key._meta.name, + ) + ) + .where(self.m2m._meta.primary_foreign_key == self.row) + .output(as_list=True) + ) + + results = await secondary_table.objects().where( + secondary_table._meta.primary_key.is_in(ids) + ) + + return results + + def run_sync(self): + return run_sync(self.run()) + + def __await__(self): + return self.run().__await__() + class M2M: def __init__( diff --git a/piccolo/table.py b/piccolo/table.py index 5849cbc7e..25fc8a3eb 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -16,7 +16,7 @@ ) from piccolo.columns.defaults.base import Default from piccolo.columns.indexes import IndexMethod -from piccolo.columns.m2m import M2M +from piccolo.columns.m2m import M2M, M2MAddRelated, M2MGetRelated from piccolo.columns.readable import Readable from piccolo.columns.reference import LAZY_COLUMN_REFERENCES from piccolo.engine import Engine, engine_finder @@ -371,7 +371,7 @@ def remove(self) -> Delete: def get_related(self, foreign_key: t.Union[ForeignKey, str]) -> Objects: """ - Used to fetch a Table instance, for the target of a foreign key. + Used to fetch a ``Table`` instance, for the target of a foreign key. .. code-block:: python @@ -412,6 +412,51 @@ def get_related(self, foreign_key: t.Union[ForeignKey, str]) -> Objects: .first() ) + # TODO - I might merge this with ``get_related``. + def get_m2m(self, m2m: M2M) -> M2MGetRelated: + """ + Get all matching rows via the join table. + + .. code-block:: python + + >>> band = await Band.objects().get(name="Pythonistas") + >>> await band.get_m2m(Band.genres) + [, ] + + """ + return M2MGetRelated(row=self, m2m=m2m) + + def add_m2m( + self, + *rows: Table, + m2m: M2M, + extra_column_values: t.Dict[t.Union[Column, str], t.Any] = {}, + ) -> M2MAddRelated: + """ + Save the row if it doesn't already exist in the database, and insert + an entry into the joining table. + + .. code-block:: python + + >>> band = await Band.objects().get(name="Pythonistas") + >>> await band.add_m2m( + >>> Genre(name="Punk rock"), m2m=Band.genres + >>> ) + + + :param extra_column_values: + If the joining table has additional columns besides the two + required foreign keys, you can specify the values for those + additional columns. + + """ + return M2MAddRelated( + target_row=self, + rows=rows, + m2m=m2m, + extra_column_values=extra_column_values, + ) + def to_dict(self, *columns: Column) -> t.Dict[str, t.Any]: """ A convenience method which returns a dictionary, mapping column names diff --git a/tests/columns/test_m2m.py b/tests/columns/test_m2m.py index f835ca343..96ed20164 100644 --- a/tests/columns/test_m2m.py +++ b/tests/columns/test_m2m.py @@ -55,7 +55,7 @@ def setUp(self): def tearDown(self): drop_tables(*TABLES) - def test_m2m(self): + def test_select(self): response = Band.select(Band.name, Band.genres(Genre.name)).run_sync() self.assertEqual( response, @@ -65,3 +65,36 @@ def test_m2m(self): {"name": "C-Sharps", "genres": ["Rock", "Classical"]}, ], ) + + def test_add_m2m(self): + """ + Make sure we can add items to the joining table. + """ + band: Band = Band.objects().get(Band.name == "Pythonistas").run_sync() + band.add_m2m(Genre(name="Punk Rock"), m2m=Band.genres).run_sync() + + self.assertTrue( + Genre.exists().where(Genre.name == "Punk Rock").run_sync() + ) + + self.assertEqual( + GenreToBand.count() + .where( + GenreToBand.band.name == "Pythonistas", + GenreToBand.genre.name == "Punk Rock", + ) + .run_sync(), + 1, + ) + + def test_get_m2m(self): + """ + Make sure we can get related items via the joining table. + """ + band: Band = Band.objects().get(Band.name == "Pythonistas").run_sync() + + genres = band.get_m2m(Band.genres).run_sync() + + self.assertTrue(all([isinstance(i, Table) for i in genres])) + + self.assertEqual([i.name for i in genres], ["Rock", "Folk"]) From cd15b85a4e13d1c7c45b899e8af4df96a36b185a Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 17 Dec 2021 14:31:08 +0000 Subject: [PATCH 179/727] added docs for `add_m2m` and `get_m2m` --- docs/src/piccolo/schema/m2m.rst | 17 +++++++++++------ piccolo/columns/m2m.py | 4 ++-- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/docs/src/piccolo/schema/m2m.rst b/docs/src/piccolo/schema/m2m.rst index d519df058..86f4bf3a8 100644 --- a/docs/src/piccolo/schema/m2m.rst +++ b/docs/src/piccolo/schema/m2m.rst @@ -70,13 +70,18 @@ If we want to select each band, along with a list of genres, we can do this: Objects queries =============== -add_related ------------ +Piccolo makes it easy working with objects and ``M2M`` relationship. -... +add_m2m +------- -get_related ------------ +.. currentmodule:: piccolo.table -... +.. automethod:: Table.add_m2m + + +get_m2m +------- + +.. automethod:: Table.get_m2m diff --git a/piccolo/columns/m2m.py b/piccolo/columns/m2m.py index 2a67574a9..96d706d86 100644 --- a/piccolo/columns/m2m.py +++ b/piccolo/columns/m2m.py @@ -30,13 +30,13 @@ def get_select_string(self, engine_type: str, just_alias=False) -> str: m2m_table_name = self.m2m._meta.resolved_joining_table._meta.tablename m2m_relationship_name = self.m2m._meta.name - fk_1 = self.m2m._meta.foreign_key_columns[0] + fk_1 = self.m2m._meta.primary_foreign_key fk_1_name = fk_1._meta.db_column_name table_1 = fk_1._foreign_key_meta.resolved_references table_1_name = table_1._meta.tablename table_1_pk_name = table_1._meta.primary_key._meta.db_column_name - fk_2 = self.m2m._meta.foreign_key_columns[1] + fk_2 = self.m2m._meta.secondary_foreign_key fk_2_name = fk_2._meta.db_column_name table_2 = fk_2._foreign_key_meta.resolved_references table_2_name = table_2._meta.tablename From f8c36bb2c35ae7e3c6acc7e1cfb94fa64cdcfcae Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 17 Dec 2021 14:52:07 +0000 Subject: [PATCH 180/727] storing `m2m_relationships` in `TableMeta` --- piccolo/table.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/piccolo/table.py b/piccolo/table.py index 25fc8a3eb..9564f8064 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -67,6 +67,7 @@ class TableMeta: tags: t.List[str] = field(default_factory=list) help_text: t.Optional[str] = None _db: t.Optional[Engine] = None + m2m_relationships: t.List[M2M] = field(default_factory=list) # Records reverse foreign key relationships - i.e. when the current table # is the target of a foreign key. Used by external libraries such as @@ -179,6 +180,7 @@ def __init_subclass__( secret_columns: t.List[Secret] = [] json_columns: t.List[t.Union[JSON, JSONB]] = [] primary_key: t.Optional[Column] = None + m2m_relationships: t.List[M2M] = [] attribute_names = itertools.chain( *[i.__dict__.keys() for i in reversed(cls.__mro__)] @@ -219,6 +221,7 @@ def __init_subclass__( if isinstance(attribute, M2M): attribute._meta._name = attribute_name attribute._meta._table = cls + m2m_relationships.append(attribute) if not primary_key: primary_key = cls._create_serial_primary_key() @@ -239,6 +242,7 @@ def __init_subclass__( tags=tags, help_text=help_text, _db=db, + m2m_relationships=m2m_relationships, ) for foreign_key_column in foreign_key_columns: From 9e332088fa341e4c9d5a87f607c3b308822566e9 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 17 Dec 2021 16:12:00 +0000 Subject: [PATCH 181/727] update m2m select docs --- docs/src/piccolo/schema/m2m.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/src/piccolo/schema/m2m.rst b/docs/src/piccolo/schema/m2m.rst index 86f4bf3a8..4941ccb12 100644 --- a/docs/src/piccolo/schema/m2m.rst +++ b/docs/src/piccolo/schema/m2m.rst @@ -65,6 +65,17 @@ If we want to select each band, along with a list of genres, we can do this: {"name": "C-Sharps", "genres": ["Rock", "Classical"]}, ] +You can request whichever column you like from the related table: + +.. code-block:: python + + >>> await Band.select(Band.name, Band.genres(Genre.id)) + [ + {"name": "Pythonistas", "genres": [1, 2]}, + {"name": "Rustaceans", "genres": [2]}, + {"name": "C-Sharps", "genres": [1, 3]}, + ] + ------------------------------------------------------------------------------- Objects queries From fc5c954bc2629dfdde216fd885eb041ceda25738 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 17 Dec 2021 17:02:16 +0000 Subject: [PATCH 182/727] improved M2M intro docs --- docs/src/piccolo/schema/m2m.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/src/piccolo/schema/m2m.rst b/docs/src/piccolo/schema/m2m.rst index 4941ccb12..15ee10639 100644 --- a/docs/src/piccolo/schema/m2m.rst +++ b/docs/src/piccolo/schema/m2m.rst @@ -7,9 +7,9 @@ M2M Sometimes in database design you need `many-to-many (M2M) `_ relationships. -For example, we might have our ``Band`` table, and want to describe which genres -each band belongs to. As each band can have multiple genres, a ``ForeignKey`` -alone won't suffice. Our options are using an ``Array`` / ``JSON`` / ``JSONB`` +For example, we might have our ``Band`` table, and want to describe which genres of music +each band belongs to (e.g. rock and electronic). As each band can have multiple genres, a ``ForeignKey`` +on the ``Band`` table won't suffice. Our options are using an ``Array`` / ``JSON`` / ``JSONB`` column, or using an ``M2M`` relationship. Postgres and SQLite don't natively support ``M2M`` relationships - we create From 0c846d9b9844b1d6975cc49ca72c7f323361c6ce Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 17 Dec 2021 17:22:45 +0000 Subject: [PATCH 183/727] added image for joining table --- docs/src/piccolo/schema/images/m2m.png | Bin 0 -> 38886 bytes docs/src/piccolo/schema/m2m.rst | 6 +++++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 docs/src/piccolo/schema/images/m2m.png diff --git a/docs/src/piccolo/schema/images/m2m.png b/docs/src/piccolo/schema/images/m2m.png new file mode 100644 index 0000000000000000000000000000000000000000..4831569babe13abaab0114fc1e3b505d42c388e1 GIT binary patch literal 38886 zcmeFZ1yq$y*e(nRN{FD8g1|OV0SQT^Q4rX4cS%Wimw<|(2qGmdlADxn1Vj{sO?OCl zNJ^i1Hu}Ds_j|wp`_?&Ut+USm|GI?j>}TScx#ym1uKSt^kdqN3#HYl^z`!7s5EoIv zz&NFdfq|8Qdm5yue{oX9z`*o0eegg|;=u!$oSn6?sf7^+#;rG=AxKFqVbaDLGySkD z^8V+jBl2Y!iUYdn-elpCMVV1#!5k~evvf*l-ma&da^`~d?ySQ*E)%tIS zSfrynCLTleU)JM-%@6pGxXbPGTQM}x{JgHn>E{yd^0-0no_&=-5RGVMZJFNBb2C{D zCY$jN3~&#L+j&W0*SxN7Xj4}BopNPN(Dqx?;@K(T=u>TP?w>u6 z`@#TU{wZr9p2mY!;im~stdq)qy02BbI4v{eIG(cJyFy_5Y^TciB5{WQ-tJ(oakd@q ztC~}<%?cuhahQvr95x!a*khD(`>dNQcO<)g6Yh*hp2|%Vay~s7Bj8uJ3iDewss{Kdee#K1v)#=ua)r26Z#4(6@j&tQRP*f79P z3{vpV*xwWb`}gy1;5YO~9Q*?%9sg!yWxK<3hmGeBH|!2K_{qb@2FxN6>rbvziixNG ze8$Rvvd>Di27q69Hsb2`7#IX3&_7I!=-7)G7^h=RmDC*6q@{QatSy=K4Xqy;F}qmW zK)EpZTzJ4oOCtw;n2V)_l|7FO|5elzJm52w%yJcmdc?t;|Eij_9PEL$oe_+SnU$IK zssKI=2II3cH0DtddH6d!_#gk(#|{oQJS;5E&d$uv?9A47CMV zvUjy|(05_7vcGm*$e(gVjO-2UOl=%Yt*u~Cx%!W+9Ub_uUWF?9>(6mJja*FsuF1;& z_iKR{WP!e6xx>uL@^9H3OpX7OY|uCVX*SSD($asu*oo>~^lebtEG_@FLVE{MC-4ft zyX0SU+AF!*7_lfA*;_l>85oH=8Cf}8`?t1(M*Ke)^lzDL9$PzD+dsDcPqO~?_3^8t zhV;0n`JjH}d0=f}ZKr6XZ(t;FhY!`Y=t+N%%kk54rY=SnY9gkVMppJ<3k*u zZ~ir>I(km--_@aVqJ4vw6B;re2}^wwBUMvFhsVd&{WI~O&qMue;RuMh`*aUQefUvQ#Dd=ifm>gt_ush6mSeZHaSpMzNznlD5 zslSWVH$e5Ijh(fjqk)m(pX|Sr|MPs#kBzJZAa}}ihmGkDx6&VXs_~};)C2#_&&L9} z9}ClyopPLdvak49{`w@W6)p4*LI90qBpR z74g^X<^lNOH+&GVR}i}olMsEtb>|)z7ZV#ZFwTE{?qAD6`2+A$Mgr{U+5S%co$I)( z(EK^7Cy!IXY9W9R`t8rkBY@BDr(c1AA%r0zBCO{Dxj~-a)bv;(vpfCu4e_ktiB5%MkJ{lEu@s+HShnLe8$~zCwKO#!Hlk`wN3nr@f z=7x4iO7D@^x%h>+{{G?q{tVug3HI-CnyviXj@y^ED`tjP==-!)Uio5R;liH$^+S%y zho^fWChoit1|~M~Uq50oYDoog(I0#wgoTBzh?i)Kdm_IOYXkDH!cfJAJi(|{!g_r9 zWJSbMKiU7T75a1?Cd3+!LFtdy7?6yuxPSI!3LY-86y1|AS57nrYTl_6DgS@-K<{nB z$(bKuLIVj4!fJ>cNd0LMi~oCE73hSJN(jRY3iNR{D-z=OcHB?tl3aZa#w7XSlQ$R{ zwkaydroaYeNu9jMfw+nQ zL4FrHQ;9x&A)O%cp6A|UKJ=;#;ZO7!6xU);G}%rAB&J!@28p3pC51;UWdx63zlvsL zLNWxzQd1jCAAX*wiXPK9L+YEo4w|_We>DY(vDKa&H0Zqp-k&v`-nI25`kUTk0*Rd^ z!C$VSSCxv5o61IqbUpE=X*lzA6`Kns%lYk{Tm#>`2j6Ntig@}}a1_3;q7u!TIEDVQ zun=aSe>jTQwyPF-43BgT_c4Wm5FU3c-f4B#2UtJqNFC&i*7^O6+l* zD35=8!zzEN;`@z+^SZK~g=D--Nvb7%lcCwzine$KB+@dJss4$q$vPz;2t>RjsLGVQqTn<3a>ic!M{ z9owStxW9cLu~4EDe)8=ov*GmD2Z=@=k1MK|q7jNWtIwnSL_%CGK2eK=Zz?`ecx-(VU=^I^S-$HH;jcqHaW@Nc? zUB-N<)HcrxjD%CKwsU_yk>KU5bpj@h!aPM;{%(G6<&x!i*P)x^RM-#M1k8dV;~rB+Af>js!P5E%z4@s)eAe6oMuYjh2VRcXUx7w-EnH9cAhmT%h#*LqPk~iC<04~ z>)PXZ^clBzZ<5Fj^kFlOg4d{~byfGu7hZI0G4v4{IjuON!!Pgl?B(-`s;vQYtMKPD zL9~JbeU(RtdxO^Adz#y3pd0dZ`ZpOEXJYMNA53%C+@YprwyxTY*ef+?d~Ju>FE2)I z^x*kfyp>kA6U7GzH9g-k`nIV&@T%tMV5O(V+q)!q+J%` z{UMF#hWy^hcJ)lpTz|$#gU0fyhL;pESS*5FEa~YMV_&GO3-#-tmnoaXyQXd~4RZHo z#WRL;k9u?}`Li7D)f`csT~b8${HXQwo5|Z3s663Mrm+>srN|ch%ye^uxjA>O7~KsK z7Dqw05~PbSSmZ^G!U-?mWyMV{9M$)yp4Rbtq(v7f*Kb{OuvmZ;uy5A&+zgjg`S`f~ zYRnyj=lnsD1qCMC>QAKv@a@|GXC&-uj$e|~d{4G!7iEFx_HeHU zE7`K??)XcVA!2^QBgy1K%R#R3yriV07s%Z_hq(^DIJeqzrzJ{{p8MO&!`B4XPudS7 zn963(2US^?opq%-R#UwxsRD65dtpRukCa!tr=tbuQ!KpNjRuP?&u4NN%2vr!9?mGn znV-So6}xQvlzyo!^r4dSd{;(!6HMpYdd&#!$IH^H#ZWSqb$z4pGXt?lP_mZ}YfRo8>v zO$mYlU%2{T_zZX}A4c;WG}vXXi@VwZDIG#(g4)`zqT3oq}K^ zIsy~2!@026B3Nem^|2+%O_|%yUM+atAH;}CbkvV%y!U_VC1mDS^QgVnQ7bYFFR~sL zaC6j3pyGI}7ag_TA<06xrAj>YGQJZrU&c_wvT|Z2cJm1%T-T(1*ZjVNF+;P5T`5gw zf3DSYcM^kJ}n=6@1n8&!;fgb zWz(VT+1BPKJ8!jQy%{x0)%w=bpyl-UcUCU(9l`#7F{{;? zs<~X!KQz_U9#^?-=HHNxe#(8*QN12#rZvuder|rKtiUv#76%jCPB+nTjLrG{x83R% z&+|FQPSR(HT-=S7?)A?C?!n3YM6&}GiZnXybW=YpDlKx6xPJTG zNmdy@@|-wCI(PV^&J_IIl8F}!Pp_BWjk~7;Ry(7d7Uy9 zIN|ml0#L=i97~1{f9$kCVxu*z<|vm)ieE zlgIu4)fOaPuJPfZMX!qO8UR(7oLd7=AT4H)IKdqEfsu4BeaOJBI&eLTxT=&&}VjJ^dW zUOLftnqZj99BUpkq1S5|02&`Lgd6RH-cMx^Ox4_e*>hq@BA{{l-&Sn*h1r3k%sElv z-&Q;c2Apm*jPk@N5P|n!`*FiQ;KZAFfB~P3QMiWwB9lJgMQDZ$na`v5b|{M+%Z_qS zW;s)I+e z*b4W|_X*m+soIPVJ8|8+$h29ZQgl=H7#Xq_KVAliIz4KzC2}eOcOog8)@Abi% za^?Akfig=^7xw0pSq8A#Xr!3p1O~LTu;>M|WEtU+Otw{W=J4sh)}=wLkfh?JY*AaH zL5+ymI$A0V!xPIXvG`k|hyFYCFb3@%ug~l9<;$XqX{9E~m9>ccG0!o*%ofZ8i{Wvc zWwjcj6Q22r5DyO90DF;T{RQXj79J$lV~y z3egrZGV6Do<`Gptzff^GFAtTT>#uSz0TB9|j>k{Y!Ae(qt;W0)TbN0IP&U16rTLyy zjao!|Znb6)u@KoN-;R;d-b?20cVD-C&d}rm^ws|qk0dEGV8^xbHMJ9fm4h$NU$Z!H z3)TKC@`w)2B3&6m+D%d^Iaj9odL!v#xLO3M$>GjxFi3zclOl_ zN){RgX^~C6W9MG4C;0mM-rY9j2Ja2q!P2g9lJ-hb?urVqS82b_b?ml6$bA4ZNL%E) zwDfL`WW{Qqi?u`9>~)5dAYYfowo0c+^?bMjtb1%QY<57b{($3hUb?ck=c|e#hfe9O z74M_LO|Aw{V%`&TyA?lqjVKTl*r6rQ&|beC?ReXXUrDoC-TYZlbdy)`XAo^f(Iht{p=PaCaJQQoW2w2A&X&-FEjJRk#Gb202lHKvoA5 z;M#)CmC@1Vs_l^%2h}^%R8#^UrO@8y^x2Dg&_enCf&aj(OYB#-{;}`ZJC)HYb~bRx~@}X6>Ck5jCTXO(Whp} z3YIBhYSJls!k_C6XA~S4xal>O^-_#39d4|Ne!H>pO3BfGYiGisDp$Lr=gX^06I{b~ z%}m0u6tbJ~E=g3o+-@7QYX14>(EZ<<7R<7?%h^Ziz57E%k`Q7h@n-Y$pPOvbUo!8d zD&fZzmYfb6q&rY)iM_|3zW$vZc{bvp&NMB~7+X=t&l!F0;Swa%MImm7Xxwuzry5A3 zp{Etk@Co=%OIDrd5<$VDv*@I_RsArCGMqQra?Le3BY$%FRD*W6q#_xr3@Wgck@OVWm2X37 zOD#O46+&h^u{B@J44Qfzmxu19yAKqa*5>O*ahQhwbG?2+EH1&StPV>lU~Jhb7o|%* z>!mB@)e4VX`8H6JbUSKI;^tM2cS%J2ZX0=C2fJkhz%Q6LN3syLa+tJUt^h#V7kT@n z`RGHmjCrI~PM%m(LaFKC%JW{+kz3MVGxO~=Gg1ni;Yg97%lA6sG~aGPc-X0dOtnv0 zRom!hoI|MPP-(m61TFf|2*ES}{nF*@?7h_5{{HCAd8ByuJHbJfmUqioX}}$7%ExS3 z)vVC64!I@l(n8A_x4+W*mfwfLs6wRU{9@UV+h(=oqrM7fYbrsnO8O58zG>bE3%Xlt z%`BP|UxF;hKL=Py^N@?y(D;{p?8G2mk~K>1nj z+soK;@5(R5Qut^v$N~6Nk#FcR`bDy<$XvB#VSi_Rg8%8kaSPHpu`&*iemHA$NWeNj zmpd*{^lwS0sFYg-kQG_h%=D+{3&GdLIah2do0ro}Qoy>p)LLrw%{&!$o9ykH%1kIe z0fBUtlG{IQ27w()DK%%PeU8d#JymPjEv}0oW#)Yy$y)Iu+vf-wTFVs1QKErjtDlDE z!|QcF*a=W`_}lqJF@}4uyN)?(q1!&mg6t_K@zBzS-|Ut_Do$T3f)tiLO|nK_B#^wg zes}v-nA=CCutMLHtoCH6P=|Y}!%zHl1pIn?zKeLjfS=Y@_35DP>mR<--5odqXLp!F zLiHGQIslZ()>h~GCzWQ7B|$qKX^v5Ng6$+07Xe7~Jm>4*zza1^eJUvcvARc#iE(0d z&jX+((slPV`cz)i0ohl)u|c%q?_?hl0MstXPh3Q|%dhSrd*#u2473L^u`is5MZraa z3{F(E0?_u_p%?}Rdj9|Ah9<^S+)^+7&awtE(P&dp?imKZqB6$EOMkxr`1FE~cteWJ z*^J7Z_ZWqTUjKyq=v|y>Ah|$8C z?B=0aJ-5>*0n5e8oGQzey64J^Eyn1hC3Qxvf^zDM z+jkZr#PIjnBtL*<*LE?!thVXh_!Qh4xG=yRvq98(2XM}|A5Q?bWO?wU(p)jqa>s7b z9Pu{YuEDs#6KeGB1BMR=Zpf3XgkNJ%EJkJtV5_iHk*0P2M9sp$uK(ksLK+G01_}*g zNcM+$j{-G2Xi2s)|Ji0?sLkUYj?1AsQQX&-CD$82dt#ixR~T5P#Ny_x-AVcejKjs< zyTv&?F{97(zkBOuX&do8Kj{MuV}vyPG9uhLi&=WTEs5ggHu)53nRP9qP2i1k=+g$n z$AScgDOx;ye!vgC7w=sO*{raIwF@61e)QV$s~bj|ToNalE}MxjNKU0Z6pLfrViD6rNje{uCQxHIMa{!Q2bG~W3%cb>zW+qv*KiruaY~dcYaGV$U5(SviqP>+G!AGO<{O$u| z{eZ7c(ckcRLdX8YpxvaBy(W@13OmIs0Xym@RgDwJIOYxC0`!O5x;GFG}+v3MY3qWmJsjom>s%G{6TC?Tad#q^15 z4{$lNRzWDl#wToGcl7A6SI2FRPBp=;ODW#_(A6DmYKGa4{+^S7Ida2cyCAhWH#~@N zb7WkOX2T<_aR;zgivJ;l`%+Ql!kvzMU+O9)fT0buHHz%&*O)6;>t0-O1bn#d-fTiR zqbh{YX5@k++{%^C@o&g+FUmIiQ#(a18~|cHXrX<#AL*6aoikf@E1W{t;|^}ea@kD0 z-mUk~H{)!-T(XKv*do|AoMQ(Z*xwTX8f1O1xw67Qq+K1}O*O#KUvrur*sS;=T^r^L z!x5P;TQdR?iqxlM%@IuY5BBzV)mFr)9hHdrOe=g7LC|DyCc)e4U~fD4t_Wrfi-7#I z3)<;tFW+qo3=GuLo7ryP6Fx&diIoB9*z%d0eb)L_f!imTqj~H#wkWgye4~5qrwJ*; zG}FlD;}oz0A$oquUgi$bFDD}qu3;D&N)mmEU$R4Zc#oMbhsiC1^OI^LE{$P7_G~Q$FTeM2!$>^q`&ns-jf+mGPRb_IkTY}##l^d_S@7W{y zXApeLr4x%$2CoCsbxgl0s*_4(%wEbQV^qP$yS0BW{EPOw(ZP>1g0Q}|Tj=a{$U~o3 zhU@0V+aoI7nG$cmm>RzDWH_?ubYH2Wc!P02qb0X`cm78r*!$u%f77_T*`?@*fFdIV z2@fzMu;}TU+CJBw2w#fV2t9O5E%zE+Q0Sk^(bU$I>sE@MfOOV@hMdq20PzmV+Zk{>}%=GqIQNS5N}8wd60%!X@q zc*bUwmlK+(H@=7ZULq{wUHx>*GQo2TPbRH!5^|vcb`Ojg)l(?Shs7VfWU01U*W|S5 zYs0bULD9^T5Y62C!@*;8I^t~haReU5iRH!I4-pTO*q1I%M}{&==%-|BmM}+R)FsG} z%p=km!c%~oVT;Le}r5uAt2#3vLrNEk(D+_yN;0Q>@Bnq#8D8O>$^7i{V@^x zdJfYA4+Gqt*l}`J;7;ZXJ&b?XWW+jf5*CU-(F;sCh2~#C%P|3h#csQcjqY}cVNv5I zCm=1cxDdb&m>lQOAvGpI#J?_~gJghxL0&jU&;#_|!NByT_d0zd#RnUEzAIO?m%7+s zrWO0#3^CLw9~bO>=d7dFpqYzXe>l@c^7MjU9YQSv;0fY~nNpxG6*@=oYenrSE~Y;6 ze&=sA^h48!hY+7&cs#X5Qy7(5{LPc2*PEkd^gM*!zlk6rN}MLQRQzn2I3AU9{kN7w zg>*q_(@Id0UbwRr%A+55bGn_G|F~c?f1xTj#|4efq=&`JF@(jBx(5bqm9=od7=%2- z`p2VBnx{)-3z7zGIgcM`!1Cxd{@L>0^wmML8(%Kt0=g}l)j&|Trcvt(y6GoF=7^K7bYfvC(tV3G5pY2isAMCkh}1 zk%APAa0c{VN!$VifA^vLcd`=&{9o0ri8}c6$-jpc!35|D^`hW0)SlJnGfp6Hk@UXa zYX=Z~`RVuf5JR7UG7b5cf-Qsx^H3q0N7L*f==1BJ#GgzaTyA3z=aV$Mk3I-|kpAgI zOiRj%?pgy@7-}LReZq*ufN8lp6qG5Qu$OpX8f&*R@X&{Q4LCW5U@NZ36FkB<5-_sy zbe!nX2_YJwf0}ARUnb4fLA6UIX^ZSKhD1fNq4>WkFT05nWIJGIr1t zuGfLNiE+nBo=|m7f~x3{q0#705EcyHbZyqyKI6oz@`0)*)fGUpFrz|^ zAYYI5u+fJ!%ffe=C21RoGtQ=uAGmYF=kBFnhPndA3+8P8dUYEE^bS6h_?6{)nL#@a z288AryQ|v~Rs~L?j*GwY69Z9TDnu8&@s49nnFf^C8vCS1YWcs^NXb=z_b}@{m^4GZ zT{YR`fyCWl9P0u7Vxz`oGb(2KVf7`0yQq-|Y#QRoJ9?Ld|N9o8&f#%2G-a>wUO$2`^Q_*|F;t238T+fFrnu zL~~FPr^Nd{VyV^zvZ_m$rj^EI4xb7To*I3ih z(bd{BQ}a3%*45Rm>Z@pIm;pM{DHp`x`wsV3R^q2JG`ZPu_Ma0{S+qoRF82X(mO8-n z79cdg^|O(-!oJXIcp%Om2ryo!3wj(fYnMmOYZqI*o7;JREts!Eqj>+E0|=6E0WR2? z&@3~)x!o1tY<0CC(g3Vax1>7j8WwpSxS6!)jXB%9EiqHYWNbCr9`5g`C@C2Zmpc~F z@Vlvjz54!`gl8|O_u99c%arcN08nN0ux#1SH49eO~C9f77o%L zbt%fe)2a6-Dmj1cVQKfM$9jW&BT90yT z4&6jJ8)>~Ohh1mlZb<4*+5=76yZxk>zLk(zt!Vhnr+^X0Zhq?#zHRmbm!IP=6c6TP zy#s%J2hm6IYK#ezP-}S5u^UoiNQ@7BE5nna1xxYUXs#hj-caXfsh*`+H_xr&dQ>r* zYS7BFa2srjd-oA9ws8o^`@X)p)=Zr_K^mJb{jmu&)KVY>-@0Zzknz!Jg#ocDV`z`e z|8UG8v@9$B@+B2g@j0#X5yR7@fR16L5TD~StH+_xBHp+G)PwRLpggG~_i?J7hiv@? zA{e{NROf#xg(1^`Z!%j_Z;scYl&4`qOesr*k)0le`QwT)|XRr zxQk1KlSp|O`8_HAR#lzogPWeP%=cjvjlo1*&ATNv2ifjLrkyv`ato)z!(EoUj}AtU zM3vs_2vbuC$bLTO0o-#kKI?auj98`Ts8vL03JcPHg8lW4J_*dQ=Ud&mx}!M4_@_C- zSjwMz-|H==&~dpUZtwZ|yjWjKq|W>~`P@eDqdnbi3q=QN?h%&{ROeDR|5DG1R$U<@ z+mt!jNE74~v|zd1H5a)Rsq59xubMIu?xk79%s`p5T`_vlnw0R(j*4+7?ScQf)LzEi z%JJuUroE6jt@ZB2S?93$v<~72x~I=wejLoVCf;@$M{7(^Y>=b!&Lz=pg!j>54#Q?2 z;9y*ZizuI`ubbhf1rphj{qkIKAGKfTdYgRv8N`^?Bq;Ht?N4nVe-b@TSl}oI8PqZDomCwh7vDQ;aV^LY<`uf?lUzV=srmw8N z(Tp!%XEfXP{yb*DjUHvW34aVPFbBtmNpw`&rcco;CylUH?X!y0)yc_UIOp98cK1zm znQfj4e}sgzN3t##({$6eU6z$MkfHJQYv15NaW~;Mmz1!ApWtVNO`Ki^OuB_I1GO~3 zUgF>vTNOT|DptyTMiA*q-Ft|$Ng1}Zt=-|{a!w=PP~x){{^L5)x1C=Y#@?IFdnypA zh>8YWdK3<}y$`cgrYjJESfKrRf(cjT7Sd$m=5@wA>zdt)>HFz+cbIV;+C?$LS=^pW z3QJ0}okirV<~`6Yar=%EA}V~!7z+ayEgmqV!=zo#kz{7HN)S1lkiE(kab5^^4}wRw z$tcy7&uy|kZEqIyK@PSe7g1^L9mk9eqRdSJCpa`W)TkKhR)J{qTZvHvjm=G}y#dwws*RzL4*P|vf~E$<%}Wa$^4To#a^mGH@)3fg=c*dv{Y$~wt?s2~#D$;A zE+_JkW<7Ne+KBBLw63968U=tyuj~dvxnOi@Uu3Yr?yajI5{XhdR^GDUhjCg*%CgDJ z_lVSZJC1ztYl_ry`{uORH-_jdGLMvwVtvok{Sh2(I05&c2qj_RV}-Si0YSE-^6aJS zEs@EaA3H+!*7qlG3fhv??;-{F`jJu#T@y(G4)y$3+*EFci$3V{$V-}eBg4DI;Y^4h zdqXDnVyp(g8mB}%^)gv!axr3IwVsHZpDOvRAHIlkm%&W}I!n3Y&p=Sk_Sla+?7o2d z)Q~;02#nuwSFz0v{%-HFXB6d~g%#LYu~#(37cXly+|N|W z*3zsjiy!XT*MW!r{B|?p`hKMfS(*`u$v*|y;r_+j$9s{Gb3QMb%fF6zthea(#kbC# z%2*{Xx6uUh{FxiR2b;OxODc>mYxKqSd-QiigK7BdGDEAg;>mGH4hQx7g2wncq(MUA=<%H#)H8lrn&G>)2e4?B#QHUtF|ybsGDS((oUE?7JGTVs#0 z=mV^UYX#3$2Lj8Dst6tDNtMcFT|VSic*=Vq4R;K}MOSE#e)(&=PKDP3?^zC^9d}@v z`&P^RL+0Ftz*BDZf zhZMt&tE=A)Uf=ryP*eHr@M8Lh*KC9h1d(TGJ=#2_HgxE}s>F@WEPEWZ3o?48zaMP{ zQ=v;MF>z5+BH-ok?SY{ZN*{VSF7sxtg2WOC_>23DaDnL8Lld}`7KPms=I8QKi+KLj)MY@& zeaUE-&hyvmSgm0JLJ5y(quDlxTsw`8NM**l1fS-Ed$O6^0lZ0m_+CQSMDpH!;lyIR zg4yuxzX*=4~Kf?yEW6Wa+GwLMij-vS+k5_TOH(J6o1vD8&)eLoi_o;evQPFa?lS zcOy{;jzXxh+{ne=jXg5*Z+=bQfsfh%Be2DTbi#=T(YN^+J{Q44c1KPP)d_!b8A?DF zs{H@PN9A%6)P2WnFwiN2BX5#3b9X5qWQBo+M*zqfR_Ze+Xcz%VoxDCgGl1S}Fns{Y zp3dDpL9nfg2sa!pVZkb=2>=<1oXe9^FR8#CWj^6{=ZG>__%$`4E#<0 zq-$fv^t7^76!@@gYPy*$G%A?GGLM;Sh{C$Ubq+RZq>8)~G+Nz}C2YRN0N%}eYjBC@m8{U4c^ zI@@o%PRx_3lzRjaj-Yc}Da__6B@d9(F{77ekD zSyl)47ut@*&4=GwaJI}?@*Q)PH-E@ zc9#-`nur!U13Qc)wOG2ML+=&a?-eJ8{;O;k#)5ruI+CK~-AfeoOP3PDl%H$bz7`+c zariP-#uh2QW1t#NxI%D7)AHP`$pSfxP2;Yo3rd~;l@`d<*JgC=Q&5$Y;aUvleYQ01 z`6!v!2xM@~A!c2bX^vDb2}R4cJ`3+OHV(t|XqiZhOjkP6*t=MKpAkVOt^;*{gXQ#9 zRQEHGsMX|q;BN$Fs+K?C*`hXW>g)YDDz2zk(+Jhx$n6fo%{-@QuA()sT zHiW^GhU&1d%svlW+{;d9vq*Bi|9pYLO|>IHly+EUOR7tWyQHhtGQ~*!P($>3dS9Wj zEK7K9J@w@oK_Ex4UH)c;YG3~?z;^~YKjT7~PO|bR*~*HagP+82_Fh(8@^VRMO8EK4 z^y(=8f<}0YV9`ywasl(wgy*Noi{@8~8~2dX+{1>wTJ}Lj;M~>7gFYe@e@OuGm#e95 z*HM# z38$wQZ63eU?ytPNGaX}He0Z=o2*kNmA4h>!*8t*V1Uz^9pfj1OxjK+I0-P0l_}L5W z<3Yo}o&)xjKOxX|yzHFC{Urqdc7rE^aFk$MDuWx2spJwDogU!MxnVs#ej#!=cgA)C zF>FWoMh%>Ev>Yg;zv~3x9>ngTCuV%dxWe;*NY;}O4Y;+%uu!DXzAi7oRPL5cDz1F#VbN#b1v~2Dwwm@2VvRguP?Qb>>k=v8_hABQP!xbz zyynw?CDN^GXE$j*JI@DHx_(AAHUp+Isy{nikZJ-bV$>N zg)%P=TqSbxxsCsZXcvGBY{h!odj8ZvP+!1T$g8f$wzevA)_65@u};lW3q~1|cGtIP&x1fq<^yHk)g7XWKS) zFF4o09HJpGlPh5%+^q~rcl%8LgLD`661dI$S>ra8NWfhNq^g+K8h>=qlew*}Fbqxz5B~Z&A${q9ln+RF zkMbcti;~;Q0vxn+SWs33jB>N;S4z~_zQQG4WM;f{kNxKFu`Px;KVS`+f)xKp|mKxzz~`cx?UZN;B`mu7WprJycfhXF?c&f0F@ z_O|wErD!%0{T^Q{|FsmI+hlaviO0cQ;#{C8{R#g_dz=^VRw9DojR^3SMtf)N`u8X~ zM&7=Q`x8YLexgcP{f7Gp_Ko+<`=o}8=5$*6z-_byIJ?1{ctcl#q<=tZDMP(OZX zLP7yj^dt-*6acbYp$U{g;IAhk9m8M$LFN5Fr(^hUHqXZm?{fX`3ptXdz$P|7tXsMP z)CFpqvDh?!ua}hJ7l1dEue?|E7 z69M4e?{PWKD;>Sr{+IEz(-!gx!(#A7d|tU;t2FL(87-NnaSf;`cMkfM3p{cI=YJR_-ok!B*`4^houO zp#h6Adn*LQbkf4zAn5WKicdtQCeDW`%6tS1*x>Hwf_dlEjROsQr^xW+Uifc8gOVbO zdl0crEAHlF+o^y~sQ{s;35c-ufb&)h9Vsu}We?w_L+A5=V9RKGYS3%P^f zz_U^gK8`Xt`X39^d=WJ3CbIR{C21Ay9R#xaE1QDCcOg)GFl+(GQ?H zQ0|(K%i)a92NEUQJgoGe6O9#B;K~GRULelPQcRP{@i%Kv65Rsw(mwDK+N_Qs@TLj0 ztC{I@0W7@xmy&_x1k@`*p5A&i^wz_ymt^Wx1~}}~4+#84nfI`H zplxNXvhL4s$j($udtC%rIOftHSkDjX_8!UspOGvId$Xd+FP0LI05>I+bJ5s+y^@m# zU+LEZE^TMIjSCtZpJcr2Yl7Q`ar>`%m-7Gt6N8lj+5jK=N=z`US6sr0KEBBo?jJKR<>7y z&d@NIOFhy$=iAP1P3)nIZ{lm6%8Qk7Z$U20k{q9pJi%srgPDL0yhmYjEL3 zs-*@Eh;R~=7gm8Nd(Ih2`tHrZl62;SwT~VU zPb$s*Gxs?rgL(tFBx35uVHE%aLxm(`28CJi2fBxAk?ooG^X%XfoMC3lW2VLevxRo5 zU^Bx6-XM4_Wz5S>0A7#by+1)aSiQHrfJM*$Wa%XqLpX>p{!&)1J_6xNJM}yPa4iGF zc1r{`bU{b&G@#TuG^0g$K|`9cJ>*hmWWuMAXO!{t85r=>?Oea=Rs}M@+C*6{?xf$7 z{GS=l;J}P1MM8@#Qk=dyLC1TiS$BwCnQ}Spy^mLQ#{7nqXo5c^2nXj*J?x%K#qxT{ z5biX0-$yFtjNBUt`ooxs0Ho@$d26_gWo#y?PzEV)8t0Z#nBa!+Af5^iI9%*qM{zA7 z>Zo&!S(qDADX$^|QSdrnU8uGl+$OTar<$xMxEWMoI za-UIPZ;`#E8XPsN-I`~%7%2P}XAcC9cL?9MfBr-k9>?ok&Xo&J^##6snVp`Vo}}HJ z-OH{)#buEPBJ^qy3bG{1l{<)*lq8A*VnyhF7KX1;JdOx=WBAHXQo5h_7W3-I`#Bf3 zu0%DJYu@)m%@zeF0RDG=eBBn$)+%Fr>^m(Z9}Il+rZ_mKDCm+3ukGEK>&#X!FlNrf zI~Qm#yk0O{y9f~2y}L<3>h83VT{1A2mOuk;N{Q~B0`dZG9G>cp#s1E7g8Me0ZBa35 zeYrKurD(jcd!jCV`8 zWw!l8i=Zk(aO*7aq8nL8qDX1e_-ZY}EPsKXRpC=|JcBNW z2TSZQGFO%Vkm5P;Ry*C@ZZ$x8K75Fu$m6_h0T+MIQH%dHY=1nznVDeT9YhcoiwCXj z6S@nMX$mf}8~1@z1H%hJ+#`7sZ*So6%M6xSJMWD^hZ5)Ysb@%x_<#z&Cs~4I-BJ$b z2(E4s!*&82Ia_|~YY$)H4%9+-_7F44$rGmt!K(&$brgF9=wfgwa1_RLj#i7c)Ne#J zh4YH?qU^%#HnA+s5#aID{Colxqac<-Z3}|U7RvZ8yIV^OI_G(nChDQX0MK@sjGVmu zii@h2?X#Fpxao8MvzHzLReJt#h4UcTQBRsx0R{c=A=sxcuY&-IB@`DX2n8|;6R-<- zaks)Npq=<(cP|k7gM-2a3#J{bF^x3tx6>DZRNk3boIbt7m4j5SHrlrw+|DA_)eN0c z7V6(U-0iK2c55lhN?Kt$zYiDBO^6^9k8_=Vry-TT0Zu}A0Zbo5 z^r}md!){WXckC&Nmgn1>vRp+HNG1Aog|gK;0>6Vz^_tRKQ3~y=pJ(-&jxJ0jc!I+O zRYQ4HIEwg%8y&77up*%9@7B+;(MCad4Y-t}#~?0;JYWDm5{ASXCrnCu#}UgC>AxYW z8!SK~)u5n%Or>RCz&)dY58t>P2Iun$i}zQ!A80f0Kp4q8{^FXHX}7t;;f(j;&8w^m z+F*XM;|Sr~d02-CnAhp)hRMr)yB^k|{a}xQ5&ydfY-*bz?Lm)g*QqHI)6a1EER%eV z!pGfP~+NJ4o--wP8*QD`C$&ppEnSg=S^qq9j1M z%bS1ary$oU0~g5r_~5qmh|K8R*_(MUiip6`uf#2UVv&aG5^Tkf+G;3wEJuKipc0?E zL@!u~JfrX^U1DIp>7wTo@~Yq!35DRWz|aR!EDtYK?C}9OF<>~a|3|D}9{8i+f|-(X z81pa%j^#(V(cFN+5)$lC(y6i9O+p9BFM~Yv{+ph*ngBQ?^JzodXgfde_YcB7swTscJ=jv4L zH~ttdp;v>ZSC<*4Z~8AQ>xKfXFV1}KC=v=55{~7)H#j=9%Ey0g%73eBvqw{Sciko* z4@W0jj3M^)@U~pa+V}{NH`sN9;&`HhMZeK1#k()x+s@&rYrJePMJ#SQ1s!H7j^6+9 z{z)~(A|QbnRl|V{f?WfwdqC>yMm`VlSqT8Pjihp>Ikz^w%pY@9$xp2a)+)J@^q@9P zlci7MZ53o+D@5-J!7*zAR9SM1uk#6GR$gcfqB4O5Lm?T~;GPr}1%*%rJn2Ung>}`? zE>k)=6A3rS`6L0gi*)v>{}{dSvyP~}eDhxB8ntX^{af~Z1nm|KiT zv_0Y2%X7&NgaS*Y5%ubO9NVE6Sd#PifuT8@e&Z~Zsge@-=BJR%*c=~tLE%=P;&c^s z)BrlHop<0G36ez@dJmQ zII!h?!k%MR1HZv#)^*D{*7u>{72`_0YaX9c%Tkphr>hm@-NULRXFB;5y}8MD+GQBH4PZ__2@gjeU!~9rXbr3`=%hT5HEA#F zzLxq!nqObwQ-GDxXAol+D(pRe7mIXZL zx3;=Kk^5ep8f7<}DP$TGj4_5PA6*#ldz15NvncWPlX(?-I&x zgT!X%;NlZieF3xw+e2gk3L5+_0Tv1PAG@99b|F|A-3sQ7ejc@)QBHYQWPpf19qrLBnifZe+l)7+5P(dXqNKh|G1_eZk zrHCLIs>o4-1SKd*i57eYPz04ESwcaPgMfg5h=71(kPIq0=bU;SyjSb?-#z-}AEW!B zebTb(?6cF_Yt1#++!f$ve_mGRj4gsHGn<;1Y?s_5UR&NPpUHXYd(1)k=cs)|u~CK= z27r#}Bdy%Mc@8wg&8kP%Ifu`AkZHNUmOFp4@R8{GT;5N|Gj8&+8-U)mnH zDVK{*+bv8gJWxIkdXl#g79k>**#ExJX}qPfWWQJzvJ8f@5L^NYH?L>5i#kk`i1 z9Kfzft?-KFefF;X| z@^v48P%1Yg$$GpcF~k4y$-7S`2`z4t^xg#b(oKDYZxtwm^4$VcSziVg?KY&}@aPnm z*?vCz7XHH&aai3vYjdarbDqpomcT-}Zb5E?XdRHv%gls*T#@m6OWroL84}C2KzBq@ z#&vG5Ac0jHCA=QvrW8KqmWar?$THtgpbUfh!|Qt4CJ!m&{-k2Q4Xo^E7azK&0-Zy+ z0HVmBiJrSMvG{?41kCP?a>urf&ic$5g`aED?3(ek2xG5rgb26%tv?W*pAK#(NR#M* zRLoS&?pD&|ixxr59@nZ-MJCjFx*Nf$5g|5E2-5EmOZ4UEM3xmF0&L_%Jmd@&uHoz( z?K!2CQtLy8`e)XPkWB&FWh{{>`WP=c8#SC@DC3ExOkaM%=i zCqUw^l}v*oE=Y452uF7zsiI{tjE{Pf-;AXx+ZyJN*BWiJ03q8V10g!ZShIGa!li00 zCNYgceqPt)8d9a`8_CF8AnP-~*v0cvvM9X`S!V!$+G|ma#E*e$fO=2}vNa z=TkaPcMEfeC!g{n@GOots^}~e!v_Xvk{8U8nrcN3MjsnxtfF2jTK?~_2kf8zAd|Mz ze|PV|z#TlmanTiOxD7IcqTSSIUp-ofq+|(p;$e%uRLR>JU}aJ0P8?IQtb~H^k*@`y zLfL5;^$>uD?%H5(1BMkS60LF?d(I@*n|0-oS{v!d`4Gq>^+4f38dO|rwigCjDM-@? z*Q?fC_-GCbFW;=@D3XwMnrP#E?+YB)&!=$M>5CR#Mp)8 zk%y@82^t9_r;YKXo*$J@&m49G4FZEI?ieoZ63tTgt@&!i=#g6e$8BY5{JGX_Gw|qd zBbvA+kUwIIi)N!GlPZ;5mRrAi0wG%qTeW=3C|S`hh@q+iGX9M_UW%75<9c?MJB)gO zsDYZfi58 z@VfoGvC!j8@oy``SurF{NcqJ^>bz-y#7zVpS$}Z7n)9N%8nrX5_MX+UU60jugqw=P zLYYbLoZPZ9fn#5A7hpRE#kOVJ&IQq4NK8!0p%+tH0cbDd7tX|WywGaU`OWnfsQKvd z$w>2v@!TNjrOyWH&%9XZxF#^n_LD@C+zx4YdFhp{rc(O=?8U&ATvdh*JOCV$A9UZ@ zzj1pcKlwMl@vJyaub`pLC;o1aO@YjktCifF=nmJR7qX`(FpM4nfm8`^eY{zj1Z}#7 z++d>nxJAPj%07){R_qB7Y~ITD<|NZ~B|PTRw|zf{<~y{v*TB`u;X~oINYB4Y=taj9nQ# zkgy~kgo;G4jytV_GV-a0eLo&|jbD^-pfM2I;)fUUs$sflAM5t#sSTYJpTnY`$7iNLotsJ}H4%!aoGv5sf#^#on zCG3!3-2+9cmOaZ^j=fF)Q|_@1gEK%}VS z`eX%(yt&tQx=~r_I4xf=nezATt-J5E>M}78Mik6PI}nc|U?3Q(<&()!6Z4t?;bIEiQ#fnLfjbrk>D{!&mDbve`bRqXM;xj>9XA& zzImCY6|C1+!JDNV_pdo#Ib7zCJX!+Ye7>(vbbh+Z@TUtf6!KwUXGKIrTt#{umRv!< zE4Q2%v+{*E9~6!)K5C8KWKJ+#XF-(2C}Ts*$Rn>;RO)d%K&&(8$*;}E4pE1j ztwBQ0w?8PT5sdrc38)!z@is4uMXQW<)LEdHdSGSadXFXp?dFg_UL<9!L=-&1eIMI>m&Tgcw?5q-u$eJ4XF=QXI#dt5E5!-%imB z{AB9Lo68gJ<@%aZYvoj9c@$>aXx0K!cj`n=Iw7N=7<;WoYp(LEj;AAfi;CLvLT-2x zaufHIdEls9Gdltrfp)J{95PP1H{*iBk1LZNY3N=>)B)WyRde`c7O!&#uW=?t{y0Ct zxvynTf=q^%w!R~7JqaWq4u@(Qlt{}zdxQR>;!QA?U6f6X8@rT(lovUF)v@RT%JR1y zYLIMb#{1%&Jal?iOUBUhxF@a&LZ4I?<*bUIUqVtf?*^YcTt6F^+g<4}b8kgWe!1q=+2ZTYd1WZh z`g%u@@_k1$T6#FII#hwbq-_~Jp3DBZJ)vAOP;@-um40T(NZ^9ZWj*oAvyo4N51w&< z-;z0**)AWM4O$HaJbGm64E?L1fTw+N*mX@CDdrF(yB5GaUHUl4_{!yA_Fj+0dXYZ& zyWg+7(Fnmj}DH#XZg0Cafw-pP;?H zJk=#g==w9hk$zieI!el`l<_d(`y+YfHy6Ngu(}{%#5dFZ@2+)>6vl0_++QjQ_WLMM zhD}!gdA?MU35n09q-NYLQFq;`IeSf&HPMiXOdYUp0rN(rP)*-B?%L)lM?%p)3C6^$ zM;K1KR2&j{=gX&8V9OM#ETPG@QAbeiwC1W&)h-(biQ&T0xbpa7ZMB7rf zjesDi%OhhhgOZeM)eE@C?;dO@c1lWJy{a!SLt1%-OyR|<`;XjcO20%6lm-vq_lia) z`Ky7v+S+$aF_Ls_Xu6yiq~xURJ!ab@UKKV)X9ZtJzn+PnS(<%6vDDX4IJz~1UEGNt z?zflDHl9 zb|_MEtPJbUV>W^v>e2dgy^1Y-LY*9lmYhmZuh{ZSr7c(EnPlYNf@(!k#)XbM97aoq zmBRDhz2Z$=eDN=PcGs$sT>Iz#E z&(Nz6_Tj7#v{lU76=tohu>eLPWDGia~o!^{QMd6 z`wzD4tYO{t%*i9ao<qqT?SW3G-(trAB+anI+@G zAtND5`(&Vdtv+~zFV-Su_K`d?{5XmOQ-ZG8hw|zNk)J!VeJjZ4ejUxyA=vI9cnKMl zSU(hIl_lnDMo(b0QEPbq?d%*QS^}-o1N5;2pPEX?UL2LvSb0N<=b9u4A;Xyb;KS!5 z72VZ~RP`gukmBaFx!vB#Z$f_1 zZs+wO%vqD52!L}CGK?ww{x^fu-#~3MuBKpq$$bq3>)(V zaiWK8v$2Lv@ZVo$`@OMZ-;j}F4sEYtqlq@h&QWT`<1P^@S33 z?GfE^{qgRywtQZLoexDznWGo9ipE|p+*Im16Xi5E_tMn)_wQkWaO3>;V_f*G6>?d0 zwU##qa@{fE0@lW&r4q?6|MkUN$QJ`jde1Bm4z9|{`ZHoR-IPo3cyzz#O}sSAo0&9@ z)6i?}a5}gWO zPxK)vgKX>%2mAuQT0D7FBt(SDx~DEQ*v%t|_g@c{B1@rD+#Y+OH2%h-CvBd*41%xBJN3X61v@+aHnEF-lsa~asKph1_Ad+UBpFN^Q5+1box8?hzmCI zDw6AFu8?eLf!MwkOhV-(>l9tC(#L)~+JJ4x~N z%iv;@N|gs-^q0m<-*klxP2L<1y2m#izW zyz|HY(Cuh~>gM zD1&NjHRa<>;A=YqZb6f=K5gd}u>*IZ6x&Nmoiq9LLGW zRdoa&d6J#Sq#oP-GXtO!@Unk7!-aMM0F%_L?FyD10q#}g+85_3e1vVSHz7Yi-w{Hm zB)FKO$lj*(oN3iF9u?ACM~S-7PHUMme-|NVh^`O4a-m4@Uh`quGQOUbEAjP~*CiR_ z#fSXvZ-(4G`MfhB=-wU&M5&f4*qvYZEeDe55gYb*r`NZ?>?i6JBDq@Fm?$2|C5aGw zidxZJ8NPFuNRQ+z5SQ_5}4aFVod%eyEx6c&&yG3^9 z9!n)a;bpPCCAg>K05!TJ#o(0Han;j60^y$OA>IU{C=JFDt*D8W(l*FYB>?@fR25CC zdhV^==fG1s)KEuFpLo2s<6NJ8#UZxGwjkmCc6VpfqPl+}q#ywV6WJG5)~^%DdeE}j{#Sn=~I0%{k3pX+4$!*$&O zkS@qQ<*D|Sx1jfB=}KN^;&X?bqN0GoUz-tJD*agN^0o)OHskA|h9|fRQd2733CA^7 zw-m%ixThz&FNzo|G}7y6L(%y4?3j(Vq#QKY;VRTo2>J|@%7Gi0J6pPm-_LxHb@OVe z_UE{Zz%wgoHJ$(_Hkxu%s#!xJW2%6I_l zgP{qfJEFiz^mC!?p`_#CxGg)2Su%DC26($tSq1JWDjiw_I7@y-tjD)~-4Ancvrdcr z*&t}a*sXAu;_r!R(ggiunqXLxqY@u(R_Ua;%ee_xE@$JErsMU6#Ga8;h4j7r4);bu7CdrU#=0*@i}nVBPtd2o46 zK!fe*@n1`l1SL$(;V-~-uVR{ay1Xr{WtHDaV=MR8S3JpdbKBav#94!hiH+fDg?O-2 zhMB+|3Z_|YlBP=3mKRq+^b<;T7MPxji*Lo!x~eO~8oH|N zG5^#^k!Ag}Y>JcHF!9;xk8UhhEaJ>ERH=Sy^Ay9=Nj)3sN5lPXyK#3*Wq$nQ3h3WZ z_J9PqNSSmQ zpilE_zS_PbsQ@r;=9Z^11z-{g-H~#q=JDm-RsK{IYbjZ+4`3 z{128jw4W(t`Q_$M_q|Wz;CqtTnEF~0;^icSdeg=GBT79f+SU5&Bl1Xquc|(#$M5IX z+N5sWE5}KfmxZa=~7I;@9}v z@7svjZmCvyD>JsFEu_*aIL%J;Yqa$K1fA*C@$0N0=ChGj&Cc=_1X`*SK`Z1uCEn!q z(k!Tbuq{u$+c1iJ7RzZE!WgrooPE$O?V9%c(>J_2YPNrST|%$Wl|F&v$6YJw>%E%%nwRAS1jiPS%dcB<1~s=?6&QEQ{?8dU)-F}B6i`Hq{@1xfe#TY(rC z1sM4&!^Q9pv&9$_lE+~pv~|%6M7EW>GZ>8_!;qZ2-I;a!{`?QNDi^Y$lSGdv6+LE| za>Fx<*I&lA)(=@}fA)9cDv&>-#L3%< zMan(a<**JWnQkHDMZJjP)@Xpc^O3_yk1LwGy_qIYJ>e@ z_>&G+45B7iERvWY>i;!b=z7u#_Apo5?{+P5ZaZggtvxws+B$X})5`2N9W|ZlW~*Us zP{>1>vva>^w*i$>a<(Dm#7oA1x$^O715%8_&W``viTTNv*i*b#EzQ;VBc@fsZGzq@ z9!d3bjbGgUlGt23W3|(}*z(L}M@U8`7%-h*!-LzQ9$O{jFp)E2mSdU=ahai!qm6$S zbw@7Ted$iuo3YC7qVdEZt@%WA$Eqrb^Q6LuNZq~vwUazy;fBYZSf|RIGmw*w^78nVUySlbz9l-;kPG0JCI!mKpXkOEW}10o-@4V5IOJYZFD%cU zuGsYRN#8zv>3q6Q5l3Bw(IAl*N`lIx8S989DBTJ*w_%dFFaM_gRtV1*6ydvDqg-o1 z@p*r-xi4uJ)-@gulpUX{lDFA!)+I4cXniyx-agRyErI10K{0+@=}e(T4qip_#}69r z6(z}qA2hi3k>yGCw29E9Qx&pi5%ZEMcUvLE2$=yo{1V)o84_&1xiUQtl@W4?!gqJp z4QZYG`b6W2b|8nx2OqQRd4Fl(wo$X4Tt1WZzB4k7m{O+HZ6`&E`!8leoGYpUcrZ&5 z16z30b5S!;vB8E2H;sdmw;X?oS@u)q7bg+Siy4OZ_O(}d4ICmgW5$e!-L#KLJfR6N+(xw5J< zt$kEt#q^!g!Tl$NSH`B?3wPatv9AnVhtXCT7d>0L4x&Zb^Z^APWU}t`rf`^_i;T>~ zMR&WUA%YH7dlkKr9B+jz-rWOs+%jtGX`CBe>TC*OYmPUO)JEK>o! zKD#P0uaJT_Q?uGvO(0of1(~;p;)u4^2Ix|U0na9g30OMSTsWviRhoZ6bPljKy++sj z5PqO19u$1x8Ncsmm_#0Oy7CH5MYmh}>B_BScb%oN7k=oJv#D$54;|+lsXXU=uCO|R zN?O8eL5rJO{h&oRE*P_}NMCr8=y;CpBC}>tbUK(|^28{50d3tB_+t9;p9x=AXm@_R zO47c4dPHv7(b6coEO!>sGOLf45EEMl&nu5wt`Y~rbDWGDF5twugtQYM9pU=1H{sd@ z99H3iY)UHHy6Q_z0o9-wM`hi;K_<)#-D}E<(z1%lYw^HQ-_hV$TxwO}zdg!`_?$-- zhZ#zVj0gu^@^yi{RP@B_`vGRsn{l#SR0(n3CHK9scPeVT42vACoOO7DCEeHNqH>li9ws)ltfVQW;RB$2os8zWc*N@CDf+g*9@N^dEJ&rNr*~=c$RDYTb5S&$=~4S6 z`ODUkpbF6%J_PLzamnoSMm%1I;&1sgKXjgWY!*?+QW7VS#xJEcJqw`%t?fK)cDnpU z|9hiBM7w(*u7MP=6B%mS67?uR_8IfG%9*%VE7zM};cw#FYO8luES%E1vw>q3tGWWU zM5>()B9O^??%s;Pz`y2Qo;2mC;xt#FUvc&MQLtf(+1>-k9Bk#&xn0t_3sNM)YU!gm z2`jGnZ99gci`RGBT6NdD& z{(kfe*Sd=c!tFUOTq9Qz1fZ*AFtG%53f(tAkSDV!@IUtZ{z7#q5-%**C{u=+fA#H!8nSni~u$LIC&CYRlFN|#1jrkK>- zR3h86pVRiRT(p;U*iYqYexw2@9zsZT&Z)Wd^xHU)wp5Tj2*Wv(t3_~gTJE444tP)+ zY0MO-e;GY=>^#|@JohHXr<^l+w9F z;7C6^6UNxiELZ84uoCU)o?m42C79vdO?}O_aC4VL@{cL=>iq`od4q)^At5v+*#Bw6 zN1Ba2uJoV7C@a*l6pH6bIvBmNy%o}MeekAxT9%y; zau9HxiIq(`|6KlOek1At*1ZAy<$kzlZ!jUTB$Nw9&^9)K{YY>4Soi;A=;d?=@nDBvC0dbT5w!F27y5b@OKG>LI3V({p~;yfIotDCq zkk`%8A+MFFPt?ijXeA%Qf z8-?i4-Hyu#X8BWXVu}W}I2XwAwLk6>W81( z($qn|-;Nz*01uJEG0kgZ@Je`mWIZ6H@2$8>*9m>12n4)k@;0HcZkK=~1SgE1sEPh+ z>Uz43(fX_}eY6trmVR5z2k4Rm@KpxE+gDvD5=UEw&pPi$6Us96!+%I%{>wMDAH?-X zKiME&b<`*&7Dc4wupVlnffei&O2yN_ub0mQK#(*cFV88o!&$u0tmfs?M@oK2@kn5Y z1VTMqD3mT2v=%RcLo|Z;6yl}3>4qgm_Vuz-8_i3>neJ?dPHHol0^pxF4uk@(LT9*!aJ=5m(?JxVc;JE! zQkV~|KAD9w^8^G8vVs!ou63`?0xt!ltsnHQz)WhPFP(cR1UGVCDFu1t^9By}!d5_j z<~#Y+AJ~#fP;dTjsQa@QP{0$AjJXx~$!r}x|4tomA648gd(1ME@3hz`L-L*Hl^^@bcPoJ%P&fYI+np1(5yhIEd=v~WljqnQLUH2~p*4m5NX0DpwJZr^=lp~K=1HsO z$UoERtC9db?4n*uYYUGkr z`1fHSs6`lCSb_koXSi_ds$>8+9pU0op!7mTlxKC*2ar&3ac#IvWR6nKGzFDowQv~O zA7KSFi_r@o*j-cNWI0^3p)Pn135i_3sCNdu@i9oetNBeY_omV*H_4b(`&dg;$g^-t z663gev}2*h)*hRi!9@z^5;DqKSs^ZeWt)TXZYWjKrBWeeyhti6ceoxXs^c{Lt^PC) z$){tHI?)&t+ENez&ud)$LK$}Pj!Ds2+*9@U7?5Xma9a;-@!D!}=6 zj;$~o5rR^5#4@_lX|en9wW?l%cp#EE#2j4iX}?0>7d%VMCn%vm zp7o88R3N)T*bJEU;4~lEGAh^|So)BXIKCsuKXsvcK#NZaIZelt@4RZ6RoRj&=+Ij9 z4$Gw}S3H;}2F2_xV97A=40seuL8?H=b8GFb*5+KLb%J*EHdu|%{dRd!1->Y&O z9h*%`$7lFL;~iaDfj?XPPbVh!9NC6>m{GhMC>Yr$k+CjZ_(bUxpOg}A1wxx$>QmV# zi;2%&5Aq<9ne>lNHhpc|Iiu^s@wzL?bA7b3#BOLQW!qbeiG-TW`bX>AoBSCZ#Uuaa zID~_9LLk!N7N&JhfqL?0ie@C!Y-Ydqj3e?JBrI(kBAP)5uj)YcPxI@CEBrE zf5ELM)Cng4ZI2VM|NQN;j+$P+ZMy1!E!KMylr5K48@ z>lvv^EnxscN9<+Ng@Mj3k%G%9>ES=O?*~5knvGSBa^^r&{?och#D?NZ$%o3a*X?YL zYG3}%(MYDktwa+-gzUe4Xe)B|&@J#Z^=OmlMTU#IHu;LE7tzEwRYtSOdlDQnvpPfkO1e*l@FLr zqc`vF9f&wry3e30NReSsa7u>1J%m?trB2Bm=?Fgx(#DNy6`-X+re?S|)~ZP;wsLIS zoymNgu3iGHx literal 0 HcmV?d00001 diff --git a/docs/src/piccolo/schema/m2m.rst b/docs/src/piccolo/schema/m2m.rst index 15ee10639..4e1950604 100644 --- a/docs/src/piccolo/schema/m2m.rst +++ b/docs/src/piccolo/schema/m2m.rst @@ -16,7 +16,11 @@ Postgres and SQLite don't natively support ``M2M`` relationships - we create them using a joining table which has foreign keys to each of the related tables (in our example, ``Genre`` and ``Band``). -Take this schema as an example: +.. image:: ./images/m2m.png + :width: 500 + :align: center + +We create it in Piccolo like this: .. code-block:: python From fa33511f9c00717bf6d23c8346d5a0c1ba0c9215 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 17 Dec 2021 17:30:27 +0000 Subject: [PATCH 184/727] added extra test - M2M in the opposite direction --- piccolo/table.py | 3 ++- tests/columns/test_m2m.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/piccolo/table.py b/piccolo/table.py index 9564f8064..a90744e03 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -444,7 +444,8 @@ def add_m2m( >>> band = await Band.objects().get(name="Pythonistas") >>> await band.add_m2m( - >>> Genre(name="Punk rock"), m2m=Band.genres + >>> Genre(name="Punk rock"), + >>> m2m=Band.genres >>> ) diff --git a/tests/columns/test_m2m.py b/tests/columns/test_m2m.py index 96ed20164..9d93fc49c 100644 --- a/tests/columns/test_m2m.py +++ b/tests/columns/test_m2m.py @@ -17,6 +17,7 @@ class Band(Table): class Genre(Table): name = Varchar() + bands = M2M(LazyTableReference("GenreToBand", module_path=__name__)) class GenreToBand(Table): @@ -66,6 +67,17 @@ def test_select(self): ], ) + # Now try it in reverse. + response = Genre.select(Genre.name, Genre.bands(Band.name)).run_sync() + self.assertEqual( + response, + [ + {"name": "Rock", "bands": ["Pythonistas", "C-Sharps"]}, + {"name": "Folk", "bands": ["Pythonistas", "Rustaceans"]}, + {"name": "Classical", "bands": ["C-Sharps"]}, + ], + ) + def test_add_m2m(self): """ Make sure we can add items to the joining table. From 8a73308743be88b56c56627e318e4b8d00d71007 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 17 Dec 2021 17:40:27 +0000 Subject: [PATCH 185/727] improve test coverage for `get_related` --- tests/table/instance/test_get_related.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/table/instance/test_get_related.py b/tests/table/instance/test_get_related.py index e5a03c643..e68fdc57d 100644 --- a/tests/table/instance/test_get_related.py +++ b/tests/table/instance/test_get_related.py @@ -25,5 +25,16 @@ def test_get_related(self): band.save().run_sync() _manager = band.get_related(Band.manager).run_sync() + self.assertTrue(_manager.name == "Guido") + + # Test non-ForeignKey + with self.assertRaises(ValueError): + band.get_related(Band.name) + # Make sure it also works using a string + _manager = band.get_related("manager").run_sync() self.assertTrue(_manager.name == "Guido") + + # Test an invalid string + with self.assertRaises(ValueError): + band.get_related("abc123") From 4d4c150f3326bb689aafb1ec79e511689bb0981a Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 17 Dec 2021 22:30:14 +0000 Subject: [PATCH 186/727] bump requirements and increase code coverage (#376) * bump requirements and increase code coverage * ignore type errors --- .../apps/user/commands/change_permissions.py | 10 ++- piccolo/columns/base.py | 2 +- piccolo/conf/apps.py | 2 +- pyproject.toml | 3 + requirements/dev-requirements.txt | 6 +- requirements/doc-requirements.txt | 2 +- requirements/test-requirements.txt | 6 +- tests/apps/meta/__init__.py | 0 tests/apps/meta/commands/__init__.py | 0 tests/apps/meta/commands/test_version.py | 11 ++++ .../user/commands/test_change_permissions.py | 66 +++++++++++++++++++ 11 files changed, 97 insertions(+), 11 deletions(-) create mode 100644 tests/apps/meta/__init__.py create mode 100644 tests/apps/meta/commands/__init__.py create mode 100644 tests/apps/meta/commands/test_version.py create mode 100644 tests/apps/user/commands/test_change_permissions.py diff --git a/piccolo/apps/user/commands/change_permissions.py b/piccolo/apps/user/commands/change_permissions.py index 3f2672ed9..edf0dc443 100644 --- a/piccolo/apps/user/commands/change_permissions.py +++ b/piccolo/apps/user/commands/change_permissions.py @@ -3,7 +3,7 @@ from piccolo.apps.user.tables import BaseUser from piccolo.utils.warnings import Level, colored_string -if t.TYPE_CHECKING: +if t.TYPE_CHECKING: # pragma: no cover from piccolo.columns import Column @@ -45,6 +45,12 @@ async def change_permissions( if active is not None: params[BaseUser.active] = active - await BaseUser.update(params).where(BaseUser.username == username).run() + if params: + await BaseUser.update(params).where( + BaseUser.username == username + ).run() + else: + print(colored_string("No changes detected", level=Level.medium)) + return print(f"Updated permissions for {username}") diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index b339a782d..8174ab9d3 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -688,7 +688,7 @@ def get_default_value(self) -> t.Any: if default is not ...: default = default.value if isinstance(default, Enum) else default is_callable = hasattr(default, "__call__") - return default() if is_callable else default + return default() if is_callable else default # type: ignore return None def get_select_string(self, engine_type: str, just_alias=False) -> str: diff --git a/piccolo/conf/apps.py b/piccolo/conf/apps.py index 67ab3d76d..2b08b51f5 100644 --- a/piccolo/conf/apps.py +++ b/piccolo/conf/apps.py @@ -76,7 +76,7 @@ def table_finder( and issubclass(_object, Table) and _object is not Table ): - table: Table = _object + table: Table = _object # type: ignore if exclude_tags and set(table._meta.tags).intersection( set(exclude_tags) ): diff --git a/pyproject.toml b/pyproject.toml index f46892289..99116394f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,3 +20,6 @@ ignore_missing_imports = true markers = [ "integration", ] + +[tool.coverage.run] +omit = ["*.jinja", "**/piccolo_migrations/*", "**/piccolo_app.py"] diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt index c6f25bbd5..59adc46a1 100644 --- a/requirements/dev-requirements.txt +++ b/requirements/dev-requirements.txt @@ -1,9 +1,9 @@ black>=21.7b0 ipdb==0.13.9 -ipython==7.29.0 +ipython==7.30.1 flake8==4.0.1 isort==5.10.1 -twine==3.5.0 -mypy==0.910 +twine==3.7.1 +mypy==0.920 pip-upgrader==1.4.15 wheel==0.37.0 diff --git a/requirements/doc-requirements.txt b/requirements/doc-requirements.txt index 8c3a1b192..453dac60c 100644 --- a/requirements/doc-requirements.txt +++ b/requirements/doc-requirements.txt @@ -1,3 +1,3 @@ -Sphinx==4.2.0 +Sphinx==4.3.1 sphinx-rtd-theme==1.0.0 livereload==2.6.3 diff --git a/requirements/test-requirements.txt b/requirements/test-requirements.txt index 346f95ba5..2ab821f3f 100644 --- a/requirements/test-requirements.txt +++ b/requirements/test-requirements.txt @@ -1,4 +1,4 @@ -coveralls==2.2.0 -pytest-cov==2.10.1 +coveralls==3.3.1 +pytest-cov==3.0.0 pytest==6.2.5 -python-dateutil==2.8.1 +python-dateutil==2.8.2 diff --git a/tests/apps/meta/__init__.py b/tests/apps/meta/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/apps/meta/commands/__init__.py b/tests/apps/meta/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/apps/meta/commands/test_version.py b/tests/apps/meta/commands/test_version.py new file mode 100644 index 000000000..723c4ff35 --- /dev/null +++ b/tests/apps/meta/commands/test_version.py @@ -0,0 +1,11 @@ +from unittest import TestCase +from unittest.mock import MagicMock, patch + +from piccolo.apps.meta.commands.version import version + + +class TestVersion(TestCase): + @patch("piccolo.apps.meta.commands.version.print") + def test_version(self, print_: MagicMock): + version() + print_.assert_called_once() diff --git a/tests/apps/user/commands/test_change_permissions.py b/tests/apps/user/commands/test_change_permissions.py new file mode 100644 index 000000000..6d794a852 --- /dev/null +++ b/tests/apps/user/commands/test_change_permissions.py @@ -0,0 +1,66 @@ +from unittest import TestCase +from unittest.mock import MagicMock, patch + +from piccolo.apps.user.commands.change_permissions import ( + Level, + change_permissions, +) +from piccolo.apps.user.tables import BaseUser +from piccolo.utils.sync import run_sync + + +class TestChangePassword(TestCase): + def setUp(self): + BaseUser.create_table(if_not_exists=True).run_sync() + + BaseUser( + username="bob", + password="bob123", + first_name="Bob", + last_name="Jones", + email="bob@gmail.com", + active=False, + admin=False, + superuser=False, + ).save().run_sync() + + def tearDown(self): + BaseUser.alter().drop_table().run_sync() + + @patch("piccolo.apps.user.commands.change_permissions.colored_string") + def test_user_doesnt_exist(self, colored_string: MagicMock): + run_sync(change_permissions(username="sally")) + colored_string.assert_called_once_with( + "User sally doesn't exist!", level=Level.medium + ) + + def test_admin(self): + run_sync(change_permissions(username="bob", admin=True)) + self.assertTrue( + BaseUser.exists() + .where(BaseUser.username == "bob", BaseUser.admin.eq(True)) + .run_sync() + ) + + def test_active(self): + run_sync(change_permissions(username="bob", active=True)) + self.assertTrue( + BaseUser.exists() + .where(BaseUser.username == "bob", BaseUser.active.eq(True)) + .run_sync() + ) + + def test_superuser(self): + run_sync(change_permissions(username="bob", superuser=True)) + self.assertTrue( + BaseUser.exists() + .where(BaseUser.username == "bob", BaseUser.superuser.eq(True)) + .run_sync() + ) + + @patch("piccolo.apps.user.commands.change_permissions.colored_string") + def test_no_params(self, colored_string): + run_sync(change_permissions(username="bob")) + colored_string.assert_called_once_with( + "No changes detected", level=Level.medium + ) From dfc14536f2bf50284f6cde4d4620455edbc90531 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 18 Dec 2021 06:57:08 +0000 Subject: [PATCH 187/727] add docs showing how M2M is bidirectional --- docs/src/piccolo/schema/m2m.rst | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/src/piccolo/schema/m2m.rst b/docs/src/piccolo/schema/m2m.rst index 4e1950604..341ca3d73 100644 --- a/docs/src/piccolo/schema/m2m.rst +++ b/docs/src/piccolo/schema/m2m.rst @@ -39,6 +39,7 @@ We create it in Piccolo like this: class Genre(Table): name = Varchar() + bands = M2M(LazyTableReference("GenreToBand", module_path=__name__)) # This is our joining table: @@ -48,8 +49,8 @@ We create it in Piccolo like this: .. note:: - We use ``LazyTableReference`` because when Python evaluates ``Band``, - the ``GenreToBand`` class doesn't exist yet. + We use ``LazyTableReference`` because when Python evaluates ``Band`` and + ``Genre``, the ``GenreToBand`` class doesn't exist yet. By using ``M2M`` it unlocks some powerful and convenient features. @@ -80,6 +81,18 @@ You can request whichever column you like from the related table: {"name": "C-Sharps", "genres": [1, 3]}, ] +As we defined ``M2M`` on the ``Genre`` table too, we can get each band in a +given genre: + +.. code-block:: python + + >>> await Genre.select(Genre.name, Genre.bands(Band.name)) + [ + {"name": "Rock", "bands": ["Pythonistas", "C-Sharps"]}, + {"name": "Folk", "bands": ["Pythonistas", "Rustaceans"]}, + {"name": "Classical", "bands": ["C-Sharps"]}, + ] + ------------------------------------------------------------------------------- Objects queries From f177dbae4025a566b5495415379ea72e225b7f21 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 18 Dec 2021 07:00:29 +0000 Subject: [PATCH 188/727] tweaked wording in docs --- docs/src/piccolo/schema/m2m.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/src/piccolo/schema/m2m.rst b/docs/src/piccolo/schema/m2m.rst index 341ca3d73..98e6d422c 100644 --- a/docs/src/piccolo/schema/m2m.rst +++ b/docs/src/piccolo/schema/m2m.rst @@ -59,7 +59,8 @@ By using ``M2M`` it unlocks some powerful and convenient features. Select queries ============== -If we want to select each band, along with a list of genres, we can do this: +If we want to select each band, along with a list of genres that they belong to, +we can do this: .. code-block:: python From cd20ab31a7354e6e983b77a218aa5d868a2c79d2 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 18 Dec 2021 08:50:07 +0000 Subject: [PATCH 189/727] add tests for M2M with custom primary key columns --- piccolo/query/methods/select.py | 6 +- tests/columns/test_m2m.py | 137 +++++++++++++++++++++++++++++++- 2 files changed, 138 insertions(+), 5 deletions(-) diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 3395d6dce..ea67b2249 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -365,10 +365,14 @@ def _get_joins(self, columns: t.Sequence[Selectable]) -> t.List[str]: key._foreign_key_meta.resolved_references._meta.tablename ) + pk_name = column._meta.call_chain[ + index + ]._foreign_key_meta.resolved_references._meta.primary_key._meta.name # noqa: E501 + _joins.append( f"LEFT JOIN {right_tablename} {table_alias}" " ON " - f"({left_tablename}.{key._meta.name} = {table_alias}.id)" + f"({left_tablename}.{key._meta.name} = {table_alias}.{pk_name})" # noqa: E501 ) joins.extend(_joins) diff --git a/tests/columns/test_m2m.py b/tests/columns/test_m2m.py index 9d93fc49c..291a25114 100644 --- a/tests/columns/test_m2m.py +++ b/tests/columns/test_m2m.py @@ -1,6 +1,7 @@ from unittest import TestCase from piccolo.columns.column_types import ( + UUID, ForeignKey, LazyTableReference, Varchar, @@ -25,13 +26,13 @@ class GenreToBand(Table): genre = ForeignKey(Genre) -TABLES = [Band, Genre, GenreToBand] +TABLES_1 = [Band, Genre, GenreToBand] @postgres_only class TestM2M(TestCase): def setUp(self): - create_tables(*TABLES, if_not_exists=True) + create_tables(*TABLES_1, if_not_exists=True) Band.insert( Band(name="Pythonistas"), @@ -47,14 +48,14 @@ def setUp(self): GenreToBand.insert( GenreToBand(genre=1, band=1), + GenreToBand(genre=1, band=3), GenreToBand(genre=2, band=1), GenreToBand(genre=2, band=2), - GenreToBand(genre=1, band=3), GenreToBand(genre=3, band=3), ).run_sync() def tearDown(self): - drop_tables(*TABLES) + drop_tables(*TABLES_1) def test_select(self): response = Band.select(Band.name, Band.genres(Genre.name)).run_sync() @@ -110,3 +111,131 @@ def test_get_m2m(self): self.assertTrue(all([isinstance(i, Table) for i in genres])) self.assertEqual([i.name for i in genres], ["Rock", "Folk"]) + + +############################################################################### + +# A schema using custom primary keys + + +class Customer(Table): + uuid = UUID(primary_key=True) + name = Varchar() + concerts = M2M( + LazyTableReference("CustomerToConcert", module_path=__name__) + ) + + +class Concert(Table): + uuid = UUID(primary_key=True) + name = Varchar() + customers = M2M( + LazyTableReference("CustomerToConcert", module_path=__name__) + ) + + +class CustomerToConcert(Table): + customer = ForeignKey(Customer) + concert = ForeignKey(Concert) + + +TABLES_2 = [Customer, Concert, CustomerToConcert] + + +@postgres_only +class TestM2MCustomPrimaryKey(TestCase): + """ + Make sure the M2M functionality works correctly when the tables have custom + primary key columns. + """ + + def setUp(self): + create_tables(*TABLES_2, if_not_exists=True) + + bob = Customer.objects().create(name="Bob").run_sync() + sally = Customer.objects().create(name="Sally").run_sync() + fred = Customer.objects().create(name="Fred").run_sync() + + rockfest = Concert.objects().create(name="Rockfest").run_sync() + folkfest = Concert.objects().create(name="Folkfest").run_sync() + classicfest = Concert.objects().create(name="Classicfest").run_sync() + + CustomerToConcert.insert( + CustomerToConcert(customer=bob, concert=rockfest), + CustomerToConcert(customer=bob, concert=classicfest), + CustomerToConcert(customer=sally, concert=rockfest), + CustomerToConcert(customer=sally, concert=folkfest), + CustomerToConcert(customer=fred, concert=classicfest), + ).run_sync() + + def tearDown(self): + drop_tables(*TABLES_2) + + def test_select(self): + response = Customer.select( + Customer.name, Customer.concerts(Concert.name) + ).run_sync() + + self.assertEqual( + response, + [ + {"name": "Bob", "concerts": ["Rockfest", "Classicfest"]}, + {"name": "Sally", "concerts": ["Rockfest", "Folkfest"]}, + {"name": "Fred", "concerts": ["Classicfest"]}, + ], + ) + + # Now try it in reverse. + response = Concert.select( + Concert.name, Concert.customers(Customer.name) + ).run_sync() + + self.assertEqual( + response, + [ + {"name": "Rockfest", "customers": ["Bob", "Sally"]}, + {"name": "Folkfest", "customers": ["Sally"]}, + {"name": "Classicfest", "customers": ["Bob", "Fred"]}, + ], + ) + + def test_add_m2m(self): + """ + Make sure we can add items to the joining table. + """ + customer: Customer = ( + Customer.objects().get(Customer.name == "Bob").run_sync() + ) + customer.add_m2m( + Concert(name="Jazzfest"), m2m=Customer.concerts + ).run_sync() + + self.assertTrue( + Concert.exists().where(Concert.name == "Jazzfest").run_sync() + ) + + self.assertEqual( + CustomerToConcert.count() + .where( + CustomerToConcert.customer.name == "Bob", + CustomerToConcert.concert.name == "Jazzfest", + ) + .run_sync(), + 1, + ) + + def test_get_m2m(self): + """ + Make sure we can get related items via the joining table. + """ + customer: Customer = ( + Customer.objects().get(Customer.name == "Bob").run_sync() + ) + + concerts = customer.get_m2m(Customer.concerts).run_sync() + + self.assertTrue(all([isinstance(i, Table) for i in concerts])) + + self.assertEqual( + [i.name for i in concerts], ["Rockfest", "Classicfest"] + ) From 442d5150f01563e613b50774e80cf639d112b9ac Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 18 Dec 2021 08:55:48 +0000 Subject: [PATCH 190/727] fix typos - thanks sinisaos --- piccolo/table.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/piccolo/table.py b/piccolo/table.py index a90744e03..2b95f03ed 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -423,7 +423,7 @@ def get_m2m(self, m2m: M2M) -> M2MGetRelated: .. code-block:: python - >>> band = await Band.objects().get(name="Pythonistas") + >>> band = await Band.objects().get(Band.name == "Pythonistas") >>> await band.get_m2m(Band.genres) [, ] @@ -442,7 +442,7 @@ def add_m2m( .. code-block:: python - >>> band = await Band.objects().get(name="Pythonistas") + >>> band = await Band.objects().get(Band.name == "Pythonistas") >>> await band.add_m2m( >>> Genre(name="Punk rock"), >>> m2m=Band.genres @@ -583,7 +583,7 @@ def __str__(self) -> str: return self.querystring.__str__() def __repr__(self) -> str: - _pk = self._meta.primary_key or None + _pk = getattr(self, self._meta.primary_key._meta.name, None) return f"<{self.__class__.__name__}: {_pk}>" ########################################################################### From de4792d133aad36c94c4fc7ddb1944fcd01c4226 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 18 Dec 2021 09:04:39 +0000 Subject: [PATCH 191/727] fix tests for `Table.__repr__` --- piccolo/table.py | 8 ++++++-- tests/table/test_repr.py | 20 ++------------------ 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/piccolo/table.py b/piccolo/table.py index 2b95f03ed..7d72d1e1c 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -583,8 +583,12 @@ def __str__(self) -> str: return self.querystring.__str__() def __repr__(self) -> str: - _pk = getattr(self, self._meta.primary_key._meta.name, None) - return f"<{self.__class__.__name__}: {_pk}>" + pk = ( + None + if not self._exists_in_db + else getattr(self, self._meta.primary_key._meta.name, None) + ) + return f"<{self.__class__.__name__}: {pk}>" ########################################################################### # Classmethods diff --git a/tests/table/test_repr.py b/tests/table/test_repr.py index 08f0ff783..3e036306e 100644 --- a/tests/table/test_repr.py +++ b/tests/table/test_repr.py @@ -3,28 +3,12 @@ class TestTableRepr(DBTestCase): - @postgres_only def test_repr_postgres(self): self.assertEqual( Manager().__repr__(), - '', + "", ) self.insert_row() manager = Manager.objects().first().run_sync() - self.assertEqual( - manager.__repr__(), f"" - ) - - @sqlite_only - def test_repr_sqlite(self): - self.assertEqual( - Manager().__repr__(), - '', - ) - - self.insert_row() - manager = Manager.objects().first().run_sync() - self.assertEqual( - manager.__repr__(), f"" - ) + self.assertEqual(manager.__repr__(), "") From 30058ee371822a5b6f2ecbb7f344d153bbe59a4e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 18 Dec 2021 09:11:33 +0000 Subject: [PATCH 192/727] remove unused imports --- tests/table/test_repr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/table/test_repr.py b/tests/table/test_repr.py index 3e036306e..a7ff329a1 100644 --- a/tests/table/test_repr.py +++ b/tests/table/test_repr.py @@ -1,4 +1,4 @@ -from tests.base import DBTestCase, postgres_only, sqlite_only +from tests.base import DBTestCase from tests.example_apps.music.tables import Manager From 1b64dc75ab0681755986a2582de1cedf82eee0b8 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 18 Dec 2021 09:20:59 +0000 Subject: [PATCH 193/727] add test for adding an existing element to the joining table --- piccolo/columns/m2m.py | 3 ++- piccolo/table.py | 2 +- tests/columns/test_m2m.py | 27 +++++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/piccolo/columns/m2m.py b/piccolo/columns/m2m.py index 96d706d86..168db3c13 100644 --- a/piccolo/columns/m2m.py +++ b/piccolo/columns/m2m.py @@ -171,7 +171,8 @@ async def run(self): unsaved = [i for i in rows if not i._exists_in_db] async with rows[0]._meta.db.transaction(): - await rows[0].__class__.insert(*unsaved) + if unsaved: + await rows[0].__class__.insert(*unsaved) joining_table = self.m2m._meta.resolved_joining_table diff --git a/piccolo/table.py b/piccolo/table.py index 7d72d1e1c..f9e4063d3 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -447,7 +447,7 @@ def add_m2m( >>> Genre(name="Punk rock"), >>> m2m=Band.genres >>> ) - + [{'id': 1}] :param extra_column_values: If the joining table has additional columns besides the two diff --git a/tests/columns/test_m2m.py b/tests/columns/test_m2m.py index 291a25114..19854eb20 100644 --- a/tests/columns/test_m2m.py +++ b/tests/columns/test_m2m.py @@ -100,6 +100,33 @@ def test_add_m2m(self): 1, ) + def test_add_m2m_existing(self): + """ + Make sure we can add an existing element to the joining table. + """ + band: Band = Band.objects().get(Band.name == "Pythonistas").run_sync() + + genre: Genre = ( + Genre.objects().get(Genre.name == "Classical").run_sync() + ) + + band.add_m2m(genre, m2m=Band.genres).run_sync() + + # We shouldn't have created a duplicate genre in the database. + self.assertEqual( + Genre.count().where(Genre.name == "Classical").run_sync(), 1 + ) + + self.assertEqual( + GenreToBand.count() + .where( + GenreToBand.band.name == "Pythonistas", + GenreToBand.genre.name == "Classical", + ) + .run_sync(), + 1, + ) + def test_get_m2m(self): """ Make sure we can get related items via the joining table. From aa30e19af9f61a2d0863b59bf6aa971ae71db843 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 18 Dec 2021 11:56:10 +0000 Subject: [PATCH 194/727] added `remove_m2m` --- docs/src/piccolo/schema/m2m.rst | 5 +++++ piccolo/columns/m2m.py | 40 +++++++++++++++++++++++++++++++-- piccolo/table.py | 28 +++++++++++++++++++++-- tests/columns/test_m2m.py | 31 +++++++++++++++++++++++++ 4 files changed, 100 insertions(+), 4 deletions(-) diff --git a/docs/src/piccolo/schema/m2m.rst b/docs/src/piccolo/schema/m2m.rst index 98e6d422c..c91099856 100644 --- a/docs/src/piccolo/schema/m2m.rst +++ b/docs/src/piccolo/schema/m2m.rst @@ -114,3 +114,8 @@ get_m2m ------- .. automethod:: Table.get_m2m + +remove_m2m +---------- + +.. automethod:: Table.remove_m2m diff --git a/piccolo/columns/m2m.py b/piccolo/columns/m2m.py index 168db3c13..c270aaba9 100644 --- a/piccolo/columns/m2m.py +++ b/piccolo/columns/m2m.py @@ -172,7 +172,7 @@ async def run(self): async with rows[0]._meta.db.transaction(): if unsaved: - await rows[0].__class__.insert(*unsaved) + await rows[0].__class__.insert(*unsaved).run() joining_table = self.m2m._meta.resolved_joining_table @@ -198,7 +198,43 @@ async def run(self): ) joining_table_rows.append(joining_table_row) - return await joining_table.insert(*joining_table_rows) + return await joining_table.insert(*joining_table_rows).run() + + def run_sync(self): + return run_sync(self.run()) + + def __await__(self): + return self.run().__await__() + + +@dataclass +class M2MRemoveRelated: + + target_row: Table + m2m: M2M + rows: t.Sequence[Table] + + async def run(self): + fk = self.m2m._meta.secondary_foreign_key + related_table = fk._foreign_key_meta.resolved_references + + row_ids = [] + + for row in self.rows: + if row.__class__ != related_table: + raise ValueError("The row belongs to the wrong table!") + + row_id = getattr(row, row._meta.primary_key._meta.name) + if row_id: + row_ids.append(row_id) + + if row_ids: + joining_table = self.m2m._meta.table + return await joining_table.delete().where( + joining_table._meta.primary_key.is_in(row_ids) + ) + + return None def run_sync(self): return run_sync(self.run()) diff --git a/piccolo/table.py b/piccolo/table.py index f9e4063d3..a8ed35f19 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -16,7 +16,12 @@ ) from piccolo.columns.defaults.base import Default from piccolo.columns.indexes import IndexMethod -from piccolo.columns.m2m import M2M, M2MAddRelated, M2MGetRelated +from piccolo.columns.m2m import ( + M2M, + M2MAddRelated, + M2MGetRelated, + M2MRemoveRelated, +) from piccolo.columns.readable import Readable from piccolo.columns.reference import LAZY_COLUMN_REFERENCES from piccolo.engine import Engine, engine_finder @@ -416,7 +421,6 @@ def get_related(self, foreign_key: t.Union[ForeignKey, str]) -> Objects: .first() ) - # TODO - I might merge this with ``get_related``. def get_m2m(self, m2m: M2M) -> M2MGetRelated: """ Get all matching rows via the join table. @@ -462,6 +466,26 @@ def add_m2m( extra_column_values=extra_column_values, ) + def remove_m2m(self, *rows: Table, m2m: M2M) -> M2MRemoveRelated: + """ + Remove the rows from the joining table. + + .. code-block:: python + + >>> band = await Band.objects().get(Band.name == "Pythonistas") + >>> genre = await Genre.objects().get(Genre.name == "Rock") + >>> await band.remove_m2m( + >>> genre, + >>> m2m=Band.genres + >>> ) + + """ + return M2MRemoveRelated( + target_row=self, + rows=rows, + m2m=m2m, + ) + def to_dict(self, *columns: Column) -> t.Dict[str, t.Any]: """ A convenience method which returns a dictionary, mapping column names diff --git a/tests/columns/test_m2m.py b/tests/columns/test_m2m.py index 19854eb20..d2c4a0f8a 100644 --- a/tests/columns/test_m2m.py +++ b/tests/columns/test_m2m.py @@ -139,6 +139,37 @@ def test_get_m2m(self): self.assertEqual([i.name for i in genres], ["Rock", "Folk"]) + def test_remove_m2m(self): + """ + Make sure we can remove related items via the joining table. + """ + band: Band = Band.objects().get(Band.name == "Pythonistas").run_sync() + + genre = Genre.objects().get(Genre.name == "Classical").run_sync() + + band.remove_m2m(genre, m2m=Band.genres).run_sync() + + self.assertEqual( + GenreToBand.count() + .where( + GenreToBand.band.name == "Pythonistas", + GenreToBand.genre.name == "Classical", + ) + .run_sync(), + 0, + ) + + # Make sure the other one wasn't removed: + self.assertEqual( + GenreToBand.count() + .where( + GenreToBand.band.name == "Pythonistas", + GenreToBand.genre.name == "Rock", + ) + .run_sync(), + 1, + ) + ############################################################################### From 650c5cdc1ef24800907099cb7acf8febe061e409 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 18 Dec 2021 18:03:04 +0000 Subject: [PATCH 195/727] fixing `remove_m2m` bugs --- piccolo/columns/m2m.py | 10 +++++++--- tests/columns/test_m2m.py | 16 +++++++++++++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/piccolo/columns/m2m.py b/piccolo/columns/m2m.py index c270aaba9..7e7f736df 100644 --- a/piccolo/columns/m2m.py +++ b/piccolo/columns/m2m.py @@ -229,9 +229,13 @@ async def run(self): row_ids.append(row_id) if row_ids: - joining_table = self.m2m._meta.table - return await joining_table.delete().where( - joining_table._meta.primary_key.is_in(row_ids) + return ( + await self.m2m._meta.resolved_joining_table.delete() + .where( + self.m2m._meta.primary_foreign_key == self.target_row, + self.m2m._meta.secondary_foreign_key.is_in(row_ids), + ) + .run() ) return None diff --git a/tests/columns/test_m2m.py b/tests/columns/test_m2m.py index d2c4a0f8a..7fcb700d3 100644 --- a/tests/columns/test_m2m.py +++ b/tests/columns/test_m2m.py @@ -145,7 +145,7 @@ def test_remove_m2m(self): """ band: Band = Band.objects().get(Band.name == "Pythonistas").run_sync() - genre = Genre.objects().get(Genre.name == "Classical").run_sync() + genre = Genre.objects().get(Genre.name == "Rock").run_sync() band.remove_m2m(genre, m2m=Band.genres).run_sync() @@ -153,17 +153,27 @@ def test_remove_m2m(self): GenreToBand.count() .where( GenreToBand.band.name == "Pythonistas", - GenreToBand.genre.name == "Classical", + GenreToBand.genre.name == "Rock", ) .run_sync(), 0, ) - # Make sure the other one wasn't removed: + # Make sure the others weren't removed: self.assertEqual( GenreToBand.count() .where( GenreToBand.band.name == "Pythonistas", + GenreToBand.genre.name == "Folk", + ) + .run_sync(), + 1, + ) + + self.assertEqual( + GenreToBand.count() + .where( + GenreToBand.band.name == "C-Sharps", GenreToBand.genre.name == "Rock", ) .run_sync(), From ce16f2221c081ed5a2e45dc03241edd1ad928e4e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 18 Dec 2021 18:10:51 +0000 Subject: [PATCH 196/727] clean up SQL used by `M2MSelect` --- piccolo/columns/m2m.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/piccolo/columns/m2m.py b/piccolo/columns/m2m.py index 7e7f736df..9f4d884b0 100644 --- a/piccolo/columns/m2m.py +++ b/piccolo/columns/m2m.py @@ -47,16 +47,16 @@ def get_select_string(self, engine_type: str, just_alias=False) -> str: return f""" ARRAY( SELECT - inner_{table_2_name}.{column_name} - from {m2m_table_name} - join {table_1_name} inner_{table_1_name} on ( - {m2m_table_name}.{fk_1_name} = inner_{table_1_name}.{table_1_pk_name} + "inner_{table_2_name}"."{column_name}" + FROM "{m2m_table_name}" + JOIN "{table_1_name}" "inner_{table_1_name}" ON ( + "{m2m_table_name}"."{fk_1_name}" = "inner_{table_1_name}"."{table_1_pk_name}" ) - join {table_2_name} inner_{table_2_name} on ( - {m2m_table_name}.{fk_2_name} = inner_{table_2_name}.{table_2_pk_name} + JOIN "{table_2_name}" "inner_{table_2_name}" ON ( + "{m2m_table_name}"."{fk_2_name}" = "inner_{table_2_name}"."{table_2_pk_name}" ) - where {m2m_table_name}.{fk_1_name} = {table_1_name}.{table_1_pk_name} - ) as {m2m_relationship_name} + WHERE "{m2m_table_name}"."{fk_1_name}" = "{table_1_name}"."{table_1_pk_name}" + ) AS "{m2m_relationship_name}" """ # noqa: E501 From b15905b93a7906d96a32bd5320f8d4972cc1a3f7 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 18 Dec 2021 20:07:57 +0000 Subject: [PATCH 197/727] added sqlite support --- piccolo/columns/m2m.py | 45 +++++++++++++++++++++++---------- piccolo/engine/sqlite.py | 8 +++++- piccolo/query/methods/select.py | 22 ++++++++++++++++ tests/columns/test_m2m.py | 43 ++++++++++++++++++++++--------- 4 files changed, 91 insertions(+), 27 deletions(-) diff --git a/piccolo/columns/m2m.py b/piccolo/columns/m2m.py index 9f4d884b0..df5af57c0 100644 --- a/piccolo/columns/m2m.py +++ b/piccolo/columns/m2m.py @@ -44,21 +44,38 @@ def get_select_string(self, engine_type: str, just_alias=False) -> str: column_name = self.column._meta.db_column_name - return f""" - ARRAY( - SELECT - "inner_{table_2_name}"."{column_name}" - FROM "{m2m_table_name}" - JOIN "{table_1_name}" "inner_{table_1_name}" ON ( - "{m2m_table_name}"."{fk_1_name}" = "inner_{table_1_name}"."{table_1_pk_name}" - ) - JOIN "{table_2_name}" "inner_{table_2_name}" ON ( - "{m2m_table_name}"."{fk_2_name}" = "inner_{table_2_name}"."{table_2_pk_name}" - ) - WHERE "{m2m_table_name}"."{fk_1_name}" = "{table_1_name}"."{table_1_pk_name}" - ) AS "{m2m_relationship_name}" + inner_select = f""" + "{m2m_table_name}" + JOIN "{table_1_name}" "inner_{table_1_name}" ON ( + "{m2m_table_name}"."{fk_1_name}" = "inner_{table_1_name}"."{table_1_pk_name}" + ) + JOIN "{table_2_name}" "inner_{table_2_name}" ON ( + "{m2m_table_name}"."{fk_2_name}" = "inner_{table_2_name}"."{table_2_pk_name}" + ) + WHERE "{m2m_table_name}"."{fk_1_name}" = "{table_1_name}"."{table_1_pk_name}" """ # noqa: E501 + if engine_type == "postgres": + return f""" + ARRAY( + SELECT + "inner_{table_2_name}"."{column_name}" + FROM {inner_select} + ) AS "{m2m_relationship_name}" + """ + elif engine_type == "sqlite": + return f""" + ( + SELECT group_concat( + "inner_{table_2_name}"."{column_name}" + ) + FROM {inner_select} + ) + AS "{m2m_relationship_name} [M2M]" + """ + else: + raise ValueError(f"{engine_type} is an unrecognised engine type") + @dataclass class M2MMeta: @@ -310,7 +327,7 @@ def __init__( _foreign_key_columns=foreign_key_columns, ) - def __call__(self, column: Column) -> Selectable: + def __call__(self, column: Column) -> M2MSelect: """ :param column: Which column to include from the related table. diff --git a/piccolo/engine/sqlite.py b/piccolo/engine/sqlite.py index aece06b6e..d363f6725 100644 --- a/piccolo/engine/sqlite.py +++ b/piccolo/engine/sqlite.py @@ -165,6 +165,11 @@ def convert_array_out(value: bytes) -> t.List: return load_json(value.decode("utf8")) +def convert_M2M_out(value: bytes) -> t.List: + _value = value.decode("utf8") + return _value.split(",") + + sqlite3.register_converter("Numeric", convert_numeric_out) sqlite3.register_converter("Integer", convert_int_out) sqlite3.register_converter("UUID", convert_uuid_out) @@ -175,6 +180,7 @@ def convert_array_out(value: bytes) -> t.List: sqlite3.register_converter("Timestamp", convert_timestamp_out) sqlite3.register_converter("Timestamptz", convert_timestamptz_out) sqlite3.register_converter("Array", convert_array_out) +sqlite3.register_converter("M2M", convert_M2M_out) sqlite3.register_adapter(Decimal, convert_numeric_in) sqlite3.register_adapter(uuid.UUID, convert_uuid_in) @@ -351,7 +357,7 @@ class SQLiteEngine(Engine): def __init__( self, path: str = "piccolo.sqlite", - detect_types=sqlite3.PARSE_DECLTYPES, + detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES, isolation_level=None, **connection_kwargs, ) -> None: diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index ea67b2249..0728558b5 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -5,6 +5,7 @@ from collections import OrderedDict from piccolo.columns import Column, Selectable +from piccolo.columns.m2m import M2MSelect from piccolo.columns.readable import Readable from piccolo.engine.base import Batch from piccolo.query.base import Query @@ -20,6 +21,7 @@ ) from piccolo.querystring import QueryString from piccolo.utils.dictionary import make_nested +from piccolo.utils.warnings import colored_warning if t.TYPE_CHECKING: # pragma: no cover from piccolo.custom_types import Combinable @@ -272,6 +274,26 @@ def offset(self, number: int) -> Select: return self async def response_handler(self, response): + # With M2M queries in SQLite, we always get the value back as a list + # of strings, so we need to do some type conversion. + if self.engine_type == "sqlite": + m2m_selects = [ + i + for i in self.columns_delegate.selected_columns + if isinstance(i, M2MSelect) + ] + for m2m_select in m2m_selects: + m2m_name = m2m_select.m2m._meta.name + value_type = m2m_select.column.__class__.value_type + try: + for row in response: + row[m2m_name] = [value_type(i) for i in row[m2m_name]] + except ValueError: + colored_warning( + "Unable to do type converstion for the " + f"{m2m_name} relation" + ) + # If no columns were specified, it's a select *, so we know that # no columns were selected from related tables. was_select_star = len(self.columns_delegate.selected_columns) == 0 diff --git a/tests/columns/test_m2m.py b/tests/columns/test_m2m.py index 7fcb700d3..430e23837 100644 --- a/tests/columns/test_m2m.py +++ b/tests/columns/test_m2m.py @@ -8,7 +8,6 @@ ) from piccolo.columns.m2m import M2M from piccolo.table import Table, create_tables, drop_tables -from tests.base import postgres_only class Band(Table): @@ -29,7 +28,6 @@ class GenreToBand(Table): TABLES_1 = [Band, Genre, GenreToBand] -@postgres_only class TestM2M(TestCase): def setUp(self): create_tables(*TABLES_1, if_not_exists=True) @@ -47,17 +45,17 @@ def setUp(self): ).run_sync() GenreToBand.insert( - GenreToBand(genre=1, band=1), - GenreToBand(genre=1, band=3), - GenreToBand(genre=2, band=1), - GenreToBand(genre=2, band=2), - GenreToBand(genre=3, band=3), + GenreToBand(band=1, genre=1), + GenreToBand(band=1, genre=2), + GenreToBand(band=2, genre=2), + GenreToBand(band=3, genre=1), + GenreToBand(band=3, genre=3), ).run_sync() def tearDown(self): drop_tables(*TABLES_1) - def test_select(self): + def test_select_name(self): response = Band.select(Band.name, Band.genres(Genre.name)).run_sync() self.assertEqual( response, @@ -79,6 +77,28 @@ def test_select(self): ], ) + def test_select_id(self): + response = Band.select(Band.name, Band.genres(Genre.id)).run_sync() + self.assertEqual( + response, + [ + {"name": "Pythonistas", "genres": [1, 2]}, + {"name": "Rustaceans", "genres": [2]}, + {"name": "C-Sharps", "genres": [1, 3]}, + ], + ) + + # Now try it in reverse. + response = Genre.select(Genre.name, Genre.bands(Band.id)).run_sync() + self.assertEqual( + response, + [ + {"name": "Rock", "bands": [1, 3]}, + {"name": "Folk", "bands": [1, 2]}, + {"name": "Classical", "bands": [3]}, + ], + ) + def test_add_m2m(self): """ Make sure we can add items to the joining table. @@ -210,7 +230,6 @@ class CustomerToConcert(Table): TABLES_2 = [Customer, Concert, CustomerToConcert] -@postgres_only class TestM2MCustomPrimaryKey(TestCase): """ Make sure the M2M functionality works correctly when the tables have custom @@ -244,7 +263,7 @@ def test_select(self): Customer.name, Customer.concerts(Concert.name) ).run_sync() - self.assertEqual( + self.assertListEqual( response, [ {"name": "Bob", "concerts": ["Rockfest", "Classicfest"]}, @@ -258,7 +277,7 @@ def test_select(self): Concert.name, Concert.customers(Customer.name) ).run_sync() - self.assertEqual( + self.assertListEqual( response, [ {"name": "Rockfest", "customers": ["Bob", "Sally"]}, @@ -304,6 +323,6 @@ def test_get_m2m(self): self.assertTrue(all([isinstance(i, Table) for i in concerts])) - self.assertEqual( + self.assertCountEqual( [i.name for i in concerts], ["Rockfest", "Classicfest"] ) From e34d91cb89952659abb54afa00127de5e5c1f24e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 18 Dec 2021 20:18:48 +0000 Subject: [PATCH 198/727] add an example for `extra_column_values` --- piccolo/table.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/piccolo/table.py b/piccolo/table.py index a8ed35f19..0da3a92a7 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -456,7 +456,26 @@ def add_m2m( :param extra_column_values: If the joining table has additional columns besides the two required foreign keys, you can specify the values for those - additional columns. + additional columns. For example, if this is our joining table: + + .. code-block:: python + + class GenreToBand(Table): + band = ForeignKey(Band) + genre = ForeignKey(Genre) + reason = Text() + + We can provide the ``reason`` value: + + .. code-block:: python + + await band.add_m2m( + Genre(name="Punk rock"), + m2m=Band.genres, + extra_column_values={ + "reason": "Their second album was very punk." + } + ) """ return M2MAddRelated( From 70e6fb6379c1bd619fd8ea0f07abae78e46f3d46 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 18 Dec 2021 20:52:13 +0000 Subject: [PATCH 199/727] added test for ``extra_column_values`` arg --- tests/columns/test_m2m.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/columns/test_m2m.py b/tests/columns/test_m2m.py index 430e23837..f4cf0f3ef 100644 --- a/tests/columns/test_m2m.py +++ b/tests/columns/test_m2m.py @@ -4,6 +4,7 @@ UUID, ForeignKey, LazyTableReference, + Text, Varchar, ) from piccolo.columns.m2m import M2M @@ -23,6 +24,7 @@ class Genre(Table): class GenreToBand(Table): band = ForeignKey(Band) genre = ForeignKey(Genre) + reason = Text(help_text="For testing additional columns on join tables.") TABLES_1 = [Band, Genre, GenreToBand] @@ -120,6 +122,32 @@ def test_add_m2m(self): 1, ) + def test_add_m2m_extra_columns(self): + """ + Make sure the ``extra_column_values`` parameter works correctly. + """ + reason = "Their second album was very punk rock." + + band: Band = Band.objects().get(Band.name == "Pythonistas").run_sync() + band.add_m2m( + Genre(name="Punk Rock"), + m2m=Band.genres, + extra_column_values={ + "reason": "Their second album was very punk rock." + }, + ).run_sync() + + genre_to_band = ( + GenreToBand.objects() + .get( + (GenreToBand.band.name == "Pythonistas") + & (GenreToBand.genre.name == "Punk Rock") + ) + .run_sync() + ) + + self.assertEqual(genre_to_band.reason, reason) + def test_add_m2m_existing(self): """ Make sure we can add an existing element to the joining table. From 257432f28c5398b4940442895f7a2d71e3c79653 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 18 Dec 2021 22:02:03 +0000 Subject: [PATCH 200/727] added test for `extra_column_values` when the keys are column classes --- piccolo/columns/m2m.py | 7 +++++++ tests/columns/test_m2m.py | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/piccolo/columns/m2m.py b/piccolo/columns/m2m.py index df5af57c0..6bbe74fba 100644 --- a/piccolo/columns/m2m.py +++ b/piccolo/columns/m2m.py @@ -183,6 +183,13 @@ class M2MAddRelated: rows: t.Sequence[Table] extra_column_values: t.Dict[t.Union[Column, str], t.Any] + def __post_init__(self): + # Normalise `extra_column_values`, so we just have the column names. + self.extra_column_values: t.Dict[str, t.Any] = { + i._meta.name if isinstance(i, Column) else i: j + for i, j in self.extra_column_values.items() + } + async def run(self): rows = self.rows unsaved = [i for i in rows if not i._exists_in_db] diff --git a/tests/columns/test_m2m.py b/tests/columns/test_m2m.py index f4cf0f3ef..f5fe9c434 100644 --- a/tests/columns/test_m2m.py +++ b/tests/columns/test_m2m.py @@ -122,9 +122,10 @@ def test_add_m2m(self): 1, ) - def test_add_m2m_extra_columns(self): + def test_extra_columns_str(self): """ - Make sure the ``extra_column_values`` parameter works correctly. + Make sure the ``extra_column_values`` parameter for ``add_m2m`` works + correctly when the dictionary keys are strings. """ reason = "Their second album was very punk rock." @@ -148,6 +149,33 @@ def test_add_m2m_extra_columns(self): self.assertEqual(genre_to_band.reason, reason) + def test_extra_columns_class(self): + """ + Make sure the ``extra_column_values`` parameter for ``add_m2m`` works + correctly when the dictionary keys are ``Column`` classes. + """ + reason = "Their second album was very punk rock." + + band: Band = Band.objects().get(Band.name == "Pythonistas").run_sync() + band.add_m2m( + Genre(name="Punk Rock"), + m2m=Band.genres, + extra_column_values={ + GenreToBand.reason: "Their second album was very punk rock." + }, + ).run_sync() + + genre_to_band = ( + GenreToBand.objects() + .get( + (GenreToBand.band.name == "Pythonistas") + & (GenreToBand.genre.name == "Punk Rock") + ) + .run_sync() + ) + + self.assertEqual(genre_to_band.reason, reason) + def test_add_m2m_existing(self): """ Make sure we can add an existing element to the joining table. From e9b382d4d2935773ac3892f791c25c81f3332eb6 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 18 Dec 2021 22:51:03 +0000 Subject: [PATCH 201/727] added integration test for M2M and migrations --- .../auto/integration/test_migrations.py | 196 +++++++++++------- 1 file changed, 121 insertions(+), 75 deletions(-) diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index 6be92a612..1758a043c 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -28,6 +28,7 @@ Date, Decimal, DoublePrecision, + ForeignKey, Integer, Interval, Numeric, @@ -41,8 +42,10 @@ Varchar, ) from piccolo.columns.defaults.uuid import UUID4 +from piccolo.columns.m2m import M2M +from piccolo.columns.reference import LazyTableReference from piccolo.conf.apps import AppConfig -from piccolo.table import Table, create_table_class +from piccolo.table import Table, create_table_class, drop_tables from piccolo.utils.sync import run_sync from tests.base import DBTestCase, postgres_only @@ -94,17 +97,7 @@ def array_default_varchar(): return ["x", "y", "z"] -@postgres_only -class TestMigrations(DBTestCase): - def setUp(self): - pass - - def tearDown(self): - create_table_class("MyTable").alter().drop_table( - if_exists=True - ).run_sync() - Migration.alter().drop_table(if_exists=True).run_sync() - +class MigrationTestCase(DBTestCase): def run_migrations(self, app_config: AppConfig): manager = ForwardsMigrationManager(app_name=app_config.app_name) run_sync(manager.create_migration_table()) @@ -112,15 +105,15 @@ def run_migrations(self, app_config: AppConfig): def _test_migrations( self, - table_classes: t.List[t.Type[Table]], + table_snapshots: t.List[t.List[t.Type[Table]]], test_function: t.Optional[t.Callable[[RowMeta], None]] = None, ): """ Writes a migration file to disk and runs it. - :param table_classes: - Migrations will be created and run based on successive table - classes in this list. + :param table_snapshots: + A list of lists. Each sub list represents a snapshot of the table + state. Migrations will be created and run based each snapshot. :param test_function: After the migrations are run, this function is called. It is passed a ``RowMeta`` instance which can be used to check the column was @@ -144,8 +137,8 @@ def _test_migrations( table_classes=[], ) - for table_class in table_classes: - app_config.table_classes = [table_class] + for table_snapshot in table_snapshots: + app_config.table_classes = table_snapshot meta = run_sync( _create_new_migration( app_config=app_config, auto=True, auto_input="y" @@ -161,7 +154,7 @@ def _test_migrations( if test_function: column_name = ( - table_classes[-1] + table_snapshots[-1][-1] ._meta.non_default_columns[0] ._meta.db_column_name ) @@ -174,6 +167,18 @@ def _test_migrations( msg=f"Meta is incorrect: {row_meta}", ) + +@postgres_only +class TestMigrations(MigrationTestCase): + def setUp(self): + pass + + def tearDown(self): + create_table_class("MyTable").alter().drop_table( + if_exists=True + ).run_sync() + Migration.alter().drop_table(if_exists=True).run_sync() + ########################################################################### def table(self, column: Column): @@ -183,8 +188,8 @@ def table(self, column: Column): def test_varchar_column(self): self._test_migrations( - table_classes=[ - self.table(column) + table_snapshots=[ + [self.table(column)] for column in [ Varchar(), Varchar(length=100), @@ -207,8 +212,8 @@ def test_varchar_column(self): def test_text_column(self): self._test_migrations( - table_classes=[ - self.table(column) + table_snapshots=[ + [self.table(column)] for column in [ Text(), Text(default="hello world"), @@ -230,8 +235,8 @@ def test_text_column(self): def test_integer_column(self): self._test_migrations( - table_classes=[ - self.table(column) + table_snapshots=[ + [self.table(column)] for column in [ Integer(), Integer(default=1), @@ -253,8 +258,8 @@ def test_integer_column(self): def test_real_column(self): self._test_migrations( - table_classes=[ - self.table(column) + table_snapshots=[ + [self.table(column)] for column in [ Real(), Real(default=1.1), @@ -275,8 +280,8 @@ def test_real_column(self): def test_double_precision_column(self): self._test_migrations( - table_classes=[ - self.table(column) + table_snapshots=[ + [self.table(column)] for column in [ DoublePrecision(), DoublePrecision(default=1.1), @@ -297,8 +302,8 @@ def test_double_precision_column(self): def test_smallint_column(self): self._test_migrations( - table_classes=[ - self.table(column) + table_snapshots=[ + [self.table(column)] for column in [ SmallInt(), SmallInt(default=1), @@ -320,8 +325,8 @@ def test_smallint_column(self): def test_bigint_column(self): self._test_migrations( - table_classes=[ - self.table(column) + table_snapshots=[ + [self.table(column)] for column in [ BigInt(), BigInt(default=1), @@ -343,8 +348,8 @@ def test_bigint_column(self): def test_uuid_column(self): self._test_migrations( - table_classes=[ - self.table(column) + table_snapshots=[ + [self.table(column)] for column in [ UUID(), UUID(default="ecf338cd-6da7-464c-b31e-4b2e3e12f0f0"), @@ -373,8 +378,8 @@ def test_uuid_column(self): def test_timestamp_column(self): self._test_migrations( - table_classes=[ - self.table(column) + table_snapshots=[ + [self.table(column)] for column in [ Timestamp(), Timestamp( @@ -399,8 +404,8 @@ def test_timestamp_column(self): def test_time_column(self): self._test_migrations( - table_classes=[ - self.table(column) + table_snapshots=[ + [self.table(column)] for column in [ Time(), Time(default=datetime.time(hour=12, minute=0)), @@ -424,8 +429,8 @@ def test_time_column(self): def test_date_column(self): self._test_migrations( - table_classes=[ - self.table(column) + table_snapshots=[ + [self.table(column)] for column in [ Date(), Date(default=datetime.date(year=2021, month=1, day=1)), @@ -449,8 +454,8 @@ def test_date_column(self): def test_interval_column(self): self._test_migrations( - table_classes=[ - self.table(column) + table_snapshots=[ + [self.table(column)] for column in [ Interval(), Interval(default=datetime.timedelta(days=1)), @@ -472,8 +477,8 @@ def test_interval_column(self): def test_boolean_column(self): self._test_migrations( - table_classes=[ - self.table(column) + table_snapshots=[ + [self.table(column)] for column in [ Boolean(), Boolean(default=True), @@ -495,8 +500,8 @@ def test_boolean_column(self): def test_numeric_column(self): self._test_migrations( - table_classes=[ - self.table(column) + table_snapshots=[ + [self.table(column)] for column in [ Numeric(), Numeric(digits=(4, 2)), @@ -520,8 +525,8 @@ def test_numeric_column(self): def test_decimal_column(self): self._test_migrations( - table_classes=[ - self.table(column) + table_snapshots=[ + [self.table(column)] for column in [ Decimal(), Decimal(digits=(4, 2)), @@ -545,8 +550,8 @@ def test_decimal_column(self): def test_array_column_integer(self): self._test_migrations( - table_classes=[ - self.table(column) + table_snapshots=[ + [self.table(column)] for column in [ Array(base_column=Integer()), Array(base_column=Integer(), default=[1, 2, 3]), @@ -570,8 +575,8 @@ def test_array_column_integer(self): def test_array_column_varchar(self): self._test_migrations( - table_classes=[ - self.table(column) + table_snapshots=[ + [self.table(column)] for column in [ Array(base_column=Varchar()), Array(base_column=Varchar(), default=["a", "b", "c"]), @@ -600,8 +605,8 @@ def test_array_column_varchar(self): def test_json_column(self): self._test_migrations( - table_classes=[ - self.table(column) + table_snapshots=[ + [self.table(column)] for column in [ JSON(), JSON(default=["a", "b", "c"]), @@ -622,8 +627,8 @@ def test_json_column(self): def test_jsonb_column(self): self._test_migrations( - table_classes=[ - self.table(column) + table_snapshots=[ + [self.table(column)] for column in [ JSONB(), JSONB(default=["a", "b", "c"]), @@ -646,8 +651,8 @@ def test_jsonb_column(self): def test_db_column_name(self): self._test_migrations( - table_classes=[ - self.table(column) + table_snapshots=[ + [self.table(column)] for column in [ Varchar(), Varchar(db_column_name="custom_name"), @@ -670,8 +675,8 @@ def test_db_column_name_initial(self): ``db_column_name`` specified, then the column has the correct name. """ self._test_migrations( - table_classes=[ - self.table(column) + table_snapshots=[ + [self.table(column)] for column in [ Varchar(db_column_name="custom_name"), ] @@ -687,14 +692,16 @@ def test_db_column_name_initial(self): ########################################################################### + # Column type conversion + def test_column_type_conversion_string(self): """ We can't manage all column type conversions, but should be able to manage most simple ones (e.g. Varchar to Text). """ self._test_migrations( - table_classes=[ - self.table(column) + table_snapshots=[ + [self.table(column)] for column in [ Varchar(), Text(), @@ -705,8 +712,8 @@ def test_column_type_conversion_string(self): def test_column_type_conversion_integer(self): self._test_migrations( - table_classes=[ - self.table(column) + table_snapshots=[ + [self.table(column)] for column in [ Integer(), BigInt(), @@ -719,8 +726,8 @@ def test_column_type_conversion_integer(self): def test_column_type_conversion_string_to_integer(self): self._test_migrations( - table_classes=[ - self.table(column) + table_snapshots=[ + [self.table(column)] for column in [ Varchar(default="1"), Integer(default=1), @@ -731,8 +738,8 @@ def test_column_type_conversion_string_to_integer(self): def test_column_type_conversion_float_decimal(self): self._test_migrations( - table_classes=[ - self.table(column) + table_snapshots=[ + [self.table(column)] for column in [ Real(default=1.0), DoublePrecision(default=1.0), @@ -745,8 +752,8 @@ def test_column_type_conversion_float_decimal(self): def test_column_type_conversion_json(self): self._test_migrations( - table_classes=[ - self.table(column) + table_snapshots=[ + [self.table(column)] for column in [ JSON(), JSONB(), @@ -757,8 +764,8 @@ def test_column_type_conversion_json(self): def test_column_type_conversion_timestamp(self): self._test_migrations( - table_classes=[ - self.table(column) + table_snapshots=[ + [self.table(column)] for column in [ Timestamp(), Timestamptz(), @@ -775,8 +782,8 @@ def test_column_type_conversion_serial(self, colored_warning: MagicMock): should just output a warning. """ self._test_migrations( - table_classes=[ - self.table(column) + table_snapshots=[ + [self.table(column)] for column in [ Serial(), BigSerial(), @@ -788,3 +795,42 @@ def test_column_type_conversion_serial(self, colored_warning: MagicMock): "Unable to migrate Serial to BigSerial and vice versa. This must " "be done manually." ) + + +############################################################################### + + +class Band(Table): + name = Varchar() + genres = M2M(LazyTableReference("GenreToBand", module_path=__name__)) + + +class Genre(Table): + name = Varchar() + bands = M2M(LazyTableReference("GenreToBand", module_path=__name__)) + + +class GenreToBand(Table): + band = ForeignKey(Band) + genre = ForeignKey(Genre) + + +@postgres_only +class TestM2MMigrations(MigrationTestCase): + def setUp(self): + pass + + def tearDown(self): + drop_tables(Migration, Band, Genre, GenreToBand) + + def test_m2m(self): + """ + Make sure M2M relations can be created. + """ + + self._test_migrations( + table_snapshots=[[Band, Genre, GenreToBand]], + ) + + for table_class in [Band, Genre, GenreToBand]: + self.assertTrue(table_class.table_exists().run_sync()) From 4d24e9d25ae64ee804d5f350f7b597151a3d97fd Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 18 Dec 2021 22:54:50 +0000 Subject: [PATCH 202/727] mentioned how M2M methods can be run synchronously too --- docs/src/piccolo/schema/m2m.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/src/piccolo/schema/m2m.rst b/docs/src/piccolo/schema/m2m.rst index c91099856..c49f1d4d0 100644 --- a/docs/src/piccolo/schema/m2m.rst +++ b/docs/src/piccolo/schema/m2m.rst @@ -119,3 +119,6 @@ remove_m2m ---------- .. automethod:: Table.remove_m2m + +.. hint:: All of these methods can be run syncronously as well - for example, + ``band.get_m2m(Band.genres).run_sync()``. From 4d65eea957810c1b6834b6b00ecf940ebfb5ad9e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 18 Dec 2021 22:58:34 +0000 Subject: [PATCH 203/727] remove unnecessary start index Co-authored-by: Yasser Tahiri --- piccolo/columns/m2m.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piccolo/columns/m2m.py b/piccolo/columns/m2m.py index 6bbe74fba..55dd91fc2 100644 --- a/piccolo/columns/m2m.py +++ b/piccolo/columns/m2m.py @@ -127,7 +127,7 @@ def resolved_joining_table(self) -> t.Type[Table]: def foreign_key_columns(self) -> t.List[ForeignKey]: if not self._foreign_key_columns: self._foreign_key_columns = ( - self.resolved_joining_table._meta.foreign_key_columns[0:2] + self.resolved_joining_table._meta.foreign_key_columns[:2] ) return self._foreign_key_columns From d30d99ea6cc2e85abd412fc9c729ef68da06c5cc Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 18 Dec 2021 22:59:01 +0000 Subject: [PATCH 204/727] reduce nested of logic Co-authored-by: Yasser Tahiri --- piccolo/columns/m2m.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/piccolo/columns/m2m.py b/piccolo/columns/m2m.py index 55dd91fc2..a768da6e5 100644 --- a/piccolo/columns/m2m.py +++ b/piccolo/columns/m2m.py @@ -323,11 +323,11 @@ def __init__( columns, you can explicitly specify which two are relevant. """ - if foreign_key_columns: - if len(foreign_key_columns) != 2 or not all( - isinstance(i, ForeignKey) for i in foreign_key_columns - ): - raise ValueError("You must specify two ForeignKey columns.") + if foreign_key_columns and ( + len(foreign_key_columns) != 2 + or not all(isinstance(i, ForeignKey) for i in foreign_key_columns) + ): + raise ValueError("You must specify two ForeignKey columns.") self._meta = M2MMeta( joining_table=joining_table, From 058891f0580fd5000ecebc3e8292c4e413572980 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 18 Dec 2021 22:59:25 +0000 Subject: [PATCH 205/727] remove unnecessary list comprehension Co-authored-by: Yasser Tahiri --- tests/columns/test_m2m.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/columns/test_m2m.py b/tests/columns/test_m2m.py index f5fe9c434..3673a77a1 100644 --- a/tests/columns/test_m2m.py +++ b/tests/columns/test_m2m.py @@ -211,7 +211,7 @@ def test_get_m2m(self): genres = band.get_m2m(Band.genres).run_sync() - self.assertTrue(all([isinstance(i, Table) for i in genres])) + self.assertTrue(all(isinstance(i, Table) for i in genres)) self.assertEqual([i.name for i in genres], ["Rock", "Folk"]) From 64f4e1979442c40c7f74a1b739ce05895221182a Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 18 Dec 2021 22:59:43 +0000 Subject: [PATCH 206/727] remove more unnecessary list comprehensions Co-authored-by: Yasser Tahiri --- tests/columns/test_m2m.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/columns/test_m2m.py b/tests/columns/test_m2m.py index 3673a77a1..1d6ac3bd2 100644 --- a/tests/columns/test_m2m.py +++ b/tests/columns/test_m2m.py @@ -377,7 +377,7 @@ def test_get_m2m(self): concerts = customer.get_m2m(Customer.concerts).run_sync() - self.assertTrue(all([isinstance(i, Table) for i in concerts])) + self.assertTrue(all(isinstance(i, Table) for i in concerts)) self.assertCountEqual( [i.name for i in concerts], ["Rockfest", "Classicfest"] From 996f6f134e2c975fad28d1e6d6ed2ea576f9ab29 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 18 Dec 2021 23:10:53 +0000 Subject: [PATCH 207/727] remove transaction for now - will replace with subquery in the future We don't support nested transactions currently, so having a transaction here is problematic if the user wraps their own code in a transaction. --- piccolo/columns/m2m.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/piccolo/columns/m2m.py b/piccolo/columns/m2m.py index a768da6e5..fde4c3fe9 100644 --- a/piccolo/columns/m2m.py +++ b/piccolo/columns/m2m.py @@ -280,27 +280,27 @@ class M2MGetRelated: async def run(self): joining_table = self.m2m._meta.resolved_joining_table - async with self.row._meta.db.transaction(): - secondary_table = ( - self.m2m._meta.secondary_foreign_key._foreign_key_meta.resolved_references # noqa: E501 - ) + secondary_table = ( + self.m2m._meta.secondary_foreign_key._foreign_key_meta.resolved_references # noqa: E501 + ) - ids = ( - await joining_table.select( - getattr( - self.m2m._meta.secondary_foreign_key, - secondary_table._meta.primary_key._meta.name, - ) + # TODO - replace this with a subquery in the future. + ids = ( + await joining_table.select( + getattr( + self.m2m._meta.secondary_foreign_key, + secondary_table._meta.primary_key._meta.name, ) - .where(self.m2m._meta.primary_foreign_key == self.row) - .output(as_list=True) ) + .where(self.m2m._meta.primary_foreign_key == self.row) + .output(as_list=True) + ) - results = await secondary_table.objects().where( - secondary_table._meta.primary_key.is_in(ids) - ) + results = await secondary_table.objects().where( + secondary_table._meta.primary_key.is_in(ids) + ) - return results + return results def run_sync(self): return run_sync(self.run()) From bc5cd0c0884d31091fed11e1f81414af10e922bb Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 19 Dec 2021 12:08:37 +0000 Subject: [PATCH 208/727] M2M can now return objects --- docs/src/piccolo/schema/m2m.rst | 37 ++++++++++++++- piccolo/columns/m2m.py | 70 ++++++++++++++++++++------- piccolo/query/methods/select.py | 83 +++++++++++++++++++++++++++++---- tests/columns/test_m2m.py | 75 ++++++++++++++++++++++++++--- 4 files changed, 231 insertions(+), 34 deletions(-) diff --git a/docs/src/piccolo/schema/m2m.rst b/docs/src/piccolo/schema/m2m.rst index c49f1d4d0..2c16533c4 100644 --- a/docs/src/piccolo/schema/m2m.rst +++ b/docs/src/piccolo/schema/m2m.rst @@ -64,7 +64,7 @@ we can do this: .. code-block:: python - >>> await Band.select(Band.name, Band.genres(Genre.name)) + >>> await Band.select(Band.name, Band.genres(Genre.name, as_list=True)) [ {"name": "Pythonistas", "genres": ["Rock", "Folk"]}, {"name": "Rustaceans", "genres": ["Folk"]}, @@ -75,13 +75,46 @@ You can request whichever column you like from the related table: .. code-block:: python - >>> await Band.select(Band.name, Band.genres(Genre.id)) + >>> await Band.select(Band.name, Band.genres(Genre.id, as_list=True)) [ {"name": "Pythonistas", "genres": [1, 2]}, {"name": "Rustaceans", "genres": [2]}, {"name": "C-Sharps", "genres": [1, 3]}, ] +You can also request multiple columns from the related table: + +.. code-block:: python + + >>> await Band.select(Band.name, Band.genres(Genre.id, Genre.name)) + [ + { + 'name': 'Pythonistas', + 'genres': [ + {'id': 1, 'name': 'Rock'}, + {'id': 2, 'name': 'Folk'} + ] + }, + ... + ] + +If you omit the columns argument, then all of the columns are returned. + +.. code-block:: python + + >>> await Band.select(Band.name, Band.genres()) + [ + { + 'name': 'Pythonistas', + 'genres': [ + {'id': 1, 'name': 'Rock'}, + {'id': 2, 'name': 'Folk'} + ] + }, + ... + ] + + As we defined ``M2M`` on the ``Genre`` table too, we can get each band in a given genre: diff --git a/piccolo/columns/m2m.py b/piccolo/columns/m2m.py index fde4c3fe9..a0fcda612 100644 --- a/piccolo/columns/m2m.py +++ b/piccolo/columns/m2m.py @@ -17,13 +17,17 @@ class M2MSelect(Selectable): This is a subquery used within a select to fetch data via an M2M table. """ - def __init__(self, column: Column, m2m: M2M): + def __init__(self, *columns: Column, m2m: M2M, as_list: bool = False): """ - :param column: - Which column to include from the related table. + :param columns: + Which columns to include from the related table. + :param as_list: + If a single column is provided, and ``as_list`` is ``True`` a + flattened list will be returned, rather than a list of objects. """ - self.column = column + self.as_list = as_list + self.columns = columns self.m2m = m2m def get_select_string(self, engine_type: str, just_alias=False) -> str: @@ -42,8 +46,6 @@ def get_select_string(self, engine_type: str, just_alias=False) -> str: table_2_name = table_2._meta.tablename table_2_pk_name = table_2._meta.primary_key._meta.db_column_name - column_name = self.column._meta.db_column_name - inner_select = f""" "{m2m_table_name}" JOIN "{table_1_name}" "inner_{table_1_name}" ON ( @@ -56,14 +58,34 @@ def get_select_string(self, engine_type: str, just_alias=False) -> str: """ # noqa: E501 if engine_type == "postgres": - return f""" - ARRAY( - SELECT - "inner_{table_2_name}"."{column_name}" - FROM {inner_select} - ) AS "{m2m_relationship_name}" - """ + if self.as_list: + column_name = self.columns[0]._meta.db_column_name + return f""" + ARRAY( + SELECT + "inner_{table_2_name}"."{column_name}" + FROM {inner_select} + ) AS "{m2m_relationship_name}" + """ + else: + column_names = ", ".join( + f'"inner_{table_2_name}"."{column._meta.db_column_name}"' + for column in self.columns + ) + return f""" + ( + SELECT JSON_AGG({m2m_relationship_name}_results) + FROM ( + SELECT {column_names} FROM {inner_select} + ) AS "{m2m_relationship_name}_results" + ) AS "{m2m_relationship_name}" + """ elif engine_type == "sqlite": + if len(self.columns) > 1: + column_name = table_2_pk_name + else: + column_name = self.columns[0]._meta.db_column_name + return f""" ( SELECT group_concat( @@ -334,10 +356,24 @@ def __init__( _foreign_key_columns=foreign_key_columns, ) - def __call__(self, column: Column) -> M2MSelect: + def __call__(self, *columns: Column, as_list: bool = False) -> M2MSelect: """ - :param column: - Which column to include from the related table. + :param columns: + Which columns to include from the related table. If none are + specified, then all of the columns are returned. + :param as_list: + If a single column is provided, and ``as_list`` is ``True`` a + flattened list will be returned, rather than a list of objects. """ - return M2MSelect(column, m2m=self) + if len(columns) == 0: + columns = ( + self._meta.secondary_foreign_key._foreign_key_meta.resolved_references._meta.columns # noqa: E501 + ) + + if as_list and len(columns) != 1: + raise ValueError( + "`as_list` is only valid with a single column argument" + ) + + return M2MSelect(*columns, m2m=self, as_list=as_list) diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 0728558b5..921ed85f5 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -1,6 +1,7 @@ from __future__ import annotations import decimal +import itertools import typing as t from collections import OrderedDict @@ -21,6 +22,7 @@ ) from piccolo.querystring import QueryString from piccolo.utils.dictionary import make_nested +from piccolo.utils.encoding import load_json from piccolo.utils.warnings import colored_warning if t.TYPE_CHECKING: # pragma: no cover @@ -274,26 +276,89 @@ def offset(self, number: int) -> Select: return self async def response_handler(self, response): - # With M2M queries in SQLite, we always get the value back as a list - # of strings, so we need to do some type conversion. + m2m_selects = [ + i + for i in self.columns_delegate.selected_columns + if isinstance(i, M2MSelect) + ] if self.engine_type == "sqlite": - m2m_selects = [ - i - for i in self.columns_delegate.selected_columns - if isinstance(i, M2MSelect) - ] + # With M2M queries in SQLite, we always get the value back as a + # list of strings, so we need to do some type conversion. for m2m_select in m2m_selects: m2m_name = m2m_select.m2m._meta.name - value_type = m2m_select.column.__class__.value_type + + secondary_table = ( + m2m_select.m2m._meta.secondary_foreign_key._foreign_key_meta.resolved_references # noqa: E501 + ) + secondary_table_pk = secondary_table._meta.primary_key + + # If the user requested a single column, we just return that + # from the database. Otherwise we request the primary key + # value, so we can fetch the rest of the data in a subsequent + # SQL query - see below. + value_type = ( + m2m_select.columns[0].__class__.value_type + if m2m_select.as_list + else secondary_table_pk.value_type + ) try: for row in response: row[m2m_name] = [value_type(i) for i in row[m2m_name]] except ValueError: colored_warning( - "Unable to do type converstion for the " + "Unable to do type conversion for the " f"{m2m_name} relation" ) + if not m2m_select.as_list: + if len(m2m_select.columns) == 1: + column_name = m2m_select.columns[0]._meta.name + for row in response: + row[m2m_name] = [ + {column_name: i} for i in row[m2m_name] + ] + else: + # I haven't worked out how to replicate Postgres' + # `JSON_AGG` in SQLite, so the workaround is to do + # another SQL query. + row_ids = { + i + for i in itertools.chain( + *[row[m2m_name] for row in response] + ) + } + extra_rows = ( + await secondary_table.select( + *m2m_select.columns, + secondary_table_pk.as_alias("mapping_key"), + ) + .where(secondary_table_pk.is_in(row_ids)) + .run() + ) + extra_rows_map = { + row["mapping_key"]: { + key: value + for key, value in row.items() + if key != "mapping_key" + } + for row in extra_rows + } + for row in response: + row[m2m_name] = [ + extra_rows_map.get(i) for i in row[m2m_name] + ] + + elif self.engine_type == "postgres": + # If we requested the results as objects, then it comes back as a + # JSON string, so we need to deserialise it. + for m2m_select in m2m_selects: + if not m2m_select.as_list: + m2m_name = m2m_select.m2m._meta.name + for row in response: + row[m2m_name] = load_json(row[m2m_name]) + + ####################################################################### + # If no columns were specified, it's a select *, so we know that # no columns were selected from related tables. was_select_star = len(self.columns_delegate.selected_columns) == 0 diff --git a/tests/columns/test_m2m.py b/tests/columns/test_m2m.py index 1d6ac3bd2..c3b043918 100644 --- a/tests/columns/test_m2m.py +++ b/tests/columns/test_m2m.py @@ -58,7 +58,9 @@ def tearDown(self): drop_tables(*TABLES_1) def test_select_name(self): - response = Band.select(Band.name, Band.genres(Genre.name)).run_sync() + response = Band.select( + Band.name, Band.genres(Genre.name, as_list=True) + ).run_sync() self.assertEqual( response, [ @@ -69,7 +71,9 @@ def test_select_name(self): ) # Now try it in reverse. - response = Genre.select(Genre.name, Genre.bands(Band.name)).run_sync() + response = Genre.select( + Genre.name, Genre.bands(Band.name, as_list=True) + ).run_sync() self.assertEqual( response, [ @@ -79,8 +83,65 @@ def test_select_name(self): ], ) + def test_select_multiple(self): + response = Band.select( + Band.name, Band.genres(Genre.id, Genre.name) + ).run_sync() + + self.assertEqual( + response, + [ + { + "name": "Pythonistas", + "genres": [ + {"id": 1, "name": "Rock"}, + {"id": 2, "name": "Folk"}, + ], + }, + {"name": "Rustaceans", "genres": [{"id": 2, "name": "Folk"}]}, + { + "name": "C-Sharps", + "genres": [ + {"id": 1, "name": "Rock"}, + {"id": 3, "name": "Classical"}, + ], + }, + ], + ) + + # Now try it in reverse. + response = Genre.select( + Genre.name, Genre.bands(Band.id, Band.name) + ).run_sync() + + self.assertEqual( + response, + [ + { + "name": "Rock", + "bands": [ + {"id": 1, "name": "Pythonistas"}, + {"id": 3, "name": "C-Sharps"}, + ], + }, + { + "name": "Folk", + "bands": [ + {"id": 1, "name": "Pythonistas"}, + {"id": 2, "name": "Rustaceans"}, + ], + }, + { + "name": "Classical", + "bands": [{"id": 3, "name": "C-Sharps"}], + }, + ], + ) + def test_select_id(self): - response = Band.select(Band.name, Band.genres(Genre.id)).run_sync() + response = Band.select( + Band.name, Band.genres(Genre.id, as_list=True) + ).run_sync() self.assertEqual( response, [ @@ -91,7 +152,9 @@ def test_select_id(self): ) # Now try it in reverse. - response = Genre.select(Genre.name, Genre.bands(Band.id)).run_sync() + response = Genre.select( + Genre.name, Genre.bands(Band.id, as_list=True) + ).run_sync() self.assertEqual( response, [ @@ -316,7 +379,7 @@ def tearDown(self): def test_select(self): response = Customer.select( - Customer.name, Customer.concerts(Concert.name) + Customer.name, Customer.concerts(Concert.name, as_list=True) ).run_sync() self.assertListEqual( @@ -330,7 +393,7 @@ def test_select(self): # Now try it in reverse. response = Concert.select( - Concert.name, Concert.customers(Customer.name) + Concert.name, Concert.customers(Customer.name, as_list=True) ).run_sync() self.assertListEqual( From 82ea6d7aff1ec92db1cacd20e935e139721ddde4 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 19 Dec 2021 20:21:00 +0000 Subject: [PATCH 209/727] loads more tests for M2M edge cases --- piccolo/columns/m2m.py | 71 ++++++++-- piccolo/query/methods/select.py | 168 +++++++++++++++-------- pyproject.toml | 3 +- tests/columns/test_m2m.py | 235 +++++++++++++++++++++++++++++++- 4 files changed, 402 insertions(+), 75 deletions(-) diff --git a/piccolo/columns/m2m.py b/piccolo/columns/m2m.py index a0fcda612..7d7fe25b4 100644 --- a/piccolo/columns/m2m.py +++ b/piccolo/columns/m2m.py @@ -5,7 +5,13 @@ from dataclasses import dataclass from piccolo.columns.base import Selectable -from piccolo.columns.column_types import Column, ForeignKey, LazyTableReference +from piccolo.columns.column_types import ( + JSON, + JSONB, + Column, + ForeignKey, + LazyTableReference, +) from piccolo.utils.sync import run_sync if t.TYPE_CHECKING: @@ -17,18 +23,37 @@ class M2MSelect(Selectable): This is a subquery used within a select to fetch data via an M2M table. """ - def __init__(self, *columns: Column, m2m: M2M, as_list: bool = False): + def __init__( + self, + *columns: Column, + m2m: M2M, + as_list: bool = False, + load_json: bool = False, + ): """ :param columns: Which columns to include from the related table. :param as_list: If a single column is provided, and ``as_list`` is ``True`` a flattened list will be returned, rather than a list of objects. + :param load_json: + If ``True``, any JSON strings are loaded as Python objects. """ self.as_list = as_list self.columns = columns self.m2m = m2m + self.load_json = load_json + + safe_types = [int, str] + + # If the columns can be serialised / deserialise as JSON, then we + # can fetch the data all in one go. + self.serialisation_safe = all( + (column.__class__.value_type in safe_types) + and (type(column) not in (JSON, JSONB)) + for column in columns + ) def get_select_string(self, engine_type: str, just_alias=False) -> str: m2m_table_name = self.m2m._meta.resolved_joining_table._meta.tablename @@ -67,6 +92,15 @@ def get_select_string(self, engine_type: str, just_alias=False) -> str: FROM {inner_select} ) AS "{m2m_relationship_name}" """ + elif not self.serialisation_safe: + column_name = table_2_pk_name + return f""" + ARRAY( + SELECT + "inner_{table_2_name}"."{column_name}" + FROM {inner_select} + ) AS "{m2m_relationship_name}" + """ else: column_names = ", ".join( f'"inner_{table_2_name}"."{column._meta.db_column_name}"' @@ -81,7 +115,7 @@ def get_select_string(self, engine_type: str, just_alias=False) -> str: ) AS "{m2m_relationship_name}" """ elif engine_type == "sqlite": - if len(self.columns) > 1: + if len(self.columns) > 1 or not self.serialisation_safe: column_name = table_2_pk_name else: column_name = self.columns[0]._meta.db_column_name @@ -154,7 +188,7 @@ def foreign_key_columns(self) -> t.List[ForeignKey]: return self._foreign_key_columns @property - def primary_foreign_key(self): + def primary_foreign_key(self) -> ForeignKey: """ The joining table has two foreign keys. We need a way to distinguish between them. The primary is the one which points to the table with @@ -186,7 +220,11 @@ class GenreToBand(Table): raise ValueError("No matching foreign key column found!") @property - def secondary_foreign_key(self): + def primary_table(self) -> t.Type[Table]: + return self.primary_foreign_key._foreign_key_meta.resolved_references + + @property + def secondary_foreign_key(self) -> ForeignKey: """ See ``primary_foreign_key``. """ @@ -196,6 +234,10 @@ def secondary_foreign_key(self): raise ValueError("No matching foreign key column found!") + @property + def secondary_table(self) -> t.Type[Table]: + return self.secondary_foreign_key._foreign_key_meta.resolved_references + @dataclass class M2MAddRelated: @@ -302,9 +344,7 @@ class M2MGetRelated: async def run(self): joining_table = self.m2m._meta.resolved_joining_table - secondary_table = ( - self.m2m._meta.secondary_foreign_key._foreign_key_meta.resolved_references # noqa: E501 - ) + secondary_table = self.m2m._meta.secondary_table # TODO - replace this with a subquery in the future. ids = ( @@ -356,7 +396,9 @@ def __init__( _foreign_key_columns=foreign_key_columns, ) - def __call__(self, *columns: Column, as_list: bool = False) -> M2MSelect: + def __call__( + self, *columns: Column, as_list: bool = False, load_json: bool = False + ) -> M2MSelect: """ :param columns: Which columns to include from the related table. If none are @@ -364,16 +406,17 @@ def __call__(self, *columns: Column, as_list: bool = False) -> M2MSelect: :param as_list: If a single column is provided, and ``as_list`` is ``True`` a flattened list will be returned, rather than a list of objects. - + :param load_json: + If ``True``, any JSON strings are loaded as Python objects. """ if len(columns) == 0: - columns = ( - self._meta.secondary_foreign_key._foreign_key_meta.resolved_references._meta.columns # noqa: E501 - ) + columns = tuple(self._meta.secondary_table._meta.columns) if as_list and len(columns) != 1: raise ValueError( "`as_list` is only valid with a single column argument" ) - return M2MSelect(*columns, m2m=self, as_list=as_list) + return M2MSelect( + *columns, m2m=self, as_list=as_list, load_json=load_json + ) diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 921ed85f5..27e1e71cb 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -6,6 +6,7 @@ from collections import OrderedDict from piccolo.columns import Column, Selectable +from piccolo.columns.column_types import JSON, JSONB, PrimaryKey from piccolo.columns.m2m import M2MSelect from piccolo.columns.readable import Readable from piccolo.engine.base import Batch @@ -275,87 +276,146 @@ def offset(self, number: int) -> Select: self.offset_delegate.offset(number) return self + async def _splice_m2m_rows( + self, + response: t.List[t.Dict[str, t.Any]], + secondary_table: t.Type[Table], + secondary_table_pk: PrimaryKey, + m2m_name: str, + m2m_select: M2MSelect, + as_list: bool = False, + ): + row_ids = list( + {i for i in itertools.chain(*[row[m2m_name] for row in response])} + ) + extra_rows = ( + ( + await secondary_table.select( + *m2m_select.columns, + secondary_table_pk.as_alias("mapping_key"), + ) + .where(secondary_table_pk.is_in(row_ids)) + .output(load_json=m2m_select.load_json) + .run() + ) + if row_ids + else [] + ) + if as_list: + column_name = m2m_select.columns[0]._meta.name + extra_rows_map = { + row["mapping_key"]: row[column_name] for row in extra_rows + } + else: + extra_rows_map = { + row["mapping_key"]: { + key: value + for key, value in row.items() + if key != "mapping_key" + } + for row in extra_rows + } + for row in response: + row[m2m_name] = [extra_rows_map.get(i) for i in row[m2m_name]] + return response + async def response_handler(self, response): m2m_selects = [ i for i in self.columns_delegate.selected_columns if isinstance(i, M2MSelect) ] - if self.engine_type == "sqlite": - # With M2M queries in SQLite, we always get the value back as a - # list of strings, so we need to do some type conversion. - for m2m_select in m2m_selects: - m2m_name = m2m_select.m2m._meta.name - - secondary_table = ( - m2m_select.m2m._meta.secondary_foreign_key._foreign_key_meta.resolved_references # noqa: E501 - ) - secondary_table_pk = secondary_table._meta.primary_key - - # If the user requested a single column, we just return that - # from the database. Otherwise we request the primary key - # value, so we can fetch the rest of the data in a subsequent - # SQL query - see below. + for m2m_select in m2m_selects: + m2m_name = m2m_select.m2m._meta.name + secondary_table = m2m_select.m2m._meta.secondary_table + secondary_table_pk = secondary_table._meta.primary_key + + if self.engine_type == "sqlite": + # With M2M queries in SQLite, we always get the value back as a + # list of strings, so we need to do some type conversion. value_type = ( m2m_select.columns[0].__class__.value_type - if m2m_select.as_list + if m2m_select.as_list and m2m_select.serialisation_safe else secondary_table_pk.value_type ) try: for row in response: - row[m2m_name] = [value_type(i) for i in row[m2m_name]] + data = row[m2m_name] + row[m2m_name] = ( + [value_type(i) for i in row[m2m_name]] + if data + else [] + ) except ValueError: colored_warning( "Unable to do type conversion for the " f"{m2m_name} relation" ) - if not m2m_select.as_list: - if len(m2m_select.columns) == 1: + # If the user requested a single column, we just return that + # from the database. Otherwise we request the primary key + # value, so we can fetch the rest of the data in a subsequent + # SQL query - see below. + if m2m_select.as_list: + if m2m_select.serialisation_safe: + pass + else: + response = await self._splice_m2m_rows( + response, + secondary_table, + secondary_table_pk, + m2m_name, + m2m_select, + as_list=True, + ) + else: + if ( + len(m2m_select.columns) == 1 + and m2m_select.serialisation_safe + ): column_name = m2m_select.columns[0]._meta.name for row in response: row[m2m_name] = [ {column_name: i} for i in row[m2m_name] ] else: - # I haven't worked out how to replicate Postgres' - # `JSON_AGG` in SQLite, so the workaround is to do - # another SQL query. - row_ids = { - i - for i in itertools.chain( - *[row[m2m_name] for row in response] - ) - } - extra_rows = ( - await secondary_table.select( - *m2m_select.columns, - secondary_table_pk.as_alias("mapping_key"), - ) - .where(secondary_table_pk.is_in(row_ids)) - .run() + response = await self._splice_m2m_rows( + response, + secondary_table, + secondary_table_pk, + m2m_name, + m2m_select, ) - extra_rows_map = { - row["mapping_key"]: { - key: value - for key, value in row.items() - if key != "mapping_key" - } - for row in extra_rows - } - for row in response: - row[m2m_name] = [ - extra_rows_map.get(i) for i in row[m2m_name] - ] - elif self.engine_type == "postgres": - # If we requested the results as objects, then it comes back as a - # JSON string, so we need to deserialise it. - for m2m_select in m2m_selects: - if not m2m_select.as_list: - m2m_name = m2m_select.m2m._meta.name + elif self.engine_type == "postgres": + if m2m_select.as_list: + # We get the data back as an array, and can just return it + # unless it's JSON. + if ( + type(m2m_select.columns[0]) in (JSON, JSONB) + and m2m_select.load_json + ): + for row in response: + data = row[m2m_name] + row[m2m_name] = [load_json(i) for i in data] + elif m2m_select.serialisation_safe: + # If the columns requested can be safely serialised, they + # are returned as a JSON string, so we need to deserialise + # it. for row in response: - row[m2m_name] = load_json(row[m2m_name]) + data = row[m2m_name] + row[m2m_name] = load_json(data) if data else [] + else: + # If the data can't be safely serialisd as JSON, we get + # back an array of primary key values, and need to + # splice in the correct values using Python. + response = await self._splice_m2m_rows( + response, + secondary_table, + secondary_table_pk, + m2m_name, + m2m_select, + ) ####################################################################### diff --git a/pyproject.toml b/pyproject.toml index 99116394f..37430c5a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,8 @@ line_length = 79 module = [ "orjson", "jinja2", - "dateutil" + "dateutil", + "asyncpg.pgproto.pgproto" ] ignore_missing_imports = true diff --git a/tests/columns/test_m2m.py b/tests/columns/test_m2m.py index c3b043918..2a93c197e 100644 --- a/tests/columns/test_m2m.py +++ b/tests/columns/test_m2m.py @@ -1,10 +1,30 @@ +import datetime +import decimal +import uuid from unittest import TestCase +from asyncpg.pgproto.pgproto import UUID as asyncpgUUID + from piccolo.columns.column_types import ( + JSON, + JSONB, UUID, + Array, + BigInt, + Boolean, + Bytea, + Date, + DoublePrecision, ForeignKey, + Integer, + Interval, LazyTableReference, + Numeric, + Real, + SmallInt, Text, + Timestamp, + Timestamptz, Varchar, ) from piccolo.columns.m2m import M2M @@ -27,12 +47,12 @@ class GenreToBand(Table): reason = Text(help_text="For testing additional columns on join tables.") -TABLES_1 = [Band, Genre, GenreToBand] +SIMPLE_SCHEMA = [Band, Genre, GenreToBand] class TestM2M(TestCase): def setUp(self): - create_tables(*TABLES_1, if_not_exists=True) + create_tables(*SIMPLE_SCHEMA, if_not_exists=True) Band.insert( Band(name="Pythonistas"), @@ -55,7 +75,7 @@ def setUp(self): ).run_sync() def tearDown(self): - drop_tables(*TABLES_1) + drop_tables(*SIMPLE_SCHEMA) def test_select_name(self): response = Band.select( @@ -83,6 +103,39 @@ def test_select_name(self): ], ) + def test_no_related(self): + """ + Make sure it still works correctly if there are no related values. + """ + GenreToBand.delete(force=True).run_sync() + + # Try it with a list response + response = Band.select( + Band.name, Band.genres(Genre.name, as_list=True) + ).run_sync() + + self.assertEqual( + response, + [ + {"name": "Pythonistas", "genres": []}, + {"name": "Rustaceans", "genres": []}, + {"name": "C-Sharps", "genres": []}, + ], + ) + + # Also try it with a nested response + response = Band.select( + Band.name, Band.genres(Genre.id, Genre.name) + ).run_sync() + self.assertEqual( + response, + [ + {"name": "Pythonistas", "genres": []}, + {"name": "Rustaceans", "genres": []}, + {"name": "C-Sharps", "genres": []}, + ], + ) + def test_select_multiple(self): response = Band.select( Band.name, Band.genres(Genre.id, Genre.name) @@ -346,7 +399,7 @@ class CustomerToConcert(Table): concert = ForeignKey(Concert) -TABLES_2 = [Customer, Concert, CustomerToConcert] +CUSTOM_PK_SCHEMA = [Customer, Concert, CustomerToConcert] class TestM2MCustomPrimaryKey(TestCase): @@ -356,7 +409,7 @@ class TestM2MCustomPrimaryKey(TestCase): """ def setUp(self): - create_tables(*TABLES_2, if_not_exists=True) + create_tables(*CUSTOM_PK_SCHEMA, if_not_exists=True) bob = Customer.objects().create(name="Bob").run_sync() sally = Customer.objects().create(name="Sally").run_sync() @@ -375,7 +428,7 @@ def setUp(self): ).run_sync() def tearDown(self): - drop_tables(*TABLES_2) + drop_tables(*CUSTOM_PK_SCHEMA) def test_select(self): response = Customer.select( @@ -445,3 +498,173 @@ def test_get_m2m(self): self.assertCountEqual( [i.name for i in concerts], ["Rockfest", "Classicfest"] ) + + +############################################################################### + +# Test a very complex schema + + +class SmallTable(Table): + varchar_col = Varchar() + mega_rows = M2M(LazyTableReference("SmallToMega", module_path=__name__)) + + +class MegaTable(Table): + """ + A table containing all of the column types, and different column kwargs. + """ + + array_col = Array(Varchar()) + bigint_col = BigInt() + boolean_col = Boolean() + bytea_col = Bytea() + date_col = Date() + double_precision_col = DoublePrecision() + integer_col = Integer() + interval_col = Interval() + json_col = JSON() + jsonb_col = JSONB() + numeric_col = Numeric(digits=(5, 2)) + real_col = Real() + smallint_col = SmallInt() + text_col = Text() + timestamp_col = Timestamp() + timestamptz_col = Timestamptz() + uuid_col = UUID() + varchar_col = Varchar() + + +class SmallToMega(Table): + small = ForeignKey(MegaTable) + mega = ForeignKey(SmallTable) + + +COMPLEX_SCHEMA = [MegaTable, SmallTable, SmallToMega] + + +class TestM2MComplexSchema(TestCase): + """ + By using a very complex schema containing every column type, we can catch + more edge cases. + """ + + def setUp(self): + create_tables(*COMPLEX_SCHEMA, if_not_exists=True) + + small_table = SmallTable(varchar_col="Test") + small_table.save().run_sync() + + mega_table = MegaTable( + array_col=["bob", "sally"], + bigint_col=1, + boolean_col=True, + bytea_col="hello".encode("utf8"), + date_col=datetime.date(year=2021, month=1, day=1), + double_precision_col=1.344, + integer_col=1, + interval_col=datetime.timedelta(seconds=10), + json_col={"a": 1}, + jsonb_col={"a": 1}, + numeric_col=decimal.Decimal("1.1"), + real_col=1.1, + smallint_col=1, + text_col="hello", + timestamp_col=datetime.datetime(year=2021, month=1, day=1), + timestamptz_col=datetime.datetime( + year=2021, month=1, day=1, tzinfo=datetime.timezone.utc + ), + uuid_col=uuid.UUID("12783854-c012-4c15-8183-8eecb46f2c4e"), + varchar_col="hello", + ) + mega_table.save().run_sync() + + SmallToMega(small=small_table, mega=mega_table).save().run_sync() + + self.mega_table = mega_table + + def tearDown(self): + drop_tables(*COMPLEX_SCHEMA) + + def test_select_all(self): + """ + Fetch all of the columns from the related table to make sure they're + returned correctly. + """ + response = SmallTable.select( + SmallTable.varchar_col, SmallTable.mega_rows(load_json=True) + ).run_sync() + + self.assertTrue(len(response) == 1) + mega_rows = response[0]["mega_rows"] + + self.assertTrue(len(mega_rows) == 1) + mega_row = mega_rows[0] + + for key, value in mega_row.items(): + # Make sure that every value in the response matches what we saved. + self.assertAlmostEqual( + getattr(self.mega_table, key), + value, + msg=f"{key} doesn't match", + ) + + def test_select_single(self): + """ + Make sure each column can be selected one at a time. + """ + for column in MegaTable._meta.columns: + response = SmallTable.select( + SmallTable.varchar_col, + SmallTable.mega_rows(column, load_json=True), + ).run_sync() + + data = response[0]["mega_rows"][0] + column_name = column._meta.name + + original_value = getattr(self.mega_table, column_name) + returned_value = data[column_name] + + if type(column) == UUID: + self.assertTrue( + type(returned_value) in (uuid.UUID, asyncpgUUID) + ) + else: + self.assertEqual( + type(original_value), + type(returned_value), + msg=f"{column_name} type isn't correct", + ) + + self.assertAlmostEqual( + original_value, + returned_value, + msg=f"{column_name} doesn't match", + ) + + # Test it as a list too + response = SmallTable.select( + SmallTable.varchar_col, + SmallTable.mega_rows(column, as_list=True, load_json=True), + ).run_sync() + + original_value = getattr(self.mega_table, column_name) + returned_value = response[0]["mega_rows"][0] + + if type(column) == UUID: + self.assertTrue( + type(returned_value) in (uuid.UUID, asyncpgUUID) + ) + self.assertEqual(str(original_value), str(returned_value)) + else: + self.assertEqual( + type(original_value), + type(returned_value), + msg=f"{column_name} type isn't correct", + ) + + self.assertAlmostEqual( + original_value, + returned_value, + msg=f"{column_name} doesn't match", + ) From a57efbe37899d7e4ef76aca923ff33c70ea99e50 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 19 Dec 2021 20:25:07 +0000 Subject: [PATCH 210/727] fix minor typo in comment --- piccolo/query/methods/select.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 27e1e71cb..817ae45ee 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -406,7 +406,7 @@ async def response_handler(self, response): data = row[m2m_name] row[m2m_name] = load_json(data) if data else [] else: - # If the data can't be safely serialisd as JSON, we get + # If the data can't be safely serialised as JSON, we get # back an array of primary key values, and need to # splice in the correct values using Python. response = await self._splice_m2m_rows( From 8c7662f44016f297af9edb28cbda3ed798e02df9 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 19 Dec 2021 20:28:16 +0000 Subject: [PATCH 211/727] fix sqlite tests when asyncpg isn't installed --- tests/columns/test_m2m.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/columns/test_m2m.py b/tests/columns/test_m2m.py index 2a93c197e..f4a2de4c0 100644 --- a/tests/columns/test_m2m.py +++ b/tests/columns/test_m2m.py @@ -3,7 +3,12 @@ import uuid from unittest import TestCase -from asyncpg.pgproto.pgproto import UUID as asyncpgUUID +try: + from asyncpg.pgproto.pgproto import UUID as asyncpgUUID +except ImportError: + # In case someone is running the tests for SQLite and doesn't have asyncpg + # installed. + from uuid import UUID as asyncpgUUID from piccolo.columns.column_types import ( JSON, From 82c2e0384661b40e172571ed85a010a68c6073aa Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 19 Dec 2021 20:47:36 +0000 Subject: [PATCH 212/727] bumped version --- CHANGES.rst | 46 +++++++++++++++++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 74d2c59e1..49633835f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,52 @@ Changes ======= +0.62.0 +------ + +Added Many-To-Many support. + +.. code-block:: python + + from piccolo.columns.column_types import ( + ForeignKey, + LazyTableReference, + Varchar + ) + from piccolo.columns.m2m import M2M + + + class Band(Table): + name = Varchar() + genres = M2M(LazyTableReference("GenreToBand", module_path=__name__)) + + + class Genre(Table): + name = Varchar() + bands = M2M(LazyTableReference("GenreToBand", module_path=__name__)) + + + # This is our joining table: + class GenreToBand(Table): + band = ForeignKey(Band) + genre = ForeignKey(Genre) + + + >>> await Band.select(Band.name, Band.genres(Band.name, as_list=True)) + [ + { + "name": "Pythonistas", + "genres": ["Rock", "Folk"] + }, + ... + ] + +See the docs for more details. + +Many thanks to @sinisaos and @yezz123 for all the input. + +------------------------------------------------------------------------------- + 0.61.2 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index bbbe9d079..9faf4dd74 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.61.2" +__VERSION__ = "0.62.0" From f333371e90b72ae9619f07ebe316ce9cc8d6d3c0 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 19 Dec 2021 20:56:59 +0000 Subject: [PATCH 213/727] fix typo --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 49633835f..70ac89207 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -32,7 +32,7 @@ Added Many-To-Many support. genre = ForeignKey(Genre) - >>> await Band.select(Band.name, Band.genres(Band.name, as_list=True)) + >>> await Band.select(Band.name, Band.genres(Genre.name, as_list=True)) [ { "name": "Pythonistas", From 894be6f246d759c587c0fc437adab93aabb85b52 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 19 Dec 2021 21:10:10 +0000 Subject: [PATCH 214/727] fix typo in docs --- docs/src/piccolo/schema/m2m.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/piccolo/schema/m2m.rst b/docs/src/piccolo/schema/m2m.rst index 2c16533c4..8d41ed126 100644 --- a/docs/src/piccolo/schema/m2m.rst +++ b/docs/src/piccolo/schema/m2m.rst @@ -120,7 +120,7 @@ given genre: .. code-block:: python - >>> await Genre.select(Genre.name, Genre.bands(Band.name)) + >>> await Genre.select(Genre.name, Genre.bands(Band.name, as_list=True)) [ {"name": "Rock", "bands": ["Pythonistas", "C-Sharps"]}, {"name": "Folk", "bands": ["Pythonistas", "Rustaceans"]}, From f7c8f29f427bcf3196a8436f87fcfe6bab03f0f6 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 20 Dec 2021 07:27:44 +0000 Subject: [PATCH 215/727] fix misspelling in docs --- docs/src/piccolo/schema/m2m.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/piccolo/schema/m2m.rst b/docs/src/piccolo/schema/m2m.rst index 8d41ed126..183b8ddde 100644 --- a/docs/src/piccolo/schema/m2m.rst +++ b/docs/src/piccolo/schema/m2m.rst @@ -153,5 +153,5 @@ remove_m2m .. automethod:: Table.remove_m2m -.. hint:: All of these methods can be run syncronously as well - for example, +.. hint:: All of these methods can be run synchronously as well - for example, ``band.get_m2m(Band.genres).run_sync()``. From 2c162d09b7b033c672ec01b8336ce4c370965733 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 20 Dec 2021 11:36:46 +0000 Subject: [PATCH 216/727] add missing import to docs --- docs/src/piccolo/schema/m2m.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/src/piccolo/schema/m2m.rst b/docs/src/piccolo/schema/m2m.rst index 183b8ddde..c93bb097b 100644 --- a/docs/src/piccolo/schema/m2m.rst +++ b/docs/src/piccolo/schema/m2m.rst @@ -30,6 +30,7 @@ We create it in Piccolo like this: Varchar ) from piccolo.columns.m2m import M2M + from piccolo.table import Table class Band(Table): From b7890ebca69712286796e8e2bb6afdcef8c957d9 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 22 Dec 2021 00:08:16 +0000 Subject: [PATCH 217/727] `_get_related_readable` fix (#379) --- piccolo/table.py | 11 ++- tests/example_apps/music/tables.py | 20 +++++ .../instance/test_get_related_readable.py | 76 +++++++++++++++++-- 3 files changed, 97 insertions(+), 10 deletions(-) diff --git a/piccolo/table.py b/piccolo/table.py index 0da3a92a7..44746dfa8 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -575,13 +575,20 @@ def _get_related_readable(cls, column: ForeignKey) -> Readable: column._foreign_key_meta.resolved_references.get_readable() ) - columns = [getattr(column, i._meta.name) for i in readable.columns] + output_columns = [] + + for readable_column in readable.columns: + output_column = column + for fk in readable_column._meta.call_chain: + output_column = getattr(column, fk._meta.name) + output_column = getattr(output_column, readable_column._meta.name) + output_columns.append(output_column) output_name = f"{column._meta.name}_readable" return Readable( template=readable.template, - columns=columns, + columns=output_columns, output_name=output_name, ) diff --git a/tests/example_apps/music/tables.py b/tests/example_apps/music/tables.py index 774718cae..05ad20252 100644 --- a/tests/example_apps/music/tables.py +++ b/tests/example_apps/music/tables.py @@ -29,6 +29,10 @@ class Band(Table): manager = ForeignKey(Manager, null=True) popularity = Integer(default=0) + @classmethod + def get_readable(cls) -> Readable: + return Readable(template="%s", columns=[cls.name]) + ############################################################################### # More complex @@ -38,12 +42,28 @@ class Venue(Table): name = Varchar(length=100) capacity = Integer(default=0, secret=True) + @classmethod + def get_readable(cls) -> Readable: + return Readable(template="%s", columns=[cls.name]) + class Concert(Table): band_1 = ForeignKey(Band) band_2 = ForeignKey(Band) venue = ForeignKey(Venue) + @classmethod + def get_readable(cls) -> Readable: + return Readable( + template="%s and %s at %s, capacity %s", + columns=[ + cls.band_1.name, + cls.band_2.name, + cls.venue.name, + cls.venue.capacity, + ], + ) + class Ticket(Table): concert = ForeignKey(Concert) diff --git a/tests/table/instance/test_get_related_readable.py b/tests/table/instance/test_get_related_readable.py index 4c0467be7..3a07efed6 100644 --- a/tests/table/instance/test_get_related_readable.py +++ b/tests/table/instance/test_get_related_readable.py @@ -1,22 +1,82 @@ -from tests.base import DBTestCase -from tests.example_apps.music.tables import Band +import decimal +from unittest import TestCase +from piccolo.table import create_tables, drop_tables +from tests.example_apps.music.tables import ( + Band, + Concert, + Manager, + Ticket, + Venue, +) + +TABLES = [Band, Concert, Manager, Venue, Ticket] + + +class TestGetRelatedReadable(TestCase): + def setUp(self): + create_tables(*TABLES) + + manager_1 = Manager.objects().create(name="Guido").run_sync() + manager_2 = Manager.objects().create(name="Graydon").run_sync() + + band_1 = ( + Band.objects() + .create(name="Pythonistas", manager=manager_1) + .run_sync() + ) + band_2 = ( + Band.objects() + .create(name="Rustaceans", manager=manager_2) + .run_sync() + ) + venue = ( + Venue.objects() + .create(name="Royal Albert Hall", capacity=5900) + .run_sync() + ) + concert = ( + Concert.objects() + .create(venue=venue, band_1=band_1, band_2=band_2) + .run_sync() + ) + Ticket.objects().create( + price=decimal.Decimal(50.0), concert=concert + ).run_sync() + + def tearDown(self): + drop_tables(*TABLES) -class TestGetRelatedReadable(DBTestCase): def test_get_related_readable(self): """ Make sure you can get the `Readable` representation for related object from another object instance. """ - self.insert_row() - response = Band.select( Band.name, Band._get_related_readable(Band.manager) ).run_sync() self.assertEqual( - response, [{"name": "Pythonistas", "manager_readable": "Guido"}] + response, + [ + {"name": "Pythonistas", "manager_readable": "Guido"}, + {"manager_readable": "Graydon", "name": "Rustaceans"}, + ], ) - # TODO Need to make sure it can go two levels deep ... - # e.g. Concert._get_related_readable(Concert.band_1.manager) + # Now try something much more complex. + response = Ticket.select( + Ticket.id, Ticket._get_related_readable(Ticket.concert) + ).run_sync() + self.assertEqual( + response, + [ + { + "id": 1, + "concert_readable": ( + "Pythonistas and Rustaceans at Royal Albert Hall, " + "capacity 5900" + ), + } + ], + ) From f1b4fe120fcbef32e847e84663f634a7a5b43772 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 22 Dec 2021 00:13:04 +0000 Subject: [PATCH 218/727] bumped version --- CHANGES.rst | 10 ++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 70ac89207..9f9701923 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,16 @@ Changes ======= +0.62.1 +------ + +Fixed a bug with ``Readable`` when it contains lots of joins. + +``Readable`` is used to create a user friendly representation of a row in +Piccolo Admin. + +------------------------------------------------------------------------------- + 0.62.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 9faf4dd74..d8069b8e5 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.62.0" +__VERSION__ = "0.62.1" From 0caca87dd8932bfc5befde7d801d22243ef7757e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 31 Dec 2021 06:31:41 +0000 Subject: [PATCH 219/727] max password length (#382) * max password length * fix tests on Python 3.7 * update docs --- docs/src/piccolo/authentication/baseuser.rst | 8 +++ piccolo/apps/user/tables.py | 32 +++++++-- tests/apps/user/test_tables.py | 72 ++++++++++++++++---- 3 files changed, 92 insertions(+), 20 deletions(-) diff --git a/docs/src/piccolo/authentication/baseuser.rst b/docs/src/piccolo/authentication/baseuser.rst index d6f07e145..5146ea766 100644 --- a/docs/src/piccolo/authentication/baseuser.rst +++ b/docs/src/piccolo/authentication/baseuser.rst @@ -111,3 +111,11 @@ To change a user's password: .. warning:: Don't use bulk updates for passwords - use ``update_password`` / ``update_password_sync``, and they'll correctly hash the password. + +------------------------------------------------------------------------------- + +Limits +------ + +The maximum password length allowed is 128 characters. This should be +sufficiently long for most use cases. diff --git a/piccolo/apps/user/tables.py b/piccolo/apps/user/tables.py index 879e14f8c..c582f16a4 100644 --- a/piccolo/apps/user/tables.py +++ b/piccolo/apps/user/tables.py @@ -4,6 +4,7 @@ from __future__ import annotations import hashlib +import logging import secrets import typing as t @@ -12,6 +13,8 @@ from piccolo.table import Table from piccolo.utils.sync import run_sync +logger = logging.getLogger(__file__) + class BaseUser(Table, tablename="piccolo_user"): """ @@ -41,6 +44,8 @@ class BaseUser(Table, tablename="piccolo_user"): help_text="When this user last logged in.", ) + _max_password_length = 128 + def __init__(self, **kwargs): # Generating passwords upfront is expensive, so might need reworking. password = kwargs.get("password", None) @@ -81,7 +86,7 @@ async def update_password(cls, user: t.Union[str, int], password: str): ) password = cls.hash_password(password) - await cls.update().values({cls.password: password}).where(clause).run() + await cls.update({cls.password: password}).where(clause).run() ########################################################################### @@ -92,7 +97,15 @@ def hash_password( """ Hashes the password, ready for storage, and for comparing during login. + + :raises ValueError: + If an excessively long password is provided. + """ + if len(password) > cls._max_password_length: + logger.warning("Excessively long password provided.") + raise ValueError("The password is too long.") + if salt == "": salt = cls.get_salt() hashed = hashlib.pbkdf2_hmac( @@ -131,13 +144,20 @@ async def login(cls, username: str, password: str) -> t.Optional[int]: """ Returns the user_id if a match is found. """ - query = ( - cls.select() - .columns(cls._meta.primary_key, cls.password) - .where((cls.username == username)) + if len(username) > cls.username.length: + logger.warning("Excessively long username provided.") + return None + + if len(password) > cls._max_password_length: + logger.warning("Excessively long password provided.") + return None + + response = ( + await cls.select(cls._meta.primary_key, cls.password) + .where(cls.username == username) .first() + .run() ) - response = await query.run() if not response: # No match found return None diff --git a/tests/apps/user/test_tables.py b/tests/apps/user/test_tables.py index 4e493f229..492183de5 100644 --- a/tests/apps/user/test_tables.py +++ b/tests/apps/user/test_tables.py @@ -1,5 +1,6 @@ -import asyncio +import secrets from unittest import TestCase +from unittest.mock import MagicMock, call, patch from piccolo.apps.user.tables import BaseUser @@ -23,9 +24,23 @@ def test_create_user_table(self): self.assertFalse(exception) -class TestHashPassword(TestCase): - def test_hash_password(self): - pass +class TestInstantiateUser(TestCase): + def setUp(self): + BaseUser.create_table().run_sync() + + def tearDown(self): + BaseUser.alter().drop_table().run_sync() + + def test_valid_credentials(self): + BaseUser(username="bob", password="abc123%£1pscl") + + def test_malicious_password(self): + malicious_password = secrets.token_urlsafe(1000) + with self.assertRaises(ValueError) as manager: + BaseUser(username="bob", password=malicious_password) + self.assertEqual( + manager.exception.__str__(), "The password is too long." + ) class TestLogin(TestCase): @@ -35,22 +50,41 @@ def setUp(self): def tearDown(self): BaseUser.alter().drop_table().run_sync() - def test_login(self): + @patch("piccolo.apps.user.tables.logger") + def test_login(self, logger: MagicMock): username = "bob" password = "Bob123$$$" email = "bob@bob.com" user = BaseUser(username=username, password=password, email=email) + user.save().run_sync() - save_query = user.save() - - save_query.run_sync() - - authenticated = asyncio.run(BaseUser.login(username, password)) - self.assertTrue(authenticated is not None) - - authenticated = asyncio.run(BaseUser.login(username, "blablabla")) - self.assertTrue(not authenticated) + # Test correct password + authenticated = BaseUser.login_sync(username, password) + self.assertTrue(authenticated == user.id) # type: ignore + + # Test incorrect password + authenticated = BaseUser.login_sync(username, "blablabla") + self.assertTrue(authenticated is None) + + # Test ultra long password + malicious_password = secrets.token_urlsafe(1000) + authenticated = BaseUser.login_sync(username, malicious_password) + self.assertTrue(authenticated is None) + self.assertEqual( + logger.method_calls, + [call.warning("Excessively long password provided.")], + ) + + # Test ulta long username + logger.reset_mock() + malicious_username = secrets.token_urlsafe(1000) + authenticated = BaseUser.login_sync(malicious_username, password) + self.assertTrue(authenticated is None) + self.assertEqual( + logger.method_calls, + [call.warning("Excessively long username provided.")], + ) def test_update_password(self): username = "bob" @@ -63,7 +97,17 @@ def test_update_password(self): authenticated = BaseUser.login_sync(username, password) self.assertTrue(authenticated is not None) + # Test success new_password = "XXX111" BaseUser.update_password_sync(username, new_password) authenticated = BaseUser.login_sync(username, new_password) self.assertTrue(authenticated is not None) + + # Test ultra long password + malicious_password = secrets.token_urlsafe(1000) + with self.assertRaises(ValueError) as manager: + BaseUser.update_password_sync(username, malicious_password) + self.assertEqual( + manager.exception.__str__(), + "The password is too long.", + ) From 2d207d8f8a1ea501016337aea692e59461133301 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 31 Dec 2021 06:49:21 +0000 Subject: [PATCH 220/727] bumped version --- CHANGES.rst | 8 ++++++++ piccolo/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 9f9701923..6c986ed3e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,14 @@ Changes ======= +0.62.2 +------ + +Added a max password length to the ``BaseUser`` table. By default it's set to +128 characters. + +------------------------------------------------------------------------------- + 0.62.1 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index d8069b8e5..b620e2eed 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.62.1" +__VERSION__ = "0.62.2" From e5896042fd0ff9f6c626cc83b5e19ab0a7525f5e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 1 Jan 2022 21:07:47 +0000 Subject: [PATCH 221/727] reference Piccolo API endpoints / middleware in auth docs (#383) * reference Piccolo API endpoints / middleware in auth docs * update mypy * fix type error with latest pydantic version --- docs/src/piccolo/authentication/baseuser.rst | 12 ++++++------ docs/src/piccolo/authentication/index.rst | 13 ++++++++++++- piccolo/apps/migrations/commands/backwards.py | 2 +- piccolo/utils/pydantic.py | 2 +- requirements/dev-requirements.txt | 4 ++-- 5 files changed, 22 insertions(+), 11 deletions(-) diff --git a/docs/src/piccolo/authentication/baseuser.rst b/docs/src/piccolo/authentication/baseuser.rst index 5146ea766..68183e196 100644 --- a/docs/src/piccolo/authentication/baseuser.rst +++ b/docs/src/piccolo/authentication/baseuser.rst @@ -23,8 +23,8 @@ Commands The app comes with some useful commands. -user create -~~~~~~~~~~~ +create +~~~~~~ Creates a new user. It presents an interactive prompt, asking for the username, password etc. @@ -43,8 +43,8 @@ script), you can pass all of the arguments in as follows: If you choose this approach then be careful, as the password will be in the shell's history. -user change_password -~~~~~~~~~~~~~~~~~~~~ +change_password +~~~~~~~~~~~~~~~ Change a user's password. @@ -52,8 +52,8 @@ Change a user's password. piccolo user change_password -user change_permissions -~~~~~~~~~~~~~~~~~~~~~~~ +change_permissions +~~~~~~~~~~~~~~~~~~ Change a user's permissions. The options are ``--admin``, ``--superuser`` and ``--active``, which change the corresponding attributes on ``BaseUser``. diff --git a/docs/src/piccolo/authentication/index.rst b/docs/src/piccolo/authentication/index.rst index 731caaf00..48609827a 100644 --- a/docs/src/piccolo/authentication/index.rst +++ b/docs/src/piccolo/authentication/index.rst @@ -3,7 +3,7 @@ Authentication ============== -Piccolo ships with some authentication support out of the box. +Piccolo ships with authentication support out of the box. ------------------------------------------------------------------------------- @@ -22,3 +22,14 @@ Tables :maxdepth: 1 ./baseuser + +------------------------------------------------------------------------------- + +Web app integration +------------------- + +Our sister project, `Piccolo API `_, +contains powerful endpoints and middleware for integrating +`session auth `_ +and `token auth `_ +into your ASGI web application, using ``BaseUser``. diff --git a/piccolo/apps/migrations/commands/backwards.py b/piccolo/apps/migrations/commands/backwards.py index e8ca28c08..b7501a0e0 100644 --- a/piccolo/apps/migrations/commands/backwards.py +++ b/piccolo/apps/migrations/commands/backwards.py @@ -97,7 +97,7 @@ async def run(self) -> MigrationResult: Migration.name == migration_id ).run() - if self.clean: + if self.clean and migration_module.__file__: os.unlink(migration_module.__file__) print("ok! ✔️") diff --git a/piccolo/utils/pydantic.py b/piccolo/utils/pydantic.py index 192c0641a..26c160385 100644 --- a/piccolo/utils/pydantic.py +++ b/piccolo/utils/pydantic.py @@ -32,7 +32,7 @@ class Config(pydantic.BaseConfig): - json_encoders = JSON_ENCODERS + json_encoders: t.Dict[t.Any, t.Callable] = JSON_ENCODERS arbitrary_types_allowed = True diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt index 59adc46a1..b91536419 100644 --- a/requirements/dev-requirements.txt +++ b/requirements/dev-requirements.txt @@ -4,6 +4,6 @@ ipython==7.30.1 flake8==4.0.1 isort==5.10.1 twine==3.7.1 -mypy==0.920 +mypy==0.930 pip-upgrader==1.4.15 -wheel==0.37.0 +wheel==0.37.1 From 850d7fd833ab4a7337655e91cba42f2a9ce3816e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 7 Jan 2022 22:14:37 +0000 Subject: [PATCH 222/727] fix error message for `LazyTableReference` (#385) --- piccolo/columns/reference.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piccolo/columns/reference.py b/piccolo/columns/reference.py index 167712ab2..c4991aae4 100644 --- a/piccolo/columns/reference.py +++ b/piccolo/columns/reference.py @@ -69,8 +69,8 @@ def resolve(self) -> t.Type[Table]: return table else: raise ValueError( - f"Can't find a Table subclass called {self.app_name} " - f"in {self.module_path}" + "Can't find a Table subclass called " + f"{self.table_class_name} in {self.module_path}" ) raise ValueError("You must specify either app_name or module_path.") From 54441fdeb88e341fbbfafd44ce01a8772d27c25d Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 10 Jan 2022 13:34:43 +0000 Subject: [PATCH 223/727] Nested model fix (#389) * using __qualname__ * add test with `model_name` arg --- piccolo/utils/pydantic.py | 11 ++++++++--- tests/utils/test_pydantic.py | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/piccolo/utils/pydantic.py b/piccolo/utils/pydantic.py index 26c160385..fef39898c 100644 --- a/piccolo/utils/pydantic.py +++ b/piccolo/utils/pydantic.py @@ -198,6 +198,8 @@ def create_pydantic_model( ) ) + model_name = model_name or table.__name__ + for column in piccolo_columns: column_name = column._meta.name @@ -255,6 +257,7 @@ def create_pydantic_model( ) ) ): + nested_model_name = f"{model_name}.{column._meta.name}" _type = create_pydantic_model( table=column._foreign_key_meta.resolved_references, nested=nested, @@ -266,6 +269,7 @@ def create_pydantic_model( deserialize_json=deserialize_json, recursion_depth=recursion_depth + 1, max_recursion_depth=max_recursion_depth, + model_name=nested_model_name, ) tablename = ( @@ -288,17 +292,18 @@ def create_pydantic_model( columns[column_name] = (_type, field) - model_name = model_name or table.__name__ - class CustomConfig(Config): schema_extra = { "help_text": table._meta.help_text, **schema_extra_kwargs, } - return pydantic.create_model( + model = pydantic.create_model( model_name, __config__=CustomConfig, __validators__=validators, **columns, ) + model.__qualname__ = model_name + + return model diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index ab23187e7..b2aa6328c 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -451,6 +451,7 @@ class Concert(Table): self.assertEqual( [i for i in ManagerModel.__fields__.keys()], ["name", "country"] ) + self.assertEqual(ManagerModel.__qualname__, "Band.manager") AssistantManagerType = BandModel.__fields__["assistant_manager"].type_ self.assertTrue(AssistantManagerType is int) @@ -467,6 +468,7 @@ class Concert(Table): self.assertEqual( [i for i in ManagerModel.__fields__.keys()], ["name", "country"] ) + self.assertEqual(ManagerModel.__qualname__, "Band.manager") AssistantManagerType = BandModel.__fields__["assistant_manager"].type_ self.assertTrue(AssistantManagerType is int) @@ -474,6 +476,7 @@ class Concert(Table): CountryModel = ManagerModel.__fields__["country"].type_ self.assertTrue(issubclass(CountryModel, pydantic.BaseModel)) self.assertEqual([i for i in CountryModel.__fields__.keys()], ["name"]) + self.assertEqual(CountryModel.__qualname__, "Band.manager.country") ####################################################################### # Test three levels deep @@ -491,6 +494,7 @@ class Concert(Table): [i for i in BandModel.__fields__.keys()], ["name", "manager", "assistant_manager"], ) + self.assertEqual(BandModel.__qualname__, "Concert.band_1") ManagerModel = BandModel.__fields__["manager"].type_ self.assertTrue(issubclass(ManagerModel, pydantic.BaseModel)) @@ -498,6 +502,7 @@ class Concert(Table): [i for i in ManagerModel.__fields__.keys()], ["name", "country"], ) + self.assertEqual(ManagerModel.__qualname__, "Concert.band_1.manager") AssistantManagerType = BandModel.__fields__["assistant_manager"].type_ self.assertTrue(AssistantManagerType is int) @@ -505,6 +510,23 @@ class Concert(Table): CountryModel = ManagerModel.__fields__["country"].type_ self.assertTrue(CountryModel is int) + ####################################################################### + # Test with `model_name` arg + + MyConcertModel = create_pydantic_model( + Concert, + nested=(Concert.band_1.manager,), + model_name="MyConcertModel", + ) + + BandModel = MyConcertModel.__fields__["band_1"].type_ + self.assertEqual(BandModel.__qualname__, "MyConcertModel.band_1") + + ManagerModel = BandModel.__fields__["manager"].type_ + self.assertEqual( + ManagerModel.__qualname__, "MyConcertModel.band_1.manager" + ) + def test_cascaded_args(self): """ Make sure that arguments passed to ``create_pydantic_model`` are From e3d6a3850b0166b43df399dcca241ac60fb6a2cc Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 10 Jan 2022 13:40:10 +0000 Subject: [PATCH 224/727] bumped version --- CHANGES.rst | 16 ++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 6c986ed3e..1de6357db 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,22 @@ Changes ======= +0.62.3 +------ + +Fixed the error message in ``LazyTableReference``. + +Fixed a bug with ``create_pydantic_model`` with nested models. For example: + +.. code-block:: python + + create_pydantic_model(Band, nested=(Band.manager,)) + +Sometimes Pydantic couldn't uniquely identify the nested models. Thanks to +@wmshort and @sinisaos for their help with this. + +------------------------------------------------------------------------------- + 0.62.2 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index b620e2eed..f5f8ce129 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.62.2" +__VERSION__ = "0.62.3" From 5a337e4ee1021521eacf2ea96f37854755fcbcef Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 11 Jan 2022 16:17:40 +0000 Subject: [PATCH 225/727] added `exclude_imported` option to `table_finder` (#391) --- .../templates/app/home/piccolo_app.py.jinja | 2 +- piccolo/conf/apps.py | 29 +++++++++++++++---- tests/conf/example.py | 12 ++++++++ tests/conf/test_apps.py | 25 ++++++++++++++++ 4 files changed, 62 insertions(+), 6 deletions(-) create mode 100644 tests/conf/example.py diff --git a/piccolo/apps/asgi/commands/templates/app/home/piccolo_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/home/piccolo_app.py.jinja index 7b9eb4f4a..a2ac77d56 100644 --- a/piccolo/apps/asgi/commands/templates/app/home/piccolo_app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/home/piccolo_app.py.jinja @@ -16,7 +16,7 @@ APP_CONFIG = AppConfig( migrations_folder_path=os.path.join( CURRENT_DIRECTORY, "piccolo_migrations" ), - table_classes=table_finder(modules=["home.tables"]), + table_classes=table_finder(modules=["home.tables"], exclude_imported=True), migration_dependencies=[], commands=[], ) diff --git a/piccolo/conf/apps.py b/piccolo/conf/apps.py index 2b08b51f5..e3f9e123d 100644 --- a/piccolo/conf/apps.py +++ b/piccolo/conf/apps.py @@ -33,26 +33,41 @@ def table_finder( modules: t.Sequence[str], include_tags: t.Sequence[str] = ["__all__"], exclude_tags: t.Sequence[str] = [], + exclude_imported: bool = False, ) -> t.List[t.Type[Table]]: """ Rather than explicitly importing and registering table classes with the - AppConfig, ``table_finder`` can be used instead. It imports any ``Table`` + ``AppConfig``, ``table_finder`` can be used instead. It imports any ``Table`` subclasses in the given modules. Tags can be used to limit which ``Table`` subclasses are imported. :param modules: The module paths to check for ``Table`` subclasses. For example, - ['blog.tables']. The path should be from the root of your project, + ``['blog.tables']``. The path should be from the root of your project, not a relative path. :param include_tags: If the ``Table`` subclass has one of these tags, it will be - imported. The special tag '__all__' will import all ``Table`` + imported. The special tag ``'__all__'`` will import all ``Table`` subclasses found. :param exclude_tags: If the ``Table`` subclass has any of these tags, it won't be - imported. `exclude_tags` overrides `include_tags`. + imported. ``exclude_tags`` overrides ``include_tags``. + :param exclude_imported: + If ``True``, only ``Table`` subclasses defined within the module are + used. Any ``Table`` subclasses imported by that module from other + modules are ignored. For example: - """ + .. code-block:: python + + from piccolo.table import Table + from piccolo.column import Varchar, ForeignKey + from piccolo.apps.user.tables import BaseUser # excluded + + class Task(Table): # included + title = Varchar() + creator = ForeignKey(BaseUser) + + """ # noqa: E501 if isinstance(modules, str): # Guard against the user just entering a string, for example # 'blog.tables', instead of ['blog.tables']. @@ -77,6 +92,10 @@ def table_finder( and _object is not Table ): table: Table = _object # type: ignore + + if exclude_imported and table.__module__ != module_path: + continue + if exclude_tags and set(table._meta.tags).intersection( set(exclude_tags) ): diff --git a/tests/conf/example.py b/tests/conf/example.py new file mode 100644 index 000000000..c7dce05a5 --- /dev/null +++ b/tests/conf/example.py @@ -0,0 +1,12 @@ +""" +This file is used by test_apps.py to make sure we can exclude imported +``Table`` subclasses when using ``table_finder``. +""" +from piccolo.apps.user.tables import BaseUser +from piccolo.columns.column_types import ForeignKey, Varchar +from piccolo.table import Table + + +class Musician(Table): + name = Varchar() + user = ForeignKey(BaseUser) diff --git a/tests/conf/test_apps.py b/tests/conf/test_apps.py index dc7d49359..0435b6d5e 100644 --- a/tests/conf/test_apps.py +++ b/tests/conf/test_apps.py @@ -177,6 +177,31 @@ def test_exclude_tags(self): ], ) + def test_exclude_imported(self): + """ + Make sure we can excluded imported Tables. + """ + filtered_tables = table_finder( + modules=["tests.conf.example"], + exclude_imported=True, + ) + + self.assertEqual( + [i.__name__ for i in filtered_tables], + ["Musician"], + ) + + # Now try without filtering: + all_tables = table_finder( + modules=["tests.conf.example"], + exclude_imported=False, + ) + + self.assertEqual( + sorted([i.__name__ for i in all_tables]), + ["BaseUser", "Musician"], + ) + class TestFinder(TestCase): def test_get_table_classes(self): From d84044dcc0bce8f2f3005cf081670a2e37eb82e9 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 11 Jan 2022 16:30:53 +0000 Subject: [PATCH 226/727] bumped version --- CHANGES.rst | 31 +++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 1de6357db..990359652 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,37 @@ Changes ======= +0.63.0 +------ + +Added an ``exclude_imported`` option to ``table_finder``. + +.. code-block:: python + + APP_CONFIG = AppConfig( + table_classes=table_finder(['music.tables'], exclude_imported=True) + ) + +It's useful when we want to import ``Table`` subclasses defined within a +module itself, but not imported ones: + +.. code-block:: python + + # tables.py + from piccolo.apps.user.tables import BaseUser # excluded + from piccolo.columns.column_types import ForeignKey, Varchar + from piccolo.table import Table + + + class Musician(Table): # included + name = Varchar() + user = ForeignKey(BaseUser) + +This was also possible using tags, but was less convenient. Thanks to @sinisaos +for reporting this issue. + +------------------------------------------------------------------------------- + 0.62.3 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index f5f8ce129..23c908d73 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.62.3" +__VERSION__ = "0.63.0" From 991b95bfb0c575bac45397082a85c7fa17e89d72 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 12 Jan 2022 20:56:54 +0000 Subject: [PATCH 227/727] make the `value_type` dynamic for `ForeignKey` (#392) * make the value_type dynamic for PrimaryKey * added test for `value_type` --- piccolo/columns/column_types.py | 11 ++++++++++- tests/columns/test_foreignkey.py | 28 +++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index ba3d4bbc6..7a96168c1 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -1177,7 +1177,7 @@ class Band(Table): @property def column_type(self): """ - A ForeignKey column needs to have the same type as the primary key + A ``ForeignKey`` column needs to have the same type as the primary key column of the table being referenced. """ referenced_table = self._foreign_key_meta.resolved_references @@ -1187,6 +1187,15 @@ def column_type(self): else: return pk_column.column_type + @property + def value_type(self): + """ + The value type matches that of the primary key being referenced. + """ + referenced_table = self._foreign_key_meta.resolved_references + pk_column = referenced_table._meta.primary_key + return pk_column.value_type + def __init__( self, references: t.Union[t.Type[Table], LazyTableReference, str], diff --git a/tests/columns/test_foreignkey.py b/tests/columns/test_foreignkey.py index a4e206496..924a911c1 100644 --- a/tests/columns/test_foreignkey.py +++ b/tests/columns/test_foreignkey.py @@ -1,7 +1,14 @@ import time +import uuid from unittest import TestCase -from piccolo.columns import Column, ForeignKey, LazyTableReference, Varchar +from piccolo.columns import ( + UUID, + Column, + ForeignKey, + LazyTableReference, + Varchar, +) from piccolo.columns.base import OnDelete, OnUpdate from piccolo.table import Table from tests.base import postgres_only @@ -46,6 +53,25 @@ class Band5(Table): ) +class ManagerUUID(Table): + pk = UUID(primary_key=True) + + +class Band6(Table): + manager = ForeignKey(references=ManagerUUID) + + +class TestValueType(TestCase): + """ + The `value_type` of the `ForeignKey` should depend on the `PrimaryKey` of + the referenced table. + """ + + def test_value_type(self): + self.assertTrue(Band1.manager.value_type is int) + self.assertTrue(Band6.manager.value_type is uuid.UUID) + + class TestForeignKeyMeta(TestCase): """ Make sure that `ForeignKeyMeta` is setup correctly. From 6b23206ac1b870a9f4bf4fd290cdec28e425867f Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 12 Jan 2022 21:03:57 +0000 Subject: [PATCH 228/727] bumped version --- CHANGES.rst | 8 ++++++++ piccolo/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 990359652..9d226319b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,14 @@ Changes ======= +0.63.1 +------ + +Fixed an issue with the ``value_type`` of ``ForeignKey`` columns when +referencing a table with a custom primary key column (such as a ``UUID``). + +------------------------------------------------------------------------------- + 0.63.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 23c908d73..c7737bbd8 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.63.0" +__VERSION__ = "0.63.1" From db702b989ae4018167487bf2deaf2cf2c099c16f Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 16 Jan 2022 20:01:50 +0000 Subject: [PATCH 229/727] allow setting target column for ForeignKey (#395) * allow setting target column for ForeignKey * make sure `target_column` is added to kwargs * add a test for migrations * add a test to ensure the foreign key constraint was created correctly * break up foreign key tests The file was getting too hard to understand with all of the different permutations of table. * add more migration tests When using a column reference instead of a string for `target_column`. * add target_column attribute to pydantic schema --- piccolo/apps/migrations/auto/serialisation.py | 54 ++- piccolo/columns/base.py | 26 +- piccolo/columns/column_types.py | 28 +- piccolo/query/methods/select.py | 2 +- piccolo/utils/pydantic.py | 7 +- piccolo/utils/sql_values.py | 15 +- .../auto/integration/test_migrations.py | 91 +++++ tests/columns/foreign_key/test_all_columns.py | 51 +++ tests/columns/foreign_key/test_all_related.py | 62 +++ .../foreign_key/test_attribute_access.py | 62 +++ .../foreign_key/test_foreign_key_meta.py | 36 ++ .../test_foreign_key_references.py | 28 ++ .../foreign_key/test_foreign_key_self.py | 42 ++ .../foreign_key/test_foreign_key_string.py | 69 ++++ .../foreign_key/test_on_delete_on_update.py | 62 +++ .../columns/foreign_key/test_target_column.py | 87 ++++ tests/columns/foreign_key/test_value_type.py | 33 ++ tests/columns/test_foreignkey.py | 373 ------------------ tests/utils/test_pydantic.py | 42 +- 19 files changed, 779 insertions(+), 391 deletions(-) create mode 100644 tests/columns/foreign_key/test_all_columns.py create mode 100644 tests/columns/foreign_key/test_all_related.py create mode 100644 tests/columns/foreign_key/test_attribute_access.py create mode 100644 tests/columns/foreign_key/test_foreign_key_meta.py create mode 100644 tests/columns/foreign_key/test_foreign_key_references.py create mode 100644 tests/columns/foreign_key/test_foreign_key_self.py create mode 100644 tests/columns/foreign_key/test_foreign_key_string.py create mode 100644 tests/columns/foreign_key/test_on_delete_on_update.py create mode 100644 tests/columns/foreign_key/test_target_column.py create mode 100644 tests/columns/foreign_key/test_value_type.py delete mode 100644 tests/columns/test_foreignkey.py diff --git a/piccolo/apps/migrations/auto/serialisation.py b/piccolo/apps/migrations/auto/serialisation.py index 2217d66cf..44d99e9d3 100644 --- a/piccolo/apps/migrations/auto/serialisation.py +++ b/piccolo/apps/migrations/auto/serialisation.py @@ -363,12 +363,50 @@ def __repr__(self): serialised_params=serialise_params(params=pk_column._meta.params), ) - return ( + ####################################################################### + + # When creating a ForeignKey, the user can specify a column other than + # the primary key to reference. + serialised_target_columns: t.Set[SerialisedColumnInstance] = set() + + for fk_column in self.table_type._meta._foreign_key_references: + target_column = fk_column._foreign_key_meta.target_column + if target_column is None: + # Just references the primary key + continue + elif type(target_column) is str: + column = self.table_type._meta.get_column_by_name( + target_column + ) + elif isinstance(target_column, Column): + column = self.table_type._meta.get_column_by_name( + target_column._meta.name + ) + else: + raise ValueError("Unrecognised `target_column` value.") + + serialised_target_columns.add( + SerialisedColumnInstance( + column, + serialised_params=serialise_params( + params=column._meta.params + ), + ) + ) + + ####################################################################### + + definition = ( f"class {self.table_class_name}" f'({UniqueGlobalNames.TABLE}, tablename="{tablename}"): ' f"{pk_column_name} = {serialised_pk_column}" ) + for serialised_target_column in serialised_target_columns: + definition += f"; {serialised_target_column.instance._meta.name} = {serialised_target_column}" # noqa: E501 + + return definition + def __lt__(self, other): return repr(self) < repr(other) @@ -459,8 +497,20 @@ def serialise_params(params: t.Dict[str, t.Any]) -> SerialisedParams: params[key] = SerialisedBuiltin(builtin=value) continue - # Column instances, which are used by Array definitions. + # Column instances if isinstance(value, Column): + + # For target_column (which is used by ForeignKey), we can just + # serialise it as the column name: + if key == "target_column": + params[key] = value._meta.name + continue + + ################################################################### + + # For Array definitions, we want to serialise the full column + # definition: + column: Column = value serialised_params: SerialisedParams = serialise_params( params=column._meta.params diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 8174ab9d3..30282ad50 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -72,15 +72,16 @@ class ForeignKeyMeta: references: t.Union[t.Type[Table], LazyTableReference] on_delete: OnDelete on_update: OnUpdate + target_column: t.Union[Column, str, None] proxy_columns: t.List[Column] = field(default_factory=list) @property def resolved_references(self) -> t.Type[Table]: """ - Evaluates the ``references`` attribute if it's a LazyTableReference, + Evaluates the ``references`` attribute if it's a ``LazyTableReference``, raising a ``ValueError`` if it fails, otherwise returns a ``Table`` subclass. - """ + """ # noqa: E501 from piccolo.table import Table if isinstance(self.references, LazyTableReference): @@ -95,6 +96,21 @@ def resolved_references(self) -> t.Type[Table]: "LazyTableReference instance." ) + @property + def resolved_target_column(self) -> Column: + if self.target_column is None: + return self.resolved_references._meta.primary_key + elif isinstance(self.target_column, Column): + return self.resolved_references._meta.get_column_by_name( + self.target_column._meta.name + ) + elif isinstance(self.target_column, str): + return self.resolved_references._meta.get_column_by_name( + self.target_column + ) + else: + raise ValueError("Unable to resolve target_column.") + def copy(self) -> ForeignKeyMeta: kwargs = self.__dict__.copy() kwargs.update(proxy_columns=self.proxy_columns.copy()) @@ -783,9 +799,11 @@ def ddl(self) -> str: tablename = references._meta.tablename on_delete = foreign_key_meta.on_delete.value on_update = foreign_key_meta.on_update.value - primary_key_name = references._meta.primary_key._meta.name + target_column_name = ( + foreign_key_meta.resolved_target_column._meta.name + ) query += ( - f" REFERENCES {tablename} ({primary_key_name})" + f" REFERENCES {tablename} ({target_column_name})" f" ON DELETE {on_delete}" f" ON UPDATE {on_update}" ) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 7a96168c1..73cdd3def 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -1170,6 +1170,19 @@ class Band(Table): on_update=OnUpdate.cascade ) + :param target_column: + By default the ``ForeignKey`` references the primary key column on the + related table. You can specify an alternative column (it must have a + unique constraint on it though). For example: + + .. code-block:: python + + # Passing in a column reference: + ForeignKey(references=Manager, target_column=Manager.passport_number) + + # Or just the column name: + ForeignKey(references=Manager, target_column='passport_number') + """ # noqa: E501 _foreign_key_meta: ForeignKeyMeta @@ -1180,21 +1193,19 @@ def column_type(self): A ``ForeignKey`` column needs to have the same type as the primary key column of the table being referenced. """ - referenced_table = self._foreign_key_meta.resolved_references - pk_column = referenced_table._meta.primary_key - if isinstance(pk_column, Serial): + target_column = self._foreign_key_meta.resolved_target_column + if isinstance(target_column, Serial): return Integer().column_type else: - return pk_column.column_type + return target_column.column_type @property def value_type(self): """ The value type matches that of the primary key being referenced. """ - referenced_table = self._foreign_key_meta.resolved_references - pk_column = referenced_table._meta.primary_key - return pk_column.value_type + target_column = self._foreign_key_meta.resolved_target_column + return target_column.value_type def __init__( self, @@ -1203,6 +1214,7 @@ def __init__( null: bool = True, on_delete: OnDelete = OnDelete.cascade, on_update: OnUpdate = OnUpdate.cascade, + target_column: t.Union[str, Column, None] = None, **kwargs, ) -> None: from piccolo.table import Table @@ -1227,6 +1239,7 @@ def __init__( "on_delete": on_delete, "on_update": on_update, "null": null, + "target_column": target_column, } ) @@ -1238,6 +1251,7 @@ def __init__( references=Table if isinstance(references, str) else references, on_delete=on_delete, on_update=on_update, + target_column=target_column, ) def _setup(self, table_class: t.Type[Table]) -> ForeignKeySetupResponse: diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 817ae45ee..7e19f8b8e 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -514,7 +514,7 @@ def _get_joins(self, columns: t.Sequence[Selectable]) -> t.List[str]: pk_name = column._meta.call_chain[ index - ]._foreign_key_meta.resolved_references._meta.primary_key._meta.name # noqa: E501 + ]._foreign_key_meta.resolved_target_column._meta.name _joins.append( f"LEFT JOIN {right_tablename} {table_alias}" diff --git a/piccolo/utils/pydantic.py b/piccolo/utils/pydantic.py index fef39898c..bc9a09259 100644 --- a/piccolo/utils/pydantic.py +++ b/piccolo/utils/pydantic.py @@ -276,7 +276,12 @@ def create_pydantic_model( column._foreign_key_meta.resolved_references._meta.tablename ) field = pydantic.Field( - extra={"foreign_key": True, "to": tablename, **extra}, + extra={ + "foreign_key": True, + "to": tablename, + "target_column": column._foreign_key_meta.resolved_target_column._meta.name, # noqa: E501 + **extra, + }, **params, ) if include_readable: diff --git a/piccolo/utils/sql_values.py b/piccolo/utils/sql_values.py index cb57afcfc..766d59e00 100644 --- a/piccolo/utils/sql_values.py +++ b/piccolo/utils/sql_values.py @@ -15,11 +15,22 @@ def convert_to_sql_value(value: t.Any, column: Column) -> t.Any: database. For example, Enums, Table instances, and dictionaries for JSON columns. """ - from piccolo.columns.column_types import JSON, JSONB + from piccolo.columns.column_types import JSON, JSONB, ForeignKey from piccolo.table import Table if isinstance(value, Table): - return getattr(value, value._meta.primary_key._meta.name) + if isinstance(column, ForeignKey): + return getattr( + value, + column._foreign_key_meta.resolved_target_column._meta.name, + ) + elif column._meta.primary_key: + return getattr(value, column._meta.name) + else: + raise ValueError( + "Table instance provided, and the column isn't a ForeignKey, " + "or primary key column." + ) elif isinstance(value, Enum): return value.value elif isinstance(column, (JSON, JSONB)) and not isinstance(value, str): diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index 1758a043c..ae325e6b4 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -834,3 +834,94 @@ def test_m2m(self): for table_class in [Band, Genre, GenreToBand]: self.assertTrue(table_class.table_exists().run_sync()) + + +############################################################################### + + +class TableA(Table): + name = Varchar(unique=True) + + +class TableB(Table): + table_a = ForeignKey(TableA, target_column="name") + + +class TableC(Table): + table_a = ForeignKey(TableA, target_column=TableA.name) + + +@postgres_only +class TestTargetColumn(MigrationTestCase): + def setUp(self): + pass + + def tearDown(self): + drop_tables(Migration, TableA, TableC) + + def test_target_column(self): + """ + Make sure migrations still work when a foreign key references a column + other than the primary key. + """ + self._test_migrations( + table_snapshots=[[TableA, TableC]], + ) + + for table_class in [TableA, TableC]: + self.assertTrue(table_class.table_exists().run_sync()) + + # Make sure the constraint was created correctly. + response = TableA.raw( + """ + SELECT EXISTS( + SELECT 1 + FROM INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE CCU + JOIN INFORMATION_SCHEMA.TABLE_CONSTRAINTS TC ON + CCU.CONSTRAINT_NAME = TC.CONSTRAINT_NAME + WHERE CONSTRAINT_TYPE = 'FOREIGN KEY' + AND TC.TABLE_NAME = 'table_c' + AND CCU.TABLE_NAME = 'table_a' + AND CCU.COLUMN_NAME = 'name' + ) + """ + ).run_sync() + self.assertTrue(response[0]["exists"]) + + +@postgres_only +class TestTargetColumnString(MigrationTestCase): + def setUp(self): + pass + + def tearDown(self): + drop_tables(Migration, TableA, TableB) + + def test_target_column(self): + """ + Make sure migrations still work when a foreign key references a column + other than the primary key. + """ + self._test_migrations( + table_snapshots=[[TableA, TableB]], + ) + + for table_class in [TableA, TableB]: + self.assertTrue(table_class.table_exists().run_sync()) + + # Make sure the constraint was created correctly. + response = TableA.raw( + """ + SELECT EXISTS( + SELECT 1 + FROM INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE CCU + JOIN INFORMATION_SCHEMA.TABLE_CONSTRAINTS TC ON + CCU.CONSTRAINT_NAME = TC.CONSTRAINT_NAME + WHERE CONSTRAINT_TYPE = 'FOREIGN KEY' + AND TC.TABLE_NAME = 'table_b' + AND CCU.TABLE_NAME = 'table_a' + AND CCU.COLUMN_NAME = 'name' + ) + """ + ).run_sync() + self.assertTrue(response[0]["exists"]) diff --git a/tests/columns/foreign_key/test_all_columns.py b/tests/columns/foreign_key/test_all_columns.py new file mode 100644 index 000000000..e2718ce5b --- /dev/null +++ b/tests/columns/foreign_key/test_all_columns.py @@ -0,0 +1,51 @@ +from unittest import TestCase + +from tests.example_apps.music.tables import Band, Concert + + +class TestAllColumns(TestCase): + def test_all_columns(self): + """ + Make sure you can retrieve all columns from a related table, without + explicitly specifying them. + """ + all_columns = Band.manager.all_columns() + self.assertEqual(all_columns, [Band.manager.id, Band.manager.name]) + + # Make sure the call chains are also correct. + self.assertEqual( + all_columns[0]._meta.call_chain, Band.manager.id._meta.call_chain + ) + self.assertEqual( + all_columns[1]._meta.call_chain, Band.manager.name._meta.call_chain + ) + + def test_all_columns_deep(self): + """ + Make sure ``all_columns`` works when the joins are several layers deep. + """ + all_columns = Concert.band_1.manager.all_columns() + self.assertEqual(all_columns, [Band.manager.id, Band.manager.name]) + + # Make sure the call chains are also correct. + self.assertEqual( + all_columns[0]._meta.call_chain, + Concert.band_1.manager.id._meta.call_chain, + ) + self.assertEqual( + all_columns[1]._meta.call_chain, + Concert.band_1.manager.name._meta.call_chain, + ) + + def test_all_columns_exclude(self): + """ + Make sure you can exclude some columns. + """ + self.assertEqual( + Band.manager.all_columns(exclude=["id"]), [Band.manager.name] + ) + + self.assertEqual( + Band.manager.all_columns(exclude=[Band.manager.id]), + [Band.manager.name], + ) diff --git a/tests/columns/foreign_key/test_all_related.py b/tests/columns/foreign_key/test_all_related.py new file mode 100644 index 000000000..737ad1924 --- /dev/null +++ b/tests/columns/foreign_key/test_all_related.py @@ -0,0 +1,62 @@ +from unittest import TestCase + +from tests.example_apps.music.tables import Ticket + + +class TestAllRelated(TestCase): + def test_all_related(self): + """ + Make sure you can retrieve all foreign keys from a related table, + without explicitly specifying them. + """ + all_related = Ticket.concert.all_related() + + self.assertEqual( + all_related, + [ + Ticket.concert.band_1, + Ticket.concert.band_2, + Ticket.concert.venue, + ], + ) + + # Make sure the call chains are also correct. + self.assertEqual( + all_related[0]._meta.call_chain, + Ticket.concert.band_1._meta.call_chain, + ) + self.assertEqual( + all_related[1]._meta.call_chain, + Ticket.concert.band_2._meta.call_chain, + ) + self.assertEqual( + all_related[2]._meta.call_chain, + Ticket.concert.venue._meta.call_chain, + ) + + def test_all_related_deep(self): + """ + Make sure ``all_related`` works when the joins are several layers deep. + """ + all_related = Ticket.concert.band_1.all_related() + self.assertEqual(all_related, [Ticket.concert.band_1.manager]) + + # Make sure the call chains are also correct. + self.assertEqual( + all_related[0]._meta.call_chain, + Ticket.concert.band_1.manager._meta.call_chain, + ) + + def test_all_related_exclude(self): + """ + Make sure you can exclude some columns. + """ + self.assertEqual( + Ticket.concert.all_related(exclude=["venue"]), + [Ticket.concert.band_1, Ticket.concert.band_2], + ) + + self.assertEqual( + Ticket.concert.all_related(exclude=[Ticket.concert.venue]), + [Ticket.concert.band_1, Ticket.concert.band_2], + ) diff --git a/tests/columns/foreign_key/test_attribute_access.py b/tests/columns/foreign_key/test_attribute_access.py new file mode 100644 index 000000000..c658d7d98 --- /dev/null +++ b/tests/columns/foreign_key/test_attribute_access.py @@ -0,0 +1,62 @@ +import time +from unittest import TestCase + +from piccolo.columns import Column, ForeignKey, LazyTableReference, Varchar +from piccolo.table import Table + + +class Manager(Table): + name = Varchar() + manager = ForeignKey("self") + + +class BandA(Table): + manager = ForeignKey(references=Manager) + + +class BandB(Table): + manager = ForeignKey(references="Manager") + + +class BandC(Table): + manager = ForeignKey( + references=LazyTableReference( + table_class_name="Manager", module_path=__name__ + ) + ) + + +class BandD(Table): + manager = ForeignKey(references=f"{__name__}.Manager") + + +class TestAttributeAccess(TestCase): + def test_attribute_access(self): + """ + Make sure that attribute access still works correctly with lazy + references. + """ + for band_table in (BandA, BandB, BandC, BandD): + self.assertTrue(isinstance(band_table.manager.name, Varchar)) + + def test_recursion_limit(self): + """ + When a table has a ForeignKey to itself, an Exception should be raised + if the call chain is too large. + """ + # Should be fine: + column: Column = Manager.manager.name + self.assertTrue(len(column._meta.call_chain), 1) + self.assertTrue(isinstance(column, Varchar)) + + with self.assertRaises(Exception): + Manager.manager.manager.manager.manager.manager.manager.manager.manager.manager.manager.manager.name # noqa + + def test_recursion_time(self): + """ + Make sure that a really large call chain doesn't take too long. + """ + start = time.time() + Manager.manager.manager.manager.manager.manager.manager.name + end = time.time() + self.assertTrue(end - start < 1.0) diff --git a/tests/columns/foreign_key/test_foreign_key_meta.py b/tests/columns/foreign_key/test_foreign_key_meta.py new file mode 100644 index 000000000..78560fab8 --- /dev/null +++ b/tests/columns/foreign_key/test_foreign_key_meta.py @@ -0,0 +1,36 @@ +from unittest import TestCase + +from piccolo.columns import ForeignKey, Varchar +from piccolo.columns.base import OnDelete, OnUpdate +from piccolo.table import Table + + +class Manager(Table): + name = Varchar() + + +class Band(Table): + """ + Contains a ForeignKey with non-default `on_delete` and `on_update` values. + """ + + manager = ForeignKey( + references=Manager, + on_delete=OnDelete.set_null, + on_update=OnUpdate.set_null, + ) + + +class TestForeignKeyMeta(TestCase): + """ + Make sure that `ForeignKeyMeta` is setup correctly. + """ + + def test_foreignkeymeta(self): + self.assertTrue( + Band.manager._foreign_key_meta.on_update == OnUpdate.set_null + ) + self.assertTrue( + Band.manager._foreign_key_meta.on_delete == OnDelete.set_null + ) + self.assertTrue(Band.manager._foreign_key_meta.references == Manager) diff --git a/tests/columns/foreign_key/test_foreign_key_references.py b/tests/columns/foreign_key/test_foreign_key_references.py new file mode 100644 index 000000000..5e38a3ec2 --- /dev/null +++ b/tests/columns/foreign_key/test_foreign_key_references.py @@ -0,0 +1,28 @@ +from unittest import TestCase + +from piccolo.columns import ForeignKey, Varchar +from piccolo.table import Table + + +class Manager(Table, tablename="manager_fk_references_test"): + name = Varchar() + + +class BandA(Table): + manager = ForeignKey(references=Manager) + + +class BandB(Table): + manager = ForeignKey(references="Manager") + + +class TestReferences(TestCase): + def test_foreign_key_references(self): + """ + Make sure foreign key references are stored correctly on the table + which is the target of the ForeignKey. + """ + self.assertEqual(len(Manager._meta.foreign_key_references), 2) + + self.assertTrue(BandA.manager in Manager._meta.foreign_key_references) + self.assertTrue(BandB.manager in Manager._meta.foreign_key_references) diff --git a/tests/columns/foreign_key/test_foreign_key_self.py b/tests/columns/foreign_key/test_foreign_key_self.py new file mode 100644 index 000000000..a594379de --- /dev/null +++ b/tests/columns/foreign_key/test_foreign_key_self.py @@ -0,0 +1,42 @@ +from unittest import TestCase + +from piccolo.columns import ForeignKey, Varchar +from piccolo.table import Table + + +class Manager(Table, tablename="manager"): + name = Varchar() + manager = ForeignKey("self", null=True) + + +class TestForeignKeySelf(TestCase): + """ + Test that ForeignKey columns can be created with references to the parent + table. + """ + + def setUp(self): + Manager.create_table().run_sync() + + def tearDown(self): + Manager.alter().drop_table().run_sync() + + def test_foreign_key_self(self): + manager = Manager(name="Mr Manager") + manager.save().run_sync() + + worker = Manager(name="Mr Worker", manager=manager.id) + worker.save().run_sync() + + response = ( + Manager.select(Manager.name, Manager.manager.name) + .order_by(Manager.name) + .run_sync() + ) + self.assertEqual( + response, + [ + {"name": "Mr Manager", "manager.name": None}, + {"name": "Mr Worker", "manager.name": "Mr Manager"}, + ], + ) diff --git a/tests/columns/foreign_key/test_foreign_key_string.py b/tests/columns/foreign_key/test_foreign_key_string.py new file mode 100644 index 000000000..38d9e6bee --- /dev/null +++ b/tests/columns/foreign_key/test_foreign_key_string.py @@ -0,0 +1,69 @@ +from unittest import TestCase + +from piccolo.columns import ForeignKey, LazyTableReference, Varchar +from piccolo.table import Table + + +class Manager(Table): + name = Varchar() + + +class BandA(Table): + manager = ForeignKey(references="Manager") + + +class BandB(Table): + manager = ForeignKey( + references=LazyTableReference( + table_class_name="Manager", + module_path=__name__, + ) + ) + + +class BandC(Table, tablename="band"): + manager = ForeignKey(references=f"{__name__}.Manager") + + +class TestForeignKeyString(TestCase): + """ + Test that ForeignKey columns can be created with a `references` argument + set as a string value. + """ + + def test_foreign_key_string(self): + for band_table in (BandA, BandB, BandC): + self.assertEqual( + band_table.manager._foreign_key_meta.resolved_references, + Manager, + ) + + +class TestForeignKeyRelativeError(TestCase): + def test_foreign_key_relative_error(self): + """ + Make sure that a references argument which contains a relative module + isn't allowed. + """ + with self.assertRaises(ValueError) as manager: + + class BandRelative(Table, tablename="band"): + manager = ForeignKey("..example_app.tables.Manager", null=True) + + self.assertEqual( + manager.exception.__str__(), "Relative imports aren't allowed" + ) + + +class TestLazyTableReference(TestCase): + def test_lazy_reference_to_app(self): + """ + Make sure a LazyTableReference to a Table within a Piccolo app works. + """ + from tests.example_apps.music.tables import Manager + + reference = LazyTableReference( + table_class_name="Manager", app_name="music" + ) + + self.assertTrue(reference.resolve() is Manager) diff --git a/tests/columns/foreign_key/test_on_delete_on_update.py b/tests/columns/foreign_key/test_on_delete_on_update.py new file mode 100644 index 000000000..f82461303 --- /dev/null +++ b/tests/columns/foreign_key/test_on_delete_on_update.py @@ -0,0 +1,62 @@ +from unittest import TestCase + +from piccolo.columns import ForeignKey, Varchar +from piccolo.columns.base import OnDelete, OnUpdate +from piccolo.table import Table +from tests.base import postgres_only + + +class Manager(Table): + name = Varchar() + + +class Band(Table): + """ + Contains a ForeignKey with non-default `on_delete` and `on_update` values. + """ + + manager = ForeignKey( + references=Manager, + on_delete=OnDelete.set_null, + on_update=OnUpdate.set_null, + ) + + +@postgres_only +class TestOnDeleteOnUpdate(TestCase): + """ + Make sure that on_delete, and on_update are correctly applied in the + database. + """ + + def setUp(self): + for table_class in (Manager, Band): + table_class.create_table().run_sync() + + def tearDown(self): + for table_class in (Band, Manager): + table_class.alter().drop_table(if_exists=True).run_sync() + + def test_on_delete_on_update(self): + response = Band.raw( + """ + SELECT + rc.update_rule AS on_update, + rc.delete_rule AS on_delete + FROM information_schema.table_constraints tc + LEFT JOIN information_schema.key_column_usage kcu + ON tc.constraint_catalog = kcu.constraint_catalog + AND tc.constraint_schema = kcu.constraint_schema + AND tc.constraint_name = kcu.constraint_name + LEFT JOIN information_schema.referential_constraints rc + ON tc.constraint_catalog = rc.constraint_catalog + AND tc.constraint_schema = rc.constraint_schema + AND tc.constraint_name = rc.constraint_name + WHERE + lower(tc.constraint_type) in ('foreign key') + AND tc.table_name = 'band' + AND kcu.column_name = 'manager'; + """ + ).run_sync() + self.assertTrue(response[0]["on_update"] == "SET NULL") + self.assertTrue(response[0]["on_delete"] == "SET NULL") diff --git a/tests/columns/foreign_key/test_target_column.py b/tests/columns/foreign_key/test_target_column.py new file mode 100644 index 000000000..4e059dff8 --- /dev/null +++ b/tests/columns/foreign_key/test_target_column.py @@ -0,0 +1,87 @@ +from unittest import TestCase + +from piccolo.columns import ForeignKey, Varchar +from piccolo.table import Table, create_tables, drop_tables + + +class Manager(Table): + name = Varchar(unique=True) + + +class Band(Table): + name = Varchar() + manager = ForeignKey(Manager, target_column="name") + + +class TestTargetColumnWithString(TestCase): + """ + Make sure we can create tables with foreign keys which don't reference + the primary key. + """ + + def setUp(self): + create_tables(Manager, Band) + + def tearDown(self): + drop_tables(Manager, Band) + + def test_queries(self): + manager_1 = Manager.objects().create(name="Guido").run_sync() + manager_2 = Manager.objects().create(name="Graydon").run_sync() + + Band.insert( + Band(name="Pythonistas", manager=manager_1), + Band(name="Rustaceans", manager=manager_2), + ).run_sync() + + response = Band.select(Band.name, Band.manager.name).run_sync() + self.assertEqual( + response, + [ + {"name": "Pythonistas", "manager.name": "Guido"}, + {"name": "Rustaceans", "manager.name": "Graydon"}, + ], + ) + + +############################################################################### + + +class ManagerA(Table): + name = Varchar(unique=True) + + +class BandA(Table): + name = Varchar() + manager = ForeignKey(ManagerA, target_column=ManagerA.name) + + +class TestTargetColumnWithColumnRef(TestCase): + """ + Make sure we can create tables with foreign keys which don't reference + the primary key. + """ + + def setUp(self): + create_tables(ManagerA, BandA) + + def tearDown(self): + drop_tables(ManagerA, BandA) + + def test_queries(self): + manager_1 = ManagerA.objects().create(name="Guido").run_sync() + manager_2 = ManagerA.objects().create(name="Graydon").run_sync() + + BandA.insert( + BandA(name="Pythonistas", manager=manager_1), + BandA(name="Rustaceans", manager=manager_2), + ).run_sync() + + response = BandA.select(BandA.name, BandA.manager.name).run_sync() + self.assertEqual( + response, + [ + {"name": "Pythonistas", "manager.name": "Guido"}, + {"name": "Rustaceans", "manager.name": "Graydon"}, + ], + ) diff --git a/tests/columns/foreign_key/test_value_type.py b/tests/columns/foreign_key/test_value_type.py new file mode 100644 index 000000000..b1ca9808f --- /dev/null +++ b/tests/columns/foreign_key/test_value_type.py @@ -0,0 +1,33 @@ +import uuid +from unittest import TestCase + +from piccolo.columns import UUID, ForeignKey, Varchar +from piccolo.table import Table + + +class Manager(Table): + name = Varchar() + manager = ForeignKey("self", null=True) + + +class Band(Table): + manager = ForeignKey(references=Manager) + + +class ManagerUUID(Table): + pk = UUID(primary_key=True) + + +class BandUUID(Table): + manager = ForeignKey(references=ManagerUUID) + + +class TestValueType(TestCase): + """ + The `value_type` of the `ForeignKey` should depend on the `PrimaryKey` of + the referenced table. + """ + + def test_value_type(self): + self.assertTrue(Band.manager.value_type is int) + self.assertTrue(BandUUID.manager.value_type is uuid.UUID) diff --git a/tests/columns/test_foreignkey.py b/tests/columns/test_foreignkey.py deleted file mode 100644 index 924a911c1..000000000 --- a/tests/columns/test_foreignkey.py +++ /dev/null @@ -1,373 +0,0 @@ -import time -import uuid -from unittest import TestCase - -from piccolo.columns import ( - UUID, - Column, - ForeignKey, - LazyTableReference, - Varchar, -) -from piccolo.columns.base import OnDelete, OnUpdate -from piccolo.table import Table -from tests.base import postgres_only -from tests.example_apps.music.tables import Band, Concert, Manager, Ticket - - -class Manager1(Table, tablename="manager"): - name = Varchar() - manager = ForeignKey("self", null=True) - - -class Band1(Table, tablename="band"): - manager = ForeignKey(references=Manager1) - - -class Band2(Table, tablename="band"): - manager = ForeignKey(references="Manager1") - - -class Band3(Table, tablename="band"): - manager = ForeignKey( - references=LazyTableReference( - table_class_name="Manager1", - module_path="tests.columns.test_foreignkey", - ) - ) - - -class Band4(Table, tablename="band"): - manager = ForeignKey(references="tests.columns.test_foreignkey.Manager1") - - -class Band5(Table): - """ - Contains a ForeignKey with non-default `on_delete` and `on_update` values. - """ - - manager = ForeignKey( - references=Manager, - on_delete=OnDelete.set_null, - on_update=OnUpdate.set_null, - ) - - -class ManagerUUID(Table): - pk = UUID(primary_key=True) - - -class Band6(Table): - manager = ForeignKey(references=ManagerUUID) - - -class TestValueType(TestCase): - """ - The `value_type` of the `ForeignKey` should depend on the `PrimaryKey` of - the referenced table. - """ - - def test_value_type(self): - self.assertTrue(Band1.manager.value_type is int) - self.assertTrue(Band6.manager.value_type is uuid.UUID) - - -class TestForeignKeyMeta(TestCase): - """ - Make sure that `ForeignKeyMeta` is setup correctly. - """ - - def test_foreignkeymeta(self): - self.assertTrue( - Band5.manager._foreign_key_meta.on_update == OnUpdate.set_null - ) - self.assertTrue( - Band5.manager._foreign_key_meta.on_delete == OnDelete.set_null - ) - self.assertTrue(Band.manager._foreign_key_meta.references == Manager) - - -@postgres_only -class TestOnDeleteOnUpdate(TestCase): - """ - Make sure that on_delete, and on_update are correctly applied in the - database. - """ - - def setUp(self): - for table_class in (Manager, Band5): - table_class.create_table().run_sync() - - def tearDown(self): - for table_class in (Band5, Manager): - table_class.alter().drop_table(if_exists=True).run_sync() - - def test_on_delete_on_update(self): - response = Band5.raw( - """ - SELECT - rc.update_rule AS on_update, - rc.delete_rule AS on_delete - FROM information_schema.table_constraints tc - LEFT JOIN information_schema.key_column_usage kcu - ON tc.constraint_catalog = kcu.constraint_catalog - AND tc.constraint_schema = kcu.constraint_schema - AND tc.constraint_name = kcu.constraint_name - LEFT JOIN information_schema.referential_constraints rc - ON tc.constraint_catalog = rc.constraint_catalog - AND tc.constraint_schema = rc.constraint_schema - AND tc.constraint_name = rc.constraint_name - WHERE - lower(tc.constraint_type) in ('foreign key') - AND tc.table_name = 'band5' - AND kcu.column_name = 'manager'; - """ - ).run_sync() - self.assertTrue(response[0]["on_update"] == "SET NULL") - self.assertTrue(response[0]["on_delete"] == "SET NULL") - - -class TestForeignKeySelf(TestCase): - """ - Test that ForeignKey columns can be created with references to the parent - table. - """ - - def setUp(self): - Manager1.create_table().run_sync() - - def test_foreign_key_self(self): - manager = Manager1(name="Mr Manager") - manager.save().run_sync() - - worker = Manager1(name="Mr Worker", manager=manager.id) - worker.save().run_sync() - - response = ( - Manager1.select(Manager1.name, Manager1.manager.name) - .order_by(Manager1.name) - .run_sync() - ) - self.assertEqual( - response, - [ - {"name": "Mr Manager", "manager.name": None}, - {"name": "Mr Worker", "manager.name": "Mr Manager"}, - ], - ) - - def tearDown(self): - Manager1.alter().drop_table().run_sync() - - -class TestForeignKeyString(TestCase): - """ - Test that ForeignKey columns can be created with a `references` argument - set as a string value. - """ - - def setUp(self): - Manager1.create_table().run_sync() - - def test_foreign_key_string(self): - Band2.create_table().run_sync() - self.assertEqual( - Band2.manager._foreign_key_meta.resolved_references, - Manager1, - ) - Band2.alter().drop_table().run_sync() - - Band4.create_table().run_sync() - self.assertEqual( - Band4.manager._foreign_key_meta.resolved_references, - Manager1, - ) - Band4.alter().drop_table().run_sync() - - def tearDown(self): - Manager1.alter().drop_table().run_sync() - - -class TestForeignKeyRelativeError(TestCase): - def test_foreign_key_relative_error(self): - """ - Make sure that a references argument which contains a relative module - isn't allowed. - """ - with self.assertRaises(ValueError) as manager: - - class BandRelative(Table, tablename="band"): - manager = ForeignKey("..example_app.tables.Manager", null=True) - - self.assertEqual( - manager.exception.__str__(), "Relative imports aren't allowed" - ) - - -class TestReferences(TestCase): - def test_foreign_key_references(self): - """ - Make sure foreign key references are stored correctly on the table - which is the target of the ForeignKey. - """ - self.assertEqual(len(Manager1._meta.foreign_key_references), 5) - - self.assertTrue(Band1.manager in Manager._meta.foreign_key_references) - self.assertTrue(Band2.manager in Manager._meta.foreign_key_references) - self.assertTrue(Band3.manager in Manager._meta.foreign_key_references) - self.assertTrue(Band4.manager in Manager._meta.foreign_key_references) - self.assertTrue( - Manager1.manager in Manager1._meta.foreign_key_references - ) - - -class TestLazyTableReference(TestCase): - def test_lazy_reference_to_app(self): - """ - Make sure a LazyTableReference to a Table within a Piccolo app works. - """ - reference = LazyTableReference( - table_class_name="Manager", app_name="music" - ) - self.assertTrue(reference.resolve() is Manager) - - -class TestAttributeAccess(TestCase): - def test_attribute_access(self): - """ - Make sure that attribute access still works correctly with lazy - references. - """ - self.assertTrue(isinstance(Band1.manager.name, Varchar)) - self.assertTrue(isinstance(Band2.manager.name, Varchar)) - self.assertTrue(isinstance(Band3.manager.name, Varchar)) - self.assertTrue(isinstance(Band4.manager.name, Varchar)) - - def test_recursion_limit(self): - """ - When a table has a ForeignKey to itself, an Exception should be raised - if the call chain is too large. - """ - # Should be fine: - column: Column = Manager1.manager.name - self.assertTrue(len(column._meta.call_chain), 1) - self.assertTrue(isinstance(column, Varchar)) - - with self.assertRaises(Exception): - Manager1.manager.manager.manager.manager.manager.manager.manager.manager.manager.manager.manager.name # noqa - - def test_recursion_time(self): - """ - Make sure that a really large call chain doesn't take too long. - """ - start = time.time() - Manager1.manager.manager.manager.manager.manager.manager.name - end = time.time() - self.assertTrue(end - start < 1.0) - - -class TestAllColumns(TestCase): - def test_all_columns(self): - """ - Make sure you can retrieve all columns from a related table, without - explicitly specifying them. - """ - all_columns = Band.manager.all_columns() - self.assertEqual(all_columns, [Band.manager.id, Band.manager.name]) - - # Make sure the call chains are also correct. - self.assertEqual( - all_columns[0]._meta.call_chain, Band.manager.id._meta.call_chain - ) - self.assertEqual( - all_columns[1]._meta.call_chain, Band.manager.name._meta.call_chain - ) - - def test_all_columns_deep(self): - """ - Make sure ``all_columns`` works when the joins are several layers deep. - """ - all_columns = Concert.band_1.manager.all_columns() - self.assertEqual(all_columns, [Band.manager.id, Band.manager.name]) - - # Make sure the call chains are also correct. - self.assertEqual( - all_columns[0]._meta.call_chain, - Concert.band_1.manager.id._meta.call_chain, - ) - self.assertEqual( - all_columns[1]._meta.call_chain, - Concert.band_1.manager.name._meta.call_chain, - ) - - def test_all_columns_exclude(self): - """ - Make sure you can exclude some columns. - """ - self.assertEqual( - Band.manager.all_columns(exclude=["id"]), [Band.manager.name] - ) - - self.assertEqual( - Band.manager.all_columns(exclude=[Band.manager.id]), - [Band.manager.name], - ) - - -class TestAllRelated(TestCase): - def test_all_related(self): - """ - Make sure you can retrieve all foreign keys from a related table, - without explicitly specifying them. - """ - all_related = Ticket.concert.all_related() - - self.assertEqual( - all_related, - [ - Ticket.concert.band_1, - Ticket.concert.band_2, - Ticket.concert.venue, - ], - ) - - # Make sure the call chains are also correct. - self.assertEqual( - all_related[0]._meta.call_chain, - Ticket.concert.band_1._meta.call_chain, - ) - self.assertEqual( - all_related[1]._meta.call_chain, - Ticket.concert.band_2._meta.call_chain, - ) - self.assertEqual( - all_related[2]._meta.call_chain, - Ticket.concert.venue._meta.call_chain, - ) - - def test_all_related_deep(self): - """ - Make sure ``all_related`` works when the joins are several layers deep. - """ - all_related = Ticket.concert.band_1.all_related() - self.assertEqual(all_related, [Ticket.concert.band_1.manager]) - - # Make sure the call chains are also correct. - self.assertEqual( - all_related[0]._meta.call_chain, - Ticket.concert.band_1.manager._meta.call_chain, - ) - - def test_all_related_exclude(self): - """ - Make sure you can exclude some columns. - """ - self.assertEqual( - Ticket.concert.all_related(exclude=["venue"]), - [Ticket.concert.band_1, Ticket.concert.band_2], - ) - - self.assertEqual( - Ticket.concert.all_related(exclude=[Ticket.concert.venue]), - [Ticket.concert.band_1, Ticket.concert.band_2], - ) diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index b2aa6328c..d56c83c42 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -34,7 +34,7 @@ class Director(Table): class TestNumericColumn(TestCase): """ - Numeric and Decimal are the same - so we'll just Numeric. + Numeric and Decimal are the same - so we'll just test Numeric. """ def test_numeric_digits(self): @@ -96,6 +96,46 @@ class Band(Table): ) +class TestForeignKeyColumn(TestCase): + def test_target_column(self): + """ + Make sure the target_column is correctly set in the Pydantic schema. + """ + + class Manager(Table): + name = Varchar(unique=True) + + class BandA(Table): + manager = ForeignKey(Manager, target_column=Manager.name) + + class BandB(Table): + manager = ForeignKey(Manager, target_column="name") + + class BandC(Table): + manager = ForeignKey(Manager) + + self.assertEqual( + create_pydantic_model(table=BandA).schema()["properties"][ + "manager" + ]["extra"]["target_column"], + "name", + ) + + self.assertEqual( + create_pydantic_model(table=BandB).schema()["properties"][ + "manager" + ]["extra"]["target_column"], + "name", + ) + + self.assertEqual( + create_pydantic_model(table=BandC).schema()["properties"][ + "manager" + ]["extra"]["target_column"], + "id", + ) + + class TestTextColumn(TestCase): def test_text_format(self): class Band(Table): From 442f8988a5abb02572c3ba505d2ea8f583edbc54 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 16 Jan 2022 20:14:24 +0000 Subject: [PATCH 230/727] bumped version --- CHANGES.rst | 19 +++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 9d226319b..87e8dc5f8 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,25 @@ Changes ======= +0.64.0 +------ + +Added initial support for ``ForeignKey`` columns referencing non-primary key +columns. For example: + +.. code-block:: python + + class Manager(Table): + name = Varchar() + email = Varchar(unique=True) + + class Band(Table): + manager = ForeignKey(Manager, target_column=Manager.email) + +Thanks to @theelderbeever for suggesting this feature, and with help testing. + +------------------------------------------------------------------------------- + 0.63.1 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index c7737bbd8..afb4d13d6 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.63.1" +__VERSION__ = "0.64.0" From cd37314eae311231da7b0509d547b9f1f5e579dc Mon Sep 17 00:00:00 2001 From: Abdulwasiu Apalowo <64538336+mrbazzan@users.noreply.github.com> Date: Thu, 20 Jan 2022 21:17:41 +0100 Subject: [PATCH 231/727] Add from_fixture to BaseUser (#396) * .idea/ added to .gitignore * Add from_fixture method to BaseUser class(#265) --- .gitignore | 2 +- piccolo/apps/fixtures/commands/load.py | 2 +- piccolo/apps/user/tables.py | 12 ++++++++ piccolo/table.py | 4 +++ tests/apps/user/test_tables.py | 39 ++++++++++++++++++++++++++ tests/table/test_from_dict.py | 28 ++++++++++++++++++ 6 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 tests/table/test_from_dict.py diff --git a/.gitignore b/.gitignore index 5356d08b8..2734abca6 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,7 @@ build/ .doctrees/ .vscode/ piccolo.egg-info/ -build/ +.idea/ dist/ piccolo.sqlite _build/ diff --git a/piccolo/apps/fixtures/commands/load.py b/piccolo/apps/fixtures/commands/load.py index 22ab195ab..b9a21a858 100644 --- a/piccolo/apps/fixtures/commands/load.py +++ b/piccolo/apps/fixtures/commands/load.py @@ -56,7 +56,7 @@ async def load_json_string(json_string: str): await table_class.insert( *[ - table_class(**row.__dict__) + table_class.from_dict(row.__dict__) for row in model_instance_list ] ).run() diff --git a/piccolo/apps/user/tables.py b/piccolo/apps/user/tables.py index c582f16a4..d2ffe7a96 100644 --- a/piccolo/apps/user/tables.py +++ b/piccolo/apps/user/tables.py @@ -50,9 +50,21 @@ def __init__(self, **kwargs): # Generating passwords upfront is expensive, so might need reworking. password = kwargs.get("password", None) if password: + if password.startswith("pbkdf2_sha256"): + logger.warning("Hashed password passed to the constructor.") + raise ValueError("Do not pass hashed password.") + kwargs["password"] = self.__class__.hash_password(password) super().__init__(**kwargs) + @classmethod + def from_dict(cls, data: t.Dict[str, t.Any]) -> BaseUser: + """Create a BaseUser instance from fixture""" + password = data.get("password") + user = cls(**{k: v for k, v in data.items() if k != "password"}) + user.__setattr__("password", password) + return user + @classmethod def get_salt(cls): return secrets.token_hex(16) diff --git a/piccolo/table.py b/piccolo/table.py index 44746dfa8..f1666f592 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -310,6 +310,10 @@ def _create_serial_primary_key(cls) -> Serial: return pk + @classmethod + def from_dict(cls, data: t.Dict[str, t.Any]) -> Table: + return cls(**data) + ########################################################################### def save( diff --git a/tests/apps/user/test_tables.py b/tests/apps/user/test_tables.py index 492183de5..46b5df90f 100644 --- a/tests/apps/user/test_tables.py +++ b/tests/apps/user/test_tables.py @@ -42,6 +42,18 @@ def test_malicious_password(self): manager.exception.__str__(), "The password is too long." ) + @patch("piccolo.apps.user.tables.logger") + def test_hashed_password(self, logger: MagicMock): + with self.assertRaises(ValueError) as manager: + BaseUser(username="John", password="pbkdf2_sha256$10000") + self.assertEqual( + manager.exception.__str__(), "Do not pass hashed password." + ) + self.assertEqual( + logger.method_calls, + [call.warning("Hashed password passed to the constructor.")], + ) + class TestLogin(TestCase): def setUp(self): @@ -111,3 +123,30 @@ def test_update_password(self): manager.exception.__str__(), "The password is too long.", ) + + +class TestCreateUserFromFixture(TestCase): + def setUp(self): + BaseUser.create_table().run_sync() + + def tearDown(self): + BaseUser.alter().drop_table().run_sync() + + def test_create_user_from_fixture(self): + the_data = { + "id": 2, + "username": "", + "password": "pbkdf2_sha256$10000$19ed2c0d6cbe0868a70be6" + "446b93ed5b$c862974665ccc25b334ed42fa7e96a41" + "04d5ddff0c2e56e0e5b1d0efc67e9d03", + "first_name": "", + "last_name": "", + "email": "", + "active": False, + "admin": False, + "superuser": False, + "last_login": None, + } + user = BaseUser.from_dict(the_data) + self.assertIsInstance(user, BaseUser) + self.assertEqual(user.password, the_data["password"]) diff --git a/tests/table/test_from_dict.py b/tests/table/test_from_dict.py new file mode 100644 index 000000000..3c931f502 --- /dev/null +++ b/tests/table/test_from_dict.py @@ -0,0 +1,28 @@ +from unittest import TestCase + +from piccolo.columns import Varchar +from piccolo.table import Table + + +class BandMember(Table): + name = Varchar(length=50, index=True) + + +class TestCreateFromDict(TestCase): + def setUp(self): + BandMember.create_table().run_sync() + + def tearDown(self): + BandMember.alter().drop_table().run_sync() + + def test_create_table_from_dict(self): + BandMember.from_dict({"name": "John"}).save().run_sync() + self.assertEqual( + BandMember.select(BandMember.name).run_sync(), [{"name": "John"}] + ) + BandMember.from_dict({"name": "Town"}).save().run_sync() + self.assertEqual(BandMember.count().run_sync(), 2) + self.assertEqual( + BandMember.select(BandMember.name).run_sync(), + [{"name": "John"}, {"name": "Town"}], + ) From 849cf966143e1c98b35cff2b396a30f3f23c4f9d Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 20 Jan 2022 21:01:54 +0000 Subject: [PATCH 232/727] bumped version --- CHANGES.rst | 25 +++++++++++++++++++++++++ piccolo/__init__.py | 2 +- piccolo/apps/user/tables.py | 10 +++++++--- 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 87e8dc5f8..bee812962 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,31 @@ Changes ======= +0.65.0 +------ + +The ``BaseUser`` table hashes passwords before storing them in the database. + +When we create a fixture from the ``BaseUser`` table (using ``piccolo fixtures dump``), +it looks something like: + +.. code-block:: json + + { + "id": 11, + "username": "bob", + "password": "pbkdf2_sha256$10000$abc123", + } + +When we load the fixture (using ``piccolo fixtures load``) we need to be +careful in case ``BaseUser`` tries to hash it again (it would then be a hash of +a hash, and hence incorrect). We now have additional checks in place to prevent +this. + +Thanks to @mrbazzan for implementing this, and @sinisaos for help reviewing. + +------------------------------------------------------------------------------- + 0.64.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index afb4d13d6..dbdb8a2ea 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.64.0" +__VERSION__ = "0.65.0" diff --git a/piccolo/apps/user/tables.py b/piccolo/apps/user/tables.py index d2ffe7a96..1673db15d 100644 --- a/piccolo/apps/user/tables.py +++ b/piccolo/apps/user/tables.py @@ -61,9 +61,13 @@ def __init__(self, **kwargs): def from_dict(cls, data: t.Dict[str, t.Any]) -> BaseUser: """Create a BaseUser instance from fixture""" password = data.get("password") - user = cls(**{k: v for k, v in data.items() if k != "password"}) - user.__setattr__("password", password) - return user + if isinstance(password, str) and password.startswith("pbkdf2_sha256"): + # Password is already hashed + user = cls(**{k: v for k, v in data.items() if k != "password"}) + user.__setattr__("password", password) + return user + else: + return cls(**data) @classmethod def get_salt(cls): From 32654c1fbed91b5035dd318715033759ff94eec0 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 20 Jan 2022 21:03:54 +0000 Subject: [PATCH 233/727] fix typo --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index bee812962..57c339d2d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -18,7 +18,7 @@ it looks something like: } When we load the fixture (using ``piccolo fixtures load``) we need to be -careful in case ``BaseUser`` tries to hash it again (it would then be a hash of +careful in case ``BaseUser`` tries to hash the password again (it would then be a hash of a hash, and hence incorrect). We now have additional checks in place to prevent this. From 85be5173f672927838b2a9b6ac5777aaf1392111 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 21 Jan 2022 07:47:26 +0000 Subject: [PATCH 234/727] Fix for BaseUser and Piccolo API (#398) * don't raise value error * can use default from_dict now --- piccolo/apps/user/tables.py | 19 ++----------------- tests/apps/user/test_tables.py | 12 ------------ 2 files changed, 2 insertions(+), 29 deletions(-) diff --git a/piccolo/apps/user/tables.py b/piccolo/apps/user/tables.py index 1673db15d..8b647560f 100644 --- a/piccolo/apps/user/tables.py +++ b/piccolo/apps/user/tables.py @@ -50,25 +50,10 @@ def __init__(self, **kwargs): # Generating passwords upfront is expensive, so might need reworking. password = kwargs.get("password", None) if password: - if password.startswith("pbkdf2_sha256"): - logger.warning("Hashed password passed to the constructor.") - raise ValueError("Do not pass hashed password.") - - kwargs["password"] = self.__class__.hash_password(password) + if not password.startswith("pbkdf2_sha256"): + kwargs["password"] = self.__class__.hash_password(password) super().__init__(**kwargs) - @classmethod - def from_dict(cls, data: t.Dict[str, t.Any]) -> BaseUser: - """Create a BaseUser instance from fixture""" - password = data.get("password") - if isinstance(password, str) and password.startswith("pbkdf2_sha256"): - # Password is already hashed - user = cls(**{k: v for k, v in data.items() if k != "password"}) - user.__setattr__("password", password) - return user - else: - return cls(**data) - @classmethod def get_salt(cls): return secrets.token_hex(16) diff --git a/tests/apps/user/test_tables.py b/tests/apps/user/test_tables.py index 46b5df90f..1b97fb99b 100644 --- a/tests/apps/user/test_tables.py +++ b/tests/apps/user/test_tables.py @@ -42,18 +42,6 @@ def test_malicious_password(self): manager.exception.__str__(), "The password is too long." ) - @patch("piccolo.apps.user.tables.logger") - def test_hashed_password(self, logger: MagicMock): - with self.assertRaises(ValueError) as manager: - BaseUser(username="John", password="pbkdf2_sha256$10000") - self.assertEqual( - manager.exception.__str__(), "Do not pass hashed password." - ) - self.assertEqual( - logger.method_calls, - [call.warning("Hashed password passed to the constructor.")], - ) - class TestLogin(TestCase): def setUp(self): From ad842c116a032b859360ec78f0ea6017119ede61 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 21 Jan 2022 07:49:46 +0000 Subject: [PATCH 235/727] bumped version --- CHANGES.rst | 7 +++++++ piccolo/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 57c339d2d..69ca3c988 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,13 @@ Changes ======= +0.65.1 +------ + +Fixed bug with ``BaseUser`` and Piccolo API. + +------------------------------------------------------------------------------- + 0.65.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index dbdb8a2ea..3172f9e75 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.65.0" +__VERSION__ = "0.65.1" From c6d98b057b4310782f750df46b0ea0de393edfbd Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 21 Jan 2022 18:07:55 +0000 Subject: [PATCH 236/727] Better type inference using descriptors (#399) * prototype * fix some existing type warnings * add descriptor protocol for Varchar * add descriptors to each column type --- piccolo/apps/user/commands/create.py | 2 +- piccolo/apps/user/tables.py | 4 +- piccolo/columns/column_types.py | 489 ++++++++++++++++++++++++++- tests/apps/user/test_tables.py | 2 +- 4 files changed, 491 insertions(+), 6 deletions(-) diff --git a/piccolo/apps/user/commands/create.py b/piccolo/apps/user/commands/create.py index dabb0652a..174de888b 100644 --- a/piccolo/apps/user/commands/create.py +++ b/piccolo/apps/user/commands/create.py @@ -93,4 +93,4 @@ def create( ) user.save().run_sync() - print(f"Created User {user.id}") # type: ignore + print(f"Created User {user.id}") diff --git a/piccolo/apps/user/tables.py b/piccolo/apps/user/tables.py index 8b647560f..526c8c8dd 100644 --- a/piccolo/apps/user/tables.py +++ b/piccolo/apps/user/tables.py @@ -9,6 +9,7 @@ import typing as t from piccolo.columns import Boolean, Secret, Timestamp, Varchar +from piccolo.columns.column_types import Serial from piccolo.columns.readable import Readable from piccolo.table import Table from piccolo.utils.sync import run_sync @@ -21,6 +22,7 @@ class BaseUser(Table, tablename="piccolo_user"): Provides a basic user, with authentication support. """ + id: Serial username = Varchar(length=100, unique=True) password = Secret(length=255) first_name = Varchar(null=True) @@ -80,7 +82,7 @@ async def update_password(cls, user: t.Union[str, int], password: str): if isinstance(user, str): clause = cls.username == user elif isinstance(user, int): - clause = cls.id == user # type: ignore + clause = cls.id == user else: raise ValueError( "The `user` arg must be a user id, or a username." diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 73cdd3def..c933e95c6 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -1,3 +1,29 @@ +""" +Notes for devs +============== + +Descriptors +----------- + +Each column type implements the descriptor protocol (the ``__get__`` and +``__set__`` methods). + +This is to signal to MyPy that the following is allowed: + +.. code-block:: python + + class Band(Table): + name = Varchar() + + band = Band() + band.name = 'Pythonistas' # Without descriptors, this would be an error + +In the above example, descriptors allow us to tell MyPy that ``name`` is a +``Varchar`` when accessed on a class, but is a ``str`` when accessed on a class +instance. + +""" + from __future__ import annotations import copy @@ -198,6 +224,23 @@ def __radd__(self, value: t.Union[str, Varchar, Text]) -> QueryString: reverse=True, ) + ########################################################################### + # Descriptors + + @t.overload + def __get__(self, obj: Table, objtype=None) -> str: + ... + + @t.overload + def __get__(self, obj: None, objtype=None) -> Varchar: + ... + + def __get__(self, obj, objtype=None): + return obj.__dict__[self._meta.name] if obj else self + + def __set__(self, obj, value: t.Union[str, None]): + obj.__dict__[self._meta.name] = value + class Secret(Varchar): """ @@ -209,6 +252,23 @@ def __init__(self, *args, **kwargs): kwargs["secret"] = True super().__init__(*args, **kwargs) + ########################################################################### + # Descriptors + + @t.overload + def __get__(self, obj: Table, objtype=None) -> str: + ... + + @t.overload + def __get__(self, obj: None, objtype=None) -> Secret: + ... + + def __get__(self, obj, objtype=None): + return obj.__dict__[self._meta.name] if obj else self + + def __set__(self, obj, value: t.Union[str, None]): + obj.__dict__[self._meta.name] = value + class Text(Column): """ @@ -261,6 +321,23 @@ def __radd__(self, value: t.Union[str, Varchar, Text]) -> QueryString: reverse=True, ) + ########################################################################### + # Descriptors + + @t.overload + def __get__(self, obj: Table, objtype=None) -> str: + ... + + @t.overload + def __get__(self, obj: None, objtype=None) -> Text: + ... + + def __get__(self, obj, objtype=None): + return obj.__dict__[self._meta.name] if obj else self + + def __set__(self, obj, value: t.Union[str, None]): + obj.__dict__[self._meta.name] = value + class UUID(Column): """ @@ -309,6 +386,23 @@ def __init__(self, default: UUIDArg = UUID4(), **kwargs) -> None: kwargs.update({"default": default}) super().__init__(**kwargs) + ########################################################################### + # Descriptors + + @t.overload + def __get__(self, obj: Table, objtype=None) -> uuid.UUID: + ... + + @t.overload + def __get__(self, obj: None, objtype=None) -> UUID: + ... + + def __get__(self, obj, objtype=None): + return obj.__dict__[self._meta.name] if obj else self + + def __set__(self, obj, value: t.Union[uuid.UUID, None]): + obj.__dict__[self._meta.name] = value + class Integer(Column): """ @@ -409,6 +503,23 @@ def __rfloordiv__( reverse=True, ) + ########################################################################### + # Descriptors + + @t.overload + def __get__(self, obj: Table, objtype=None) -> int: + ... + + @t.overload + def __get__(self, obj: None, objtype=None) -> Integer: + ... + + def __get__(self, obj, objtype=None): + return obj.__dict__[self._meta.name] if obj else self + + def __set__(self, obj, value: t.Union[int, None]): + obj.__dict__[self._meta.name] = value + ############################################################################### # BigInt and SmallInt only exist on Postgres. SQLite treats them the same as @@ -446,6 +557,23 @@ def column_type(self): return "INTEGER" raise Exception("Unrecognized engine type") + ########################################################################### + # Descriptors + + @t.overload + def __get__(self, obj: Table, objtype=None) -> int: + ... + + @t.overload + def __get__(self, obj: None, objtype=None) -> BigInt: + ... + + def __get__(self, obj, objtype=None): + return obj.__dict__[self._meta.name] if obj else self + + def __set__(self, obj, value: t.Union[int, None]): + obj.__dict__[self._meta.name] = value + class SmallInt(Integer): """ @@ -477,6 +605,23 @@ def column_type(self): return "INTEGER" raise Exception("Unrecognized engine type") + ########################################################################### + # Descriptors + + @t.overload + def __get__(self, obj: Table, objtype=None) -> int: + ... + + @t.overload + def __get__(self, obj: None, objtype=None) -> SmallInt: + ... + + def __get__(self, obj, objtype=None): + return obj.__dict__[self._meta.name] if obj else self + + def __set__(self, obj, value: t.Union[int, None]): + obj.__dict__[self._meta.name] = value + ############################################################################### @@ -507,6 +652,23 @@ def default(self): return NULL raise Exception("Unrecognized engine type") + ########################################################################### + # Descriptors + + @t.overload + def __get__(self, obj: Table, objtype=None) -> int: + ... + + @t.overload + def __get__(self, obj: None, objtype=None) -> Serial: + ... + + def __get__(self, obj, objtype=None): + return obj.__dict__[self._meta.name] if obj else self + + def __set__(self, obj, value: t.Union[int, None]): + obj.__dict__[self._meta.name] = value + class BigSerial(Serial): """ @@ -522,6 +684,23 @@ def column_type(self): return "INTEGER" raise Exception("Unrecognized engine type") + ########################################################################### + # Descriptors + + @t.overload + def __get__(self, obj: Table, objtype=None) -> int: + ... + + @t.overload + def __get__(self, obj: None, objtype=None) -> BigSerial: + ... + + def __get__(self, obj, objtype=None): + return obj.__dict__[self._meta.name] if obj else self + + def __set__(self, obj, value: t.Union[int, None]): + obj.__dict__[self._meta.name] = value + class PrimaryKey(Serial): def __init__(self, **kwargs) -> None: @@ -540,6 +719,23 @@ def __init__(self, **kwargs) -> None: super().__init__(**kwargs) + ########################################################################### + # Descriptors + + @t.overload + def __get__(self, obj: Table, objtype=None) -> int: + ... + + @t.overload + def __get__(self, obj: None, objtype=None) -> PrimaryKey: + ... + + def __get__(self, obj, objtype=None): + return obj.__dict__[self._meta.name] if obj else self + + def __set__(self, obj, value: t.Union[int, None]): + obj.__dict__[self._meta.name] = value + ############################################################################### @@ -590,6 +786,23 @@ def __init__( kwargs.update({"default": default}) super().__init__(**kwargs) + ########################################################################### + # Descriptors + + @t.overload + def __get__(self, obj: Table, objtype=None) -> datetime: + ... + + @t.overload + def __get__(self, obj: None, objtype=None) -> Timestamp: + ... + + def __get__(self, obj, objtype=None): + return obj.__dict__[self._meta.name] if obj else self + + def __set__(self, obj, value: t.Union[datetime, None]): + obj.__dict__[self._meta.name] = value + class Timestamptz(Column): """ @@ -642,6 +855,23 @@ def __init__( kwargs.update({"default": default}) super().__init__(**kwargs) + ########################################################################### + # Descriptors + + @t.overload + def __get__(self, obj: Table, objtype=None) -> datetime: + ... + + @t.overload + def __get__(self, obj: None, objtype=None) -> Timestamptz: + ... + + def __get__(self, obj, objtype=None): + return obj.__dict__[self._meta.name] if obj else self + + def __set__(self, obj, value: t.Union[datetime, None]): + obj.__dict__[self._meta.name] = value + class Date(Column): """ @@ -682,6 +912,23 @@ def __init__(self, default: DateArg = DateNow(), **kwargs) -> None: kwargs.update({"default": default}) super().__init__(**kwargs) + ########################################################################### + # Descriptors + + @t.overload + def __get__(self, obj: Table, objtype=None) -> date: + ... + + @t.overload + def __get__(self, obj: None, objtype=None) -> Date: + ... + + def __get__(self, obj, objtype=None): + return obj.__dict__[self._meta.name] if obj else self + + def __set__(self, obj, value: t.Union[date, None]): + obj.__dict__[self._meta.name] = value + class Time(Column): """ @@ -719,6 +966,23 @@ def __init__(self, default: TimeArg = TimeNow(), **kwargs) -> None: kwargs.update({"default": default}) super().__init__(**kwargs) + ########################################################################### + # Descriptors + + @t.overload + def __get__(self, obj: Table, objtype=None) -> time: + ... + + @t.overload + def __get__(self, obj: None, objtype=None) -> Time: + ... + + def __get__(self, obj, objtype=None): + return obj.__dict__[self._meta.name] if obj else self + + def __set__(self, obj, value: t.Union[time, None]): + obj.__dict__[self._meta.name] = value + class Interval(Column): # lgtm [py/missing-equals] """ @@ -770,6 +1034,23 @@ def column_type(self): return "SECONDS" raise Exception("Unrecognized engine type") + ########################################################################### + # Descriptors + + @t.overload + def __get__(self, obj: Table, objtype=None) -> timedelta: + ... + + @t.overload + def __get__(self, obj: None, objtype=None) -> Interval: + ... + + def __get__(self, obj, objtype=None): + return obj.__dict__[self._meta.name] if obj else self + + def __set__(self, obj, value: t.Union[timedelta, None]): + obj.__dict__[self._meta.name] = value + ############################################################################### @@ -846,6 +1127,23 @@ def ne(self, value) -> Where: """ return self.__ne__(value) + ########################################################################### + # Descriptors + + @t.overload + def __get__(self, obj: Table, objtype=None) -> bool: + ... + + @t.overload + def __get__(self, obj: None, objtype=None) -> Boolean: + ... + + def __get__(self, obj, objtype=None): + return obj.__dict__[self._meta.name] if obj else self + + def __set__(self, obj, value: t.Union[bool, None]): + obj.__dict__[self._meta.name] = value + ############################################################################### @@ -930,13 +1228,45 @@ def __init__( kwargs.update({"default": default, "digits": digits}) super().__init__(**kwargs) + ########################################################################### + # Descriptors + + @t.overload + def __get__(self, obj: Table, objtype=None) -> decimal.Decimal: + ... + + @t.overload + def __get__(self, obj: None, objtype=None) -> Numeric: + ... + + def __get__(self, obj, objtype=None): + return obj.__dict__[self._meta.name] if obj else self + + def __set__(self, obj, value: t.Union[decimal.Decimal, None]): + obj.__dict__[self._meta.name] = value + class Decimal(Numeric): """ An alias for Numeric. """ - pass + ########################################################################### + # Descriptors + + @t.overload + def __get__(self, obj: Table, objtype=None) -> decimal.Decimal: + ... + + @t.overload + def __get__(self, obj: None, objtype=None) -> Decimal: + ... + + def __get__(self, obj, objtype=None): + return obj.__dict__[self._meta.name] if obj else self + + def __set__(self, obj, value: t.Union[decimal.Decimal, None]): + obj.__dict__[self._meta.name] = value class Real(Column): @@ -972,13 +1302,45 @@ def __init__( kwargs.update({"default": default}) super().__init__(**kwargs) + ########################################################################### + # Descriptors + + @t.overload + def __get__(self, obj: Table, objtype=None) -> float: + ... + + @t.overload + def __get__(self, obj: None, objtype=None) -> Real: + ... + + def __get__(self, obj, objtype=None): + return obj.__dict__[self._meta.name] if obj else self + + def __set__(self, obj, value: t.Union[float, None]): + obj.__dict__[self._meta.name] = value + class Float(Real): """ An alias for Real. """ - pass + ########################################################################### + # Descriptors + + @t.overload + def __get__(self, obj: Table, objtype=None) -> float: + ... + + @t.overload + def __get__(self, obj: None, objtype=None) -> Float: + ... + + def __get__(self, obj, objtype=None): + return obj.__dict__[self._meta.name] if obj else self + + def __set__(self, obj, value: t.Union[float, None]): + obj.__dict__[self._meta.name] = value class DoublePrecision(Real): @@ -990,6 +1352,23 @@ class DoublePrecision(Real): def column_type(self): return "DOUBLE PRECISION" + ########################################################################### + # Descriptors + + @t.overload + def __get__(self, obj: Table, objtype=None) -> float: + ... + + @t.overload + def __get__(self, obj: None, objtype=None) -> DoublePrecision: + ... + + def __get__(self, obj, objtype=None): + return obj.__dict__[self._meta.name] if obj else self + + def __set__(self, obj, value: t.Union[float, None]): + obj.__dict__[self._meta.name] = value + ############################################################################### @@ -1511,6 +1890,27 @@ def __getattribute__(self, name: str): else: return value + ########################################################################### + # Descriptors + + @t.overload + def __get__(self, obj: Table, objtype=None) -> t.Any: + ... + + @t.overload + def __get__(self, obj: None, objtype=None) -> ForeignKey: + ... + + @t.overload + def __get__(self, obj: t.Any, objtype=None) -> t.Any: + ... + + def __get__(self, obj, objtype=None): + return obj.__dict__[self._meta.name] if obj else self + + def __set__(self, obj, value: t.Any): + obj.__dict__[self._meta.name] = value + ############################################################################### @@ -1551,6 +1951,23 @@ def __init__( self.json_operator: t.Optional[str] = None + ########################################################################### + # Descriptors + + @t.overload + def __get__(self, obj: Table, objtype=None) -> str: + ... + + @t.overload + def __get__(self, obj: None, objtype=None) -> JSON: + ... + + def __get__(self, obj, objtype=None): + return obj.__dict__[self._meta.name] if obj else self + + def __set__(self, obj, value: t.Union[str, t.Dict]): + obj.__dict__[self._meta.name] = value + class JSONB(JSON): """ @@ -1586,6 +2003,23 @@ def get_select_string(self, engine_type: str, just_alias=False) -> str: else: return f"{select_string} {self.json_operator} AS {self.alias}" + ########################################################################### + # Descriptors + + @t.overload + def __get__(self, obj: Table, objtype=None) -> str: + ... + + @t.overload + def __get__(self, obj: None, objtype=None) -> JSONB: + ... + + def __get__(self, obj, objtype=None): + return obj.__dict__[self._meta.name] if obj else self + + def __set__(self, obj, value: t.Union[str, t.Dict]): + obj.__dict__[self._meta.name] = value + ############################################################################### @@ -1642,13 +2076,45 @@ def __init__( kwargs.update({"default": default}) super().__init__(**kwargs) + ########################################################################### + # Descriptors + + @t.overload + def __get__(self, obj: Table, objtype=None) -> bytes: + ... + + @t.overload + def __get__(self, obj: None, objtype=None) -> Bytea: + ... + + def __get__(self, obj, objtype=None): + return obj.__dict__[self._meta.name] if obj else self + + def __set__(self, obj, value: bytes): + obj.__dict__[self._meta.name] = value + class Blob(Bytea): """ An alias for Bytea. """ - pass + ########################################################################### + # Descriptors + + @t.overload + def __get__(self, obj: Table, objtype=None) -> bytes: + ... + + @t.overload + def __get__(self, obj: None, objtype=None) -> Blob: + ... + + def __get__(self, obj, objtype=None): + return obj.__dict__[self._meta.name] if obj else self + + def __set__(self, obj, value: bytes): + obj.__dict__[self._meta.name] = value ############################################################################### @@ -1789,3 +2255,20 @@ def all(self, value: t.Any) -> Where: raise ValueError("Unsupported by SQLite") else: raise ValueError("Unrecognised engine type") + + ########################################################################### + # Descriptors + + @t.overload + def __get__(self, obj: Table, objtype=None) -> t.List[t.Any]: + ... + + @t.overload + def __get__(self, obj: None, objtype=None) -> Array: + ... + + def __get__(self, obj, objtype=None): + return obj.__dict__[self._meta.name] if obj else self + + def __set__(self, obj, value: t.List[t.Any]): + obj.__dict__[self._meta.name] = value diff --git a/tests/apps/user/test_tables.py b/tests/apps/user/test_tables.py index 1b97fb99b..578786faa 100644 --- a/tests/apps/user/test_tables.py +++ b/tests/apps/user/test_tables.py @@ -61,7 +61,7 @@ def test_login(self, logger: MagicMock): # Test correct password authenticated = BaseUser.login_sync(username, password) - self.assertTrue(authenticated == user.id) # type: ignore + self.assertTrue(authenticated == user.id) # Test incorrect password authenticated = BaseUser.login_sync(username, "blablabla") From 2c316b04ce0fb5aea3af55a25f03156cf4837d4d Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 21 Jan 2022 18:14:52 +0000 Subject: [PATCH 237/727] bumped version --- CHANGES.rst | 23 +++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 69ca3c988..c23d29695 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,29 @@ Changes ======= +0.66.0 +------ + +Using descriptors to improve MyPy support (`PR 399 `_). + +MyPy is now able to correctly infer the type in lots of different scenarios: + +.. code-block:: python + + class Band(Table): + name = Varchar() + + # MyPy knows this is a Varchar + Band.name + + band = Band() + band.name = "Pythonistas" # MyPy knows we can assign strings when it's a class instance + band.name # MyPy knows we will get a string back + + band.name = 1 # MyPy knows this is an error, as we should only be allowed to assign strings + +------------------------------------------------------------------------------- + 0.65.1 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 3172f9e75..6e65089fe 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.65.1" +__VERSION__ = "0.66.0" From 1e384ddd4401fc8dac49506e109001a1a33f17e7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Jan 2022 20:08:53 +0000 Subject: [PATCH 238/727] Bump ipython from 7.30.1 to 7.31.1 in /requirements (#401) Bumps [ipython](https://github.com/ipython/ipython) from 7.30.1 to 7.31.1. - [Release notes](https://github.com/ipython/ipython/releases) - [Commits](https://github.com/ipython/ipython/compare/7.30.1...7.31.1) --- updated-dependencies: - dependency-name: ipython dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt index b91536419..281a72943 100644 --- a/requirements/dev-requirements.txt +++ b/requirements/dev-requirements.txt @@ -1,6 +1,6 @@ black>=21.7b0 ipdb==0.13.9 -ipython==7.30.1 +ipython==7.31.1 flake8==4.0.1 isort==5.10.1 twine==3.7.1 From 439afcdc8bba1e0173add98922348988a766defd Mon Sep 17 00:00:00 2001 From: Taylor Beever Date: Tue, 25 Jan 2022 15:30:17 -0700 Subject: [PATCH 239/727] Add UUID type conversion and fix datetime comparison (#405) * Add UUID type conversion and fix datetime comparison * Import sorting and black for linter * Allow multiple UUID types to be checked. * isort --- piccolo/querystring.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/piccolo/querystring.py b/piccolo/querystring.py index 2d68dca70..eeccbf5ad 100644 --- a/piccolo/querystring.py +++ b/piccolo/querystring.py @@ -1,13 +1,21 @@ from __future__ import annotations -import datetime import typing as t from dataclasses import dataclass +from datetime import datetime +from importlib.util import find_spec from string import Formatter if t.TYPE_CHECKING: from piccolo.table import Table +from uuid import UUID + +if find_spec("asyncpg"): + from asyncpg.pgproto.pgproto import UUID as apgUUID +else: + apgUUID = UUID + @dataclass class Unquoted: @@ -91,8 +99,10 @@ def __str__(self): if _type == str: converted_args.append(f"'{arg}'") elif _type == datetime: - dt_string = arg.isoformat().replace("T", " ") + dt_string = arg.isoformat() converted_args.append(f"'{dt_string}'") + elif _type == UUID or _type == apgUUID: + converted_args.append(f"'{arg}'") elif arg is None: converted_args.append("null") else: From 65586a79d6bf8be6d4e9cd2ab64c9c9d906921b3 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 25 Jan 2022 22:42:09 +0000 Subject: [PATCH 240/727] bumped version --- CHANGES.rst | 15 +++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index c23d29695..cd018aaf0 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,21 @@ Changes ======= +0.66.1 +------ + +In Piccolo you can print out any query to see the SQL which will be generated: + +.. code-block:: python + + >>> print(Band.select()) + SELECT "band"."id", "band"."name", "band"."manager", "band"."popularity" FROM band + +It didn't represent ``UUID`` and ``datetime`` values correctly, which is now fixed (courtesy @theelderbeever). +See `PR 405 `_. + +------------------------------------------------------------------------------- + 0.66.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 6e65089fe..bdba0ec49 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.66.0" +__VERSION__ = "0.66.1" From 2ac947f36c9e90c493378449442a51ba73ec03ba Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 27 Jan 2022 11:50:22 +0000 Subject: [PATCH 241/727] Added `create_user` method to `BaseUser` (#402) * added BaseUser.create_user * added `_min_password_length` variable * make sure already hashed password is rejected * more tests --- docs/src/piccolo/authentication/baseuser.rst | 33 ++++++++- piccolo/apps/user/commands/create.py | 6 +- piccolo/apps/user/tables.py | 73 +++++++++++++++++++- piccolo/table.py | 5 ++ tests/apps/user/test_tables.py | 69 ++++++++++++++++++ 5 files changed, 176 insertions(+), 10 deletions(-) diff --git a/docs/src/piccolo/authentication/baseuser.rst b/docs/src/piccolo/authentication/baseuser.rst index 68183e196..11b0fd3af 100644 --- a/docs/src/piccolo/authentication/baseuser.rst +++ b/docs/src/piccolo/authentication/baseuser.rst @@ -76,8 +76,27 @@ can login and what they can do. Within your code ---------------- -login -~~~~~ +create_user / create_user_sync +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To create a new user: + +.. code-block:: python + + # From within a coroutine: + await BaseUser.create_user(username="bob", password="abc123", active=True) + + # When not in an event loop: + BaseUser.create_user_sync(username="bob", password="abc123", active=True) + +It saves the user in the database, and returns the created ``BaseUser`` +instance. + +.. note:: It is preferable to use this rather than instantiating and saving + ``BaseUser`` directly, as we add additional validation. + +login / login_sync +~~~~~~~~~~~~~~~~~~ To check a user's credentials, do the following: @@ -119,3 +138,13 @@ Limits The maximum password length allowed is 128 characters. This should be sufficiently long for most use cases. + +------------------------------------------------------------------------------- + +Source +------ + +.. currentmodule:: piccolo.apps.user.tables + +.. autoclass:: BaseUser + :members: create_user, create_user_sync, login, login_sync, update_password, update_password_sync diff --git a/piccolo/apps/user/commands/create.py b/piccolo/apps/user/commands/create.py index 174de888b..caec89c5a 100644 --- a/piccolo/apps/user/commands/create.py +++ b/piccolo/apps/user/commands/create.py @@ -76,14 +76,11 @@ def create( if password != confirmed_password: sys.exit("Passwords don't match!") - if len(password) < 4: - sys.exit("The password is too short") - is_admin = get_is_admin() if is_admin is None else is_admin is_superuser = get_is_superuser() if is_superuser is None else is_superuser is_active = get_is_active() if is_active is None else is_active - user = BaseUser( + user = BaseUser.create_user_sync( username=username, password=password, admin=is_admin, @@ -91,6 +88,5 @@ def create( active=is_active, superuser=is_superuser, ) - user.save().run_sync() print(f"Created User {user.id}") diff --git a/piccolo/apps/user/tables.py b/piccolo/apps/user/tables.py index 526c8c8dd..7fc084c91 100644 --- a/piccolo/apps/user/tables.py +++ b/piccolo/apps/user/tables.py @@ -3,6 +3,7 @@ """ from __future__ import annotations +import datetime import hashlib import logging import secrets @@ -46,6 +47,7 @@ class BaseUser(Table, tablename="piccolo_user"): help_text="When this user last logged in.", ) + _min_password_length = 6 _max_password_length = 128 def __init__(self, **kwargs): @@ -71,12 +73,15 @@ def get_readable(cls) -> Readable: @classmethod def update_password_sync(cls, user: t.Union[str, int], password: str): + """ + A sync equivalent of ``update_password``. + """ return run_sync(cls.update_password(user, password)) @classmethod async def update_password(cls, user: t.Union[str, int], password: str): """ - The password is the raw password string e.g. password123. + The password is the raw password string e.g. ``'password123'``. The user can be a user ID, or a username. """ if isinstance(user, str): @@ -135,17 +140,24 @@ def split_stored_password(cls, password: str) -> t.List[str]: raise ValueError("Unable to split hashed password") return elements + ########################################################################### + @classmethod def login_sync(cls, username: str, password: str) -> t.Optional[int]: """ - Returns the user_id if a match is found. + A sync equivalent of ``login``. """ return run_sync(cls.login(username, password)) @classmethod async def login(cls, username: str, password: str) -> t.Optional[int]: """ - Returns the user_id if a match is found. + Make sure the user exists and the password is valid. If so, the + ``last_login`` value is updated in the database. + + :returns: + The id of the user if a match is found, otherwise ``None``. + """ if len(username) > cls.username.length: logger.warning("Excessively long username provided.") @@ -175,6 +187,61 @@ async def login(cls, username: str, password: str) -> t.Optional[int]: cls.hash_password(password, salt, int(iterations)) == stored_password ): + await cls.update({cls.last_login: datetime.datetime.now()}).where( + cls.username == username + ) return response["id"] else: return None + + ########################################################################### + + @classmethod + def create_user_sync( + cls, username: str, password: str, **extra_params + ) -> BaseUser: + """ + A sync equivalent of ``create_user``. + """ + return run_sync( + cls.create_user( + username=username, password=password, **extra_params + ) + ) + + @classmethod + async def create_user( + cls, username: str, password: str, **extra_params + ) -> BaseUser: + """ + Creates a new user, and saves it in the database. It is recommended to + use this rather than instantiating and saving ``BaseUser`` directly, as + we add extra validation. + + :raises ValueError: + If the username or password is invalid. + :returns: + The created ``BaseUser`` instance. + + """ + if not username: + raise ValueError("A username must be provided.") + + if not password: + raise ValueError("A password must be provided.") + + if len(password) < cls._min_password_length: + raise ValueError("The password is too short.") + + if len(password) > cls._max_password_length: + raise ValueError("The password is too long.") + + if password.startswith("pbkdf2_sha256"): + logger.warning( + "Tried to create a user with an already hashed password." + ) + raise ValueError("Do not pass a hashed password.") + + user = cls(username=username, password=password, **extra_params) + await user.save() + return user diff --git a/piccolo/table.py b/piccolo/table.py index f1666f592..4edef51d2 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -312,6 +312,11 @@ def _create_serial_primary_key(cls) -> Serial: @classmethod def from_dict(cls, data: t.Dict[str, t.Any]) -> Table: + """ + Used when loading fixtures. It can be overriden by subclasses in case + they have specific logic / validation which needs running when loading + fixtures. + """ return cls(**data) ########################################################################### diff --git a/tests/apps/user/test_tables.py b/tests/apps/user/test_tables.py index 578786faa..ac4926466 100644 --- a/tests/apps/user/test_tables.py +++ b/tests/apps/user/test_tables.py @@ -138,3 +138,72 @@ def test_create_user_from_fixture(self): user = BaseUser.from_dict(the_data) self.assertIsInstance(user, BaseUser) self.assertEqual(user.password, the_data["password"]) + + +class TestCreateUser(TestCase): + def setUp(self): + BaseUser.create_table().run_sync() + + def tearDown(self): + BaseUser.alter().drop_table().run_sync() + + def test_success(self): + user = BaseUser.create_user_sync(username="bob", password="abc123") + self.assertTrue(isinstance(user, BaseUser)) + self.assertEqual( + BaseUser.login_sync(username="bob", password="abc123"), user.id + ) + + @patch("piccolo.apps.user.tables.logger") + def test_hashed_password_error(self, logger: MagicMock): + with self.assertRaises(ValueError) as manager: + BaseUser.create_user_sync( + username="bob", password="pbkdf2_sha256$10000" + ) + + self.assertEqual( + manager.exception.__str__(), "Do not pass a hashed password." + ) + self.assertEqual( + logger.method_calls, + [ + call.warning( + "Tried to create a user with an already hashed password." + ) + ], + ) + + def test_short_password_error(self): + with self.assertRaises(ValueError) as manager: + BaseUser.create_user_sync(username="bob", password="abc") + + self.assertEqual( + manager.exception.__str__(), "The password is too short." + ) + + def test_long_password_error(self): + with self.assertRaises(ValueError) as manager: + BaseUser.create_user_sync( + username="bob", + password="x" * (BaseUser._max_password_length + 1), + ) + + self.assertEqual( + manager.exception.__str__(), "The password is too long." + ) + + def test_no_username_error(self): + with self.assertRaises(ValueError) as manager: + BaseUser.create_user_sync(username=None, password="abc123") + + self.assertEqual( + manager.exception.__str__(), "A username must be provided." + ) + + def test_no_password_error(self): + with self.assertRaises(ValueError) as manager: + BaseUser.create_user_sync(username="bob", password=None) + + self.assertEqual( + manager.exception.__str__(), "A password must be provided." + ) From 7300175ad3451e2d285e4c755fe4f56c60819713 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 28 Jan 2022 09:30:52 +0000 Subject: [PATCH 242/727] make docs async first (#407) --- README.md | 16 +-- docs/src/piccolo/getting_started/index.rst | 2 +- .../piccolo/getting_started/playground.rst | 8 +- .../getting_started/sync_and_async.rst | 75 ++++------- docs/src/piccolo/query_clauses/batch.rst | 4 - docs/src/piccolo/query_clauses/distinct.rst | 2 +- docs/src/piccolo/query_clauses/first.rst | 6 +- docs/src/piccolo/query_clauses/group_by.rst | 4 +- docs/src/piccolo/query_clauses/limit.rst | 4 +- docs/src/piccolo/query_clauses/offset.rst | 4 +- docs/src/piccolo/query_clauses/order_by.rst | 12 +- docs/src/piccolo/query_clauses/output.rst | 12 +- docs/src/piccolo/query_clauses/where.rst | 96 +++++++------- docs/src/piccolo/query_types/alter.rst | 16 +-- docs/src/piccolo/query_types/create_table.rst | 10 +- docs/src/piccolo/query_types/delete.rst | 6 +- docs/src/piccolo/query_types/exists.rst | 2 +- docs/src/piccolo/query_types/insert.rst | 10 +- docs/src/piccolo/query_types/objects.rst | 82 ++++++------ docs/src/piccolo/query_types/raw.rst | 7 +- docs/src/piccolo/query_types/select.rst | 79 ++++++------ docs/src/piccolo/query_types/update.rst | 36 +++--- docs/src/piccolo/schema/advanced.rst | 14 +- docs/src/piccolo/schema/column_types.rst | 14 +- docs/src/piccolo/serialization/index.rst | 32 ++--- piccolo/columns/base.py | 6 +- piccolo/columns/column_types.py | 120 +++++++++--------- piccolo/columns/combination.py | 10 +- piccolo/conf/apps.py | 5 +- piccolo/query/base.py | 2 +- piccolo/query/methods/select.py | 10 +- piccolo/table.py | 4 +- piccolo/utils/dictionary.py | 2 +- 33 files changed, 338 insertions(+), 374 deletions(-) diff --git a/README.md b/README.md index 50d8560d4..6433ea0ac 100644 --- a/README.md +++ b/README.md @@ -33,23 +33,23 @@ await Band.select( Band.name ).where( Band.popularity > 100 -).run() +) # Join: await Band.select( Band.name, Band.manager.name -).run() +) # Delete: await Band.delete().where( Band.popularity < 1000 -).run() +) # Update: await Band.update({Band.popularity: 10000}).where( Band.name == 'Pythonistas' -).run() +) ``` Or like a typical ORM: @@ -57,15 +57,15 @@ Or like a typical ORM: ```python # To create a new object: b = Band(name='C-Sharps', popularity=100) -await b.save().run() +await b.save() # To fetch an object from the database, and update it: -b = await Band.objects().get(Band.name == 'Pythonistas').run() +b = await Band.objects().get(Band.name == 'Pythonistas') b.popularity = 10000 -await b.save().run() +await b.save() # To delete: -await b.remove().run() +await b.remove() ``` ## Installation diff --git a/docs/src/piccolo/getting_started/index.rst b/docs/src/piccolo/getting_started/index.rst index 9403952d4..c53234319 100644 --- a/docs/src/piccolo/getting_started/index.rst +++ b/docs/src/piccolo/getting_started/index.rst @@ -10,5 +10,5 @@ Getting Started ./installing_piccolo ./playground ./setup_postgres - ./sync_and_async ./example_schema + ./sync_and_async diff --git a/docs/src/piccolo/getting_started/playground.rst b/docs/src/piccolo/getting_started/playground.rst index 20ca54414..971f6e354 100644 --- a/docs/src/piccolo/getting_started/playground.rst +++ b/docs/src/piccolo/getting_started/playground.rst @@ -52,10 +52,10 @@ Give these queries a go: .. code-block:: python - Band.select().run_sync() - Band.objects().run_sync() - Band.select(Band.name).run_sync() - Band.select(Band.name, Band.manager.name).run_sync() + await Band.select() + await Band.objects() + await Band.select(Band.name) + await Band.select(Band.name, Band.manager.name) ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/getting_started/sync_and_async.rst b/docs/src/piccolo/getting_started/sync_and_async.rst index bc28ac14a..a1f981cc5 100644 --- a/docs/src/piccolo/getting_started/sync_and_async.rst +++ b/docs/src/piccolo/getting_started/sync_and_async.rst @@ -3,81 +3,58 @@ Sync and Async ============== -One of the main motivations for making Piccolo was the lack of options for -ORMs which support asyncio. +One of the motivations for making Piccolo was the lack of ORMs and query +builders which support asyncio. -However, you can use Piccolo in synchronous apps as well, whether that be a -WSGI web app, or a data science script. - -------------------------------------------------------------------------------- - -Sync example ------------- - -.. code-block:: python - - from my_schema import Band - - - def main(): - print(Band.select().run_sync()) - - - if __name__ == '__main__': - main() +Piccolo is designed to be async first. However, you can use Piccolo in +synchronous apps as well, whether that be a WSGI web app, or a data science +script. ------------------------------------------------------------------------------- Async example ------------- -.. code-block:: python - - import asyncio - from my_schema import Band - +You can await a query to run it: - async def main(): - print(await Band.select().run()) - - - if __name__ == '__main__': - asyncio.run(main()) +.. code-block:: python -Direct await -~~~~~~~~~~~~ + >>> await Band.select(Band.name) + [{'name': 'Pythonistas'}] -You can directly await a query if you prefer. For example: +If you need more control over how the query is executed, you can await the +``run`` method of a query: .. code-block:: python - >>> await Band.select() - [{'id': 1, 'name': 'Pythonistas', 'manager': 1, 'popularity': 1000}, - {'id': 2, 'name': 'Rustaceans', 'manager': 2, 'popularity': 500}] + >>> await Band.select(Band.name).run(in_pool=False) + [{'name': 'Pythonistas'}] -By convention, we await the ``run`` method (``await Band.select().run()``), but -you can use this shorter form if you prefer. + +Using the async version is useful for applications which require high +throughput. Piccolo makes building an ASGI web app really simple - see +:ref:`ASGICommand`. ------------------------------------------------------------------------------- -Which to use? -------------- +Sync example +------------ + +This lets you execute a query in an application which isn't using asyncio: -A lot of the time, using the sync version works perfectly fine. Many of the -examples use the sync version. +.. code-block:: python -Using the async version is useful for web applications which require high -throughput, based on `ASGI frameworks `_. -Piccolo makes building an ASGI web app really simple - see :ref:`ASGICommand`. + >>> Band.select(Band.name).run_sync() + [{'name': 'Pythonistas'}] ------------------------------------------------------------------------------- Explicit -------- -By using ``run`` and ``run_sync``, it makes it very explicit when a query is +By using ``await`` and ``run_sync``, it makes it very explicit when a query is actually being executed. -Until you execute one of those methods, you can chain as many methods onto your +Until you execute ``await`` or ``run_sync``, you can chain as many methods onto your query as you like, safe in the knowledge that no database queries are being made. diff --git a/docs/src/piccolo/query_clauses/batch.rst b/docs/src/piccolo/query_clauses/batch.rst index 2fc80a016..1fd0829de 100644 --- a/docs/src/piccolo/query_clauses/batch.rst +++ b/docs/src/piccolo/query_clauses/batch.rst @@ -20,10 +20,6 @@ responses. async for _batch in batch: print(_batch) -.. note:: ``batch`` is one of the few query clauses which doesn't require - .run() to be used after it in order to execute. ``batch`` effectively - replaces ``run``. - There's currently no synchronous version. However, it's easy enough to achieve: .. code-block:: python diff --git a/docs/src/piccolo/query_clauses/distinct.rst b/docs/src/piccolo/query_clauses/distinct.rst index 9fda873f5..e8d101125 100644 --- a/docs/src/piccolo/query_clauses/distinct.rst +++ b/docs/src/piccolo/query_clauses/distinct.rst @@ -9,7 +9,7 @@ You can use ``distinct`` clauses with the following queries: .. code-block:: python - >>> Band.select(Band.name).distinct().run_sync() + >>> await Band.select(Band.name).distinct() [{'title': 'Pythonistas'}] This is equivalent to ``SELECT DISTINCT name FROM band`` in SQL. diff --git a/docs/src/piccolo/query_clauses/first.rst b/docs/src/piccolo/query_clauses/first.rst index 6332656c1..1c04150f1 100644 --- a/docs/src/piccolo/query_clauses/first.rst +++ b/docs/src/piccolo/query_clauses/first.rst @@ -12,14 +12,14 @@ Rather than returning a list of results, just the first result is returned. .. code-block:: python - >>> Band.select().first().run_sync() + >>> await Band.select().first() {'name': 'Pythonistas', 'manager': 1, 'popularity': 1000, 'id': 1} Likewise, with objects: .. code-block:: python - >>> Band.objects().first().run_sync() - + >>> await Band.objects().first() + If no match is found, then ``None`` is returned instead. diff --git a/docs/src/piccolo/query_clauses/group_by.rst b/docs/src/piccolo/query_clauses/group_by.rst index 7cba6f9db..5853126cd 100644 --- a/docs/src/piccolo/query_clauses/group_by.rst +++ b/docs/src/piccolo/query_clauses/group_by.rst @@ -21,12 +21,12 @@ In the following query, we get a count of the number of bands per manager: >>> from piccolo.query.methods.select import Count - >>> Band.select( + >>> await Band.select( >>> Band.manager.name, >>> Count(Band.manager) >>> ).group_by( >>> Band.manager - >>> ).run_sync() + >>> ) [ {"manager.name": "Graydon", "count": 1}, diff --git a/docs/src/piccolo/query_clauses/limit.rst b/docs/src/piccolo/query_clauses/limit.rst index 090f1a6f7..8eee56b52 100644 --- a/docs/src/piccolo/query_clauses/limit.rst +++ b/docs/src/piccolo/query_clauses/limit.rst @@ -13,10 +13,10 @@ number you ask for. .. code-block:: python - Band.select().limit(2).run_sync() + await Band.select().limit(2) Likewise, with objects: .. code-block:: python - Band.objects().limit(2).run_sync() + await Band.objects().limit(2) diff --git a/docs/src/piccolo/query_clauses/offset.rst b/docs/src/piccolo/query_clauses/offset.rst index c24b1a4a1..66915ad4f 100644 --- a/docs/src/piccolo/query_clauses/offset.rst +++ b/docs/src/piccolo/query_clauses/offset.rst @@ -15,12 +15,12 @@ otherwise the results returned could be different each time. .. code-block:: python - >>> Band.select(Band.name).offset(1).order_by(Band.name).run_sync() + >>> await Band.select(Band.name).offset(1).order_by(Band.name) [{'name': 'Pythonistas'}, {'name': 'Rustaceans'}] Likewise, with objects: .. code-block:: python - >>> Band.objects().offset(1).order_by(Band.name).run_sync() + >>> await Band.objects().offset(1).order_by(Band.name) [Band2, Band3] diff --git a/docs/src/piccolo/query_clauses/order_by.rst b/docs/src/piccolo/query_clauses/order_by.rst index 2eed639ed..f6a66c23b 100644 --- a/docs/src/piccolo/query_clauses/order_by.rst +++ b/docs/src/piccolo/query_clauses/order_by.rst @@ -12,24 +12,24 @@ To order the results by a certain column (ascending): .. code-block:: python - Band.select().order_by( + await Band.select().order_by( Band.name - ).run_sync() + ) To order by descending: .. code-block:: python - Band.select().order_by( + await Band.select().order_by( Band.name, ascending=False - ).run_sync() + ) You can order by multiple columns, and even use joins: .. code-block:: python - Band.select().order_by( + await Band.select().order_by( Band.name, Band.manager.name - ).run_sync() + ) diff --git a/docs/src/piccolo/query_clauses/output.rst b/docs/src/piccolo/query_clauses/output.rst index f18b37723..6c3a422c3 100644 --- a/docs/src/piccolo/query_clauses/output.rst +++ b/docs/src/piccolo/query_clauses/output.rst @@ -20,8 +20,8 @@ To return the data as a JSON string: .. code-block:: python - >>> Band.select().output(as_json=True).run_sync() - '[{"name":"Pythonistas","manager":1,"popularity":1000,"id":1},{"name":"Rustaceans","manager":2,"popularity":500,"id":2}]' + >>> await Band.select(Band.name).output(as_json=True) + '[{"name":"Pythonistas"}]' Piccolo can use `orjson `_ for JSON serialisation, which is blazing fast, and can handle most Python types, including dates, @@ -36,7 +36,7 @@ If you're just querying a single column from a database table, you can use .. code-block:: python - >>> Band.select(Band.id).output(as_list=True).run_sync() + >>> await Band.select(Band.id).output(as_list=True) [1, 2] nested @@ -46,7 +46,7 @@ Output any data from related tables in nested dictionaries. .. code-block:: python - >>> Band.select(Band.name, Band.manager.name).first().output(nested=True).run_sync() + >>> await Band.select(Band.name, Band.manager.name).first().output(nested=True) {'name': 'Pythonistas', 'manager': {'name': 'Guido'}} ------------------------------------------------------------------------------- @@ -62,9 +62,9 @@ values automatically. .. code-block:: python - >>> RecordingStudio.select().output(load_json=True).run_sync() + >>> await RecordingStudio.select().output(load_json=True) [{'id': 1, 'name': 'Abbey Road', 'facilities': {'restaurant': True, 'mixing_desk': True}}] - >>> studio = RecordingStudio.objects().first().output(load_json=True).run_sync() + >>> studio = await RecordingStudio.objects().first().output(load_json=True) >>> studio.facilities {'restaurant': True, 'mixing_desk': True} diff --git a/docs/src/piccolo/query_clauses/where.rst b/docs/src/piccolo/query_clauses/where.rst index 0c8f6d7fc..dd6c50225 100644 --- a/docs/src/piccolo/query_clauses/where.rst +++ b/docs/src/piccolo/query_clauses/where.rst @@ -20,15 +20,15 @@ Equal / Not Equal .. code-block:: python - Band.select().where( + await Band.select().where( Band.name == 'Pythonistas' - ).run_sync() + ) .. code-block:: python - Band.select().where( + await Band.select().where( Band.name != 'Rustaceans' - ).run_sync() + ) .. hint:: With ``Boolean`` columns, some linters will complain if you write ``SomeTable.some_column == True`` (because it's more Pythonic to do @@ -45,9 +45,9 @@ You can use the ``<, >, <=, >=`` operators, which work as you expect. .. code-block:: python - Band.select().where( + await Band.select().where( Band.popularity >= 100 - ).run_sync() + ) ------------------------------------------------------------------------------- @@ -58,21 +58,21 @@ The percentage operator is required to designate where the match should occur. .. code-block:: python - Band.select().where( + await Band.select().where( Band.name.like('Py%') # Matches the start of the string - ).run_sync() + ) - Band.select().where( + await Band.select().where( Band.name.like('%istas') # Matches the end of the string - ).run_sync() + ) - Band.select().where( + await Band.select().where( Band.name.like('%is%') # Matches anywhere in string - ).run_sync() + ) - Band.select().where( + await Band.select().where( Band.name.like('Pythonistas') # Matches the entire string - ).run_sync() + ) ``ilike`` is identical, except it's Postgres specific and case insensitive. @@ -85,9 +85,9 @@ Usage is the same as ``like`` excepts it excludes matching rows. .. code-block:: python - Band.select().where( + await Band.select().where( Band.name.not_like('Py%') - ).run_sync() + ) ------------------------------------------------------------------------------- @@ -96,15 +96,15 @@ is_in / not_in .. code-block:: python - Band.select().where( + await Band.select().where( Band.name.is_in(['Pythonistas']) - ).run_sync() + ) .. code-block:: python - Band.select().where( + await Band.select().where( Band.name.not_in(['Rustaceans']) - ).run_sync() + ) ------------------------------------------------------------------------------- @@ -117,28 +117,28 @@ with None: .. code-block:: python # Fetch all bands with a manager - Band.select().where( + await Band.select().where( Band.manager != None - ).run_sync() + ) # Fetch all bands without a manager - Band.select().where( + await Band.select().where( Band.manager == None - ).run_sync() + ) To avoid the linter errors, you can use `is_null` and `is_not_null` instead. .. code-block:: python # Fetch all bands with a manager - Band.select().where( + await Band.select().where( Band.manager.is_not_null() - ).run_sync() + ) # Fetch all bands without a manager - Band.select().where( + await Band.select().where( Band.manager.is_null() - ).run_sync() + ) ------------------------------------------------------------------------------- @@ -149,13 +149,13 @@ You can make complex ``where`` queries using ``&`` for AND, and ``|`` for OR. .. code-block:: python - Band.select().where( + await Band.select().where( (Band.popularity >= 100) & (Band.popularity < 1000) - ).run_sync() + ) - Band.select().where( + await Band.select().where( (Band.popularity >= 100) | (Band.name == 'Pythonistas') - ).run_sync() + ) You can make really complex ``where`` clauses if you so choose - just be careful to include brackets in the correct place. @@ -169,28 +169,28 @@ Using multiple ``where`` clauses is equivalent to an AND. .. code-block:: python # These are equivalent: - Band.select().where( + await Band.select().where( (Band.popularity >= 100) & (Band.popularity < 1000) - ).run_sync() + ) - Band.select().where( + await Band.select().where( Band.popularity >= 100 ).where( Band.popularity < 1000 - ).run_sync() + ) Also, multiple arguments inside ``where`` clause is equivalent to an AND. .. code-block:: python # These are equivalent: - Band.select().where( + await Band.select().where( (Band.popularity >= 100) & (Band.popularity < 1000) - ).run_sync() + ) - Band.select().where( + await Band.select().where( Band.popularity >= 100, Band.popularity < 1000 - ).run_sync() + ) Using And / Or directly ~~~~~~~~~~~~~~~~~~~~~~~ @@ -202,12 +202,12 @@ Rather than using the ``|`` and ``&`` characters, you can use the ``And`` and from piccolo.columns.combination import And, Or - Band.select().where( + await Band.select().where( Or( And(Band.popularity >= 100, Band.popularity < 1000), Band.name == 'Pythonistas' ) - ).run_sync() + ) ------------------------------------------------------------------------------- @@ -220,9 +220,9 @@ In certain situations you may want to have raw SQL in your where clause. from piccolo.columns.combination import WhereRaw - Band.select().where( + await Band.select().where( WhereRaw("name = 'Pythonistas'") - ).run_sync() + ) It's important to parameterise your SQL statements if the values come from an untrusted source, otherwise it could lead to a SQL injection attack. @@ -233,9 +233,9 @@ untrusted source, otherwise it could lead to a SQL injection attack. value = "Could be dangerous" - Band.select().where( + await Band.select().where( WhereRaw("name = {}", value) - ).run_sync() + ) ``WhereRaw`` can be combined into complex queries, just as you'd expect: @@ -243,6 +243,6 @@ untrusted source, otherwise it could lead to a SQL injection attack. from piccolo.columns.combination import WhereRaw - Band.select().where( + await Band.select().where( WhereRaw("name = 'Pythonistas'") | (Band.popularity > 1000) - ).run_sync() + ) diff --git a/docs/src/piccolo/query_types/alter.rst b/docs/src/piccolo/query_types/alter.rst index db1ff378a..4361ec68c 100644 --- a/docs/src/piccolo/query_types/alter.rst +++ b/docs/src/piccolo/query_types/alter.rst @@ -16,7 +16,7 @@ Used to add a column to an existing table. .. code-block:: python - Band.alter().add_column('members', Integer()).run_sync() + await Band.alter().add_column('members', Integer()) ------------------------------------------------------------------------------- @@ -27,7 +27,7 @@ Used to drop an existing column. .. code-block:: python - Band.alter().drop_column('popularity').run_sync() + await Band.alter().drop_column('popularity') ------------------------------------------------------------------------------- @@ -38,7 +38,7 @@ Used to drop the table - use with caution! .. code-block:: python - Band.alter().drop_table().run_sync() + await Band.alter().drop_table() If you have several tables which you want to drop, you can use ``drop_tables`` instead. It will drop them in the correct order. @@ -58,7 +58,7 @@ Used to rename an existing column. .. code-block:: python - Band.alter().rename_column(Band.popularity, 'rating').run_sync() + await Band.alter().rename_column(Band.popularity, 'rating') ------------------------------------------------------------------------------- @@ -70,10 +70,10 @@ Set whether a column is nullable or not. .. code-block:: python # To make a row nullable: - Band.alter().set_null(Band.name, True).run_sync() + await Band.alter().set_null(Band.name, True) # To stop a row being nullable: - Band.alter().set_null(Band.name, False).run_sync() + await Band.alter().set_null(Band.name, False) ------------------------------------------------------------------------------- @@ -85,7 +85,7 @@ Used to change whether a column is unique or not. .. code-block:: python # To make a row unique: - Band.alter().set_unique(Band.name, True).run_sync() + await Band.alter().set_unique(Band.name, True) # To stop a row being unique: - Band.alter().set_unique(Band.name, False).run_sync() + await Band.alter().set_unique(Band.name, False) diff --git a/docs/src/piccolo/query_types/create_table.rst b/docs/src/piccolo/query_types/create_table.rst index 0144d90f3..2e6e98cc5 100644 --- a/docs/src/piccolo/query_types/create_table.rst +++ b/docs/src/piccolo/query_types/create_table.rst @@ -9,7 +9,7 @@ This creates the table and columns in the database. .. code-block:: python - >>> Band.create_table().run_sync() + >>> await Band.create_table() [] @@ -17,7 +17,7 @@ To prevent an error from being raised if the table already exists: .. code-block:: python - >>> Band.create_table(if_not_exists=True).run_sync() + >>> await Band.create_table(if_not_exists=True) [] Also, you can create multiple tables at once. @@ -25,7 +25,7 @@ Also, you can create multiple tables at once. This function will automatically sort tables based on their foreign keys so they're created in the right order: .. code-block:: python - - >>> from piccolo.table import create_tables + + >>> from piccolo.table import create_tables >>> create_tables(Band, Manager, if_not_exists=True) - + diff --git a/docs/src/piccolo/query_types/delete.rst b/docs/src/piccolo/query_types/delete.rst index 9ca85bc9d..90cbff8de 100644 --- a/docs/src/piccolo/query_types/delete.rst +++ b/docs/src/piccolo/query_types/delete.rst @@ -7,7 +7,7 @@ This deletes any matching rows from the table. .. code-block:: python - >>> Band.delete().where(Band.name == 'Rustaceans').run_sync() + >>> await Band.delete().where(Band.name == 'Rustaceans') [] ------------------------------------------------------------------------------- @@ -21,11 +21,11 @@ the data from a table. .. code-block:: python - >>> Band.delete().run_sync() + >>> await Band.delete() Raises: DeletionError # Works fine: - >>> Band.delete(force=True).run_sync() + >>> await Band.delete(force=True) [] ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/query_types/exists.rst b/docs/src/piccolo/query_types/exists.rst index a9325f253..982ba54f5 100644 --- a/docs/src/piccolo/query_types/exists.rst +++ b/docs/src/piccolo/query_types/exists.rst @@ -7,7 +7,7 @@ This checks whether any rows exist which match the criteria. .. code-block:: python - >>> Band.exists().where(Band.name == 'Pythonistas').run_sync() + >>> await Band.exists().where(Band.name == 'Pythonistas') True ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/query_types/insert.rst b/docs/src/piccolo/query_types/insert.rst index 9775f95ba..eda460de1 100644 --- a/docs/src/piccolo/query_types/insert.rst +++ b/docs/src/piccolo/query_types/insert.rst @@ -7,17 +7,17 @@ This is used to insert rows into the table. .. code-block:: python - >>> Band.insert(Band(name="Pythonistas")).run_sync() + >>> await Band.insert(Band(name="Pythonistas")) [{'id': 3}] We can insert multiple rows in one go: .. code-block:: python - Band.insert( + await Band.insert( Band(name="Darts"), Band(name="Gophers") - ).run_sync() + ) ------------------------------------------------------------------------------- @@ -28,8 +28,8 @@ You can also compose it as follows: .. code-block:: python - Band.insert().add( + await Band.insert().add( Band(name="Darts") ).add( Band(name="Gophers") - ).run_sync() + ) diff --git a/docs/src/piccolo/query_types/objects.rst b/docs/src/piccolo/query_types/objects.rst index 851df01a8..7183e7d38 100644 --- a/docs/src/piccolo/query_types/objects.rst +++ b/docs/src/piccolo/query_types/objects.rst @@ -20,28 +20,28 @@ To get all objects: .. code-block:: python - >>> Band.objects().run_sync() + >>> await Band.objects() [, ] To get certain rows: .. code-block:: python - >>> Band.objects().where(Band.name == 'Pythonistas').run_sync() + >>> await Band.objects().where(Band.name == 'Pythonistas') [] To get a single row (or ``None`` if it doesn't exist): .. code-block:: python - >>> Band.objects().get(Band.name == 'Pythonistas').run_sync() + >>> await Band.objects().get(Band.name == 'Pythonistas') To get the first row: .. code-block:: python - >>> Band.objects().first().run_sync() + >>> await Band.objects().first() You'll notice that the API is similar to :ref:`Select` - except it returns all @@ -55,13 +55,13 @@ Creating objects .. code-block:: python >>> band = Band(name="C-Sharps", popularity=100) - >>> band.save().run_sync() + >>> await band.save() This can also be done like this: .. code-block:: python - >>> band = Band.objects().create(name="C-Sharps", popularity=100).run_sync() + >>> band = await Band.objects().create(name="C-Sharps", popularity=100) ------------------------------------------------------------------------------- @@ -72,17 +72,17 @@ Objects have a ``save`` method, which is convenient for updating values: .. code-block:: python - band = Band.objects().where( + band = await Band.objects().where( Band.name == 'Pythonistas' - ).first().run_sync() + ).first() band.popularity = 100000 # This saves all values back to the database. - band.save().run_sync() + await band.save() # Or specify specific columns to save: - band.save([Band.popularity]).run_sync() + await band.save([Band.popularity]) ------------------------------------------------------------------------------- @@ -93,11 +93,11 @@ Similarly, we can delete objects, using the ``remove`` method. .. code-block:: python - band = Band.objects().where( + band = await Band.objects().where( Band.name == 'Pythonistas' - ).first().run_sync() + ).first() - band.remove().run_sync() + await band.remove() ------------------------------------------------------------------------------- @@ -112,11 +112,11 @@ to fetch the related row as an object, you can do so using ``get_related``. .. code-block:: python - band = Band.objects().where( + band = await Band.objects().where( Band.name == 'Pythonistas' - ).first().run_sync() + ).first() - manager = band.get_related(Band.manager).run_sync() + manager = await band.get_related(Band.manager) >>> manager >>> manager.name @@ -131,9 +131,9 @@ refer to the related rows you want to load. .. code-block:: python - band = Band.objects(Band.manager).where( + band = await Band.objects(Band.manager).where( Band.name == 'Pythonistas' - ).first().run_sync() + ).first() >>> band.manager @@ -145,9 +145,9 @@ prefetch them all you can do so using ``all_related``. .. code-block:: python - ticket = Ticket.objects( + ticket = await Ticket.objects( Ticket.concert.all_related() - ).first().run_sync() + ).first() # Any intermediate objects will also be loaded: >>> ticket.concert @@ -172,13 +172,13 @@ can use the ``prefetch`` clause if you prefer. .. code-block:: python # These are equivalent: - ticket = Ticket.objects( + ticket = await Ticket.objects( Ticket.concert.all_related() - ).first().run_sync() + ).first() - ticket = Ticket.objects().prefetch( + ticket = await Ticket.objects().prefetch( Ticket.concert.all_related() - ).run_sync() + ) ------------------------------------------------------------------------------- @@ -190,22 +190,22 @@ or create a new one with the ``defaults`` arguments: .. code-block:: python - band = Band.objects().get_or_create( + band = await Band.objects().get_or_create( Band.name == 'Pythonistas', defaults={Band.popularity: 100} - ).run_sync() + ) # Or using string column names - band = Band.objects().get_or_create( + band = await Band.objects().get_or_create( Band.name == 'Pythonistas', defaults={'popularity': 100} - ).run_sync() + ) You can find out if an existing row was found, or if a new row was created: .. code-block:: python - band = Band.objects.get_or_create( + band = await Band.objects.get_or_create( Band.name == 'Pythonistas' - ).run_sync() + ) band._was_created # True if it was created, otherwise False if it was already in the db Complex where clauses are supported, but only within reason. For example: @@ -213,16 +213,16 @@ Complex where clauses are supported, but only within reason. For example: .. code-block:: python # This works OK: - band = Band.objects().get_or_create( + band = await Band.objects().get_or_create( (Band.name == 'Pythonistas') & (Band.popularity == 1000), - ).run_sync() + ) # This is problematic, as it's unclear what the name should be if we # need to create the row: - band = Band.objects().get_or_create( + band = await Band.objects().get_or_create( (Band.name == 'Pythonistas') | (Band.name == 'Rustaceans'), defaults={'popularity': 100} - ).run_sync() + ) ------------------------------------------------------------------------------- @@ -234,7 +234,7 @@ If you need to convert an object into a dictionary, you can do so using the .. code-block:: python - band = Band.objects().first().run_sync() + band = await Band.objects().first() >>> band.to_dict() {'id': 1, 'name': 'Pythonistas', 'manager': 1, 'popularity': 1000} @@ -244,7 +244,7 @@ the columns: .. code-block:: python - band = Band.objects().first().run_sync() + band = await Band.objects().first() >>> band.to_dict(Band.id, Band.name.as_alias('title')) {'id': 1, 'title': 'Pythonistas'} @@ -262,27 +262,27 @@ See :ref:`batch`. limit ~~~~~ -See  :ref:`limit`. +See :ref:`limit`. offset ~~~~~~ -See  :ref:`offset`. +See :ref:`offset`. first ~~~~~ -See  :ref:`first`. +See :ref:`first`. order_by ~~~~~~~~ -See  :ref:`order_by`. +See :ref:`order_by`. output ~~~~~~ -See  :ref:`output`. +See :ref:`output`. where ~~~~~ diff --git a/docs/src/piccolo/query_types/raw.rst b/docs/src/piccolo/query_types/raw.rst index c06d265f7..71c295855 100644 --- a/docs/src/piccolo/query_types/raw.rst +++ b/docs/src/piccolo/query_types/raw.rst @@ -7,16 +7,15 @@ Should you need to, you can execute raw SQL. .. code-block:: python - >>> Band.raw('select * from band').run_sync() - [{'name': 'Pythonistas', 'manager': 1, 'popularity': 1000, 'id': 1}, - {'name': 'Rustaceans', 'manager': 2, 'popularity': 500, 'id': 2}] + >>> await Band.raw('select name from band').run_sync() + [{'name': 'Pythonistas'}] It's recommended that you parameterise any values. Use curly braces ``{}`` as placeholders: .. code-block:: python - >>> Band.raw('select * from band where name = {}', 'Pythonistas').run_sync() + >>> await Band.raw('select * from band where name = {}', 'Pythonistas') [{'name': 'Pythonistas', 'manager': 1, 'popularity': 1000, 'id': 1}] .. warning:: Be careful to avoid SQL injection attacks. Don't add any user submitted data into your SQL strings, unless it's parameterised. diff --git a/docs/src/piccolo/query_types/select.rst b/docs/src/piccolo/query_types/select.rst index 9dc612937..5f2509287 100644 --- a/docs/src/piccolo/query_types/select.rst +++ b/docs/src/piccolo/query_types/select.rst @@ -9,7 +9,7 @@ To get all rows: .. code-block:: python - >>> Band.select().run_sync() + >>> await Band.select() [{'id': 1, 'name': 'Pythonistas', 'manager': 1, 'popularity': 1000}, {'id': 2, 'name': 'Rustaceans', 'manager': 2, 'popularity': 500}] @@ -17,7 +17,7 @@ To get certain columns: .. code-block:: python - >>> Band.select(Band.name).run_sync() + >>> await Band.select(Band.name) [{'name': 'Rustaceans'}, {'name': 'Pythonistas'}] Or use an alias to make it shorter: @@ -25,7 +25,7 @@ Or use an alias to make it shorter: .. code-block:: python >>> b = Band - >>> b.select(b.name).run_sync() + >>> await b.select(b.name) [{'id': 1, 'name': 'Pythonistas', 'manager': 1, 'popularity': 1000}, {'id': 2, 'name': 'Rustaceans', 'manager': 2, 'popularity': 500}] @@ -40,7 +40,7 @@ By using ``as_alias``, the name of the row can be overriden in the response. .. code-block:: python - >>> Band.select(Band.name.as_alias('title')).run_sync() + >>> await Band.select(Band.name.as_alias('title')) [{'title': 'Rustaceans'}, {'title': 'Pythonistas'}] This is equivalent to ``SELECT name AS title FROM band`` in SQL. @@ -54,7 +54,7 @@ One of the most powerful things about ``select`` is it's support for joins. .. code-block:: python - >>> Band.select(Band.name, Band.manager.name).run_sync() + >>> await Band.select(Band.name, Band.manager.name) [ {'name': 'Pythonistas', 'manager.name': 'Guido'}, {'name': 'Rustaceans', 'manager.name': 'Graydon'} @@ -65,7 +65,7 @@ The joins can go several layers deep. .. code-block:: python - >>> Concert.select(Concert.id, Concert.band_1.manager.name).run_sync() + >>> await Concert.select(Concert.id, Concert.band_1.manager.name) [{'id': 1, 'band_1.manager.name': 'Guido'}] all_columns @@ -77,7 +77,7 @@ all out: .. code-block:: python - >>> Band.select(Band.name, Band.manager.all_columns()).run_sync() + >>> await Band.select(Band.name, Band.manager.all_columns()) [ {'name': 'Pythonistas', 'manager.id': 1, 'manager.name': 'Guido'}, {'name': 'Rustaceans', 'manager.id': 2, 'manager.name': 'Graydon'} @@ -89,17 +89,17 @@ equivalent to the code above: .. code-block:: python - >>> Band.select(Band.name, *Band.manager.all_columns()).run_sync() + >>> await Band.select(Band.name, *Band.manager.all_columns()) You can exclude some columns if you like: .. code-block:: python - >>> Band.select( + >>> await Band.select( >>> Band.name, >>> Band.manager.all_columns(exclude=[Band.manager.id) - >>> ).run_sync() + >>> ) [ {'name': 'Pythonistas', 'manager.name': 'Guido'}, {'name': 'Rustaceans', 'manager.name': 'Graydon'} @@ -110,10 +110,10 @@ Strings are supported too if you prefer: .. code-block:: python - >>> Band.select( + >>> await Band.select( >>> Band.name, >>> Band.manager.all_columns(exclude=['id']) - >>> ).run_sync() + >>> ) [ {'name': 'Pythonistas', 'manager.name': 'Guido'}, {'name': 'Rustaceans', 'manager.name': 'Graydon'} @@ -124,10 +124,10 @@ you have lots of columns. It works identically to related tables: .. code-block:: python - >>> Band.select( + >>> await Band.select( >>> Band.all_columns(exclude=[Band.id]), >>> Band.manager.all_columns(exclude=[Band.manager.id]) - >>> ).run_sync() + >>> ) [ {'name': 'Pythonistas', 'popularity': 1000, 'manager.name': 'Guido'}, {'name': 'Rustaceans', 'popularity': 500, 'manager.name': 'Graydon'} @@ -140,7 +140,7 @@ You can also get the response as nested dictionaries, which can be very useful: .. code-block:: python - >>> Band.select(Band.name, Band.manager.all_columns()).output(nested=True).run_sync() + >>> await Band.select(Band.name, Band.manager.all_columns()).output(nested=True) [ {'name': 'Pythonistas', 'manager': {'id': 1, 'name': 'Guido'}}, {'name': 'Rustaceans', 'manager': {'id': 2, 'manager.name': 'Graydon'}} @@ -157,10 +157,10 @@ convenient. .. code-block:: python - Band.select('name').run_sync() + await Band.select('name') # For joins: - Band.select('manager.name').run_sync() + await Band.select('manager.name') ------------------------------------------------------------------------------- @@ -174,7 +174,7 @@ Returns the number of rows which match the query: .. code-block:: python - >>> Band.count().where(Band.name == 'Pythonistas').run_sync() + >>> await Band.count().where(Band.name == 'Pythonistas') 1 Avg @@ -185,7 +185,7 @@ Returns the average for a given column: .. code-block:: python >>> from piccolo.query import Avg - >>> response = Band.select(Avg(Band.popularity)).first().run_sync() + >>> response = await Band.select(Avg(Band.popularity)).first() >>> response["avg"] 750.0 @@ -197,7 +197,7 @@ Returns the sum for a given column: .. code-block:: python >>> from piccolo.query import Sum - >>> response = Band.select(Sum(Band.popularity)).first().run_sync() + >>> response = await Band.select(Sum(Band.popularity)).first() >>> response["sum"] 1500 @@ -209,7 +209,7 @@ Returns the maximum for a given column: .. code-block:: python >>> from piccolo.query import Max - >>> response = Band.select(Max(Band.popularity)).first().run_sync() + >>> response = await Band.select(Max(Band.popularity)).first() >>> response["max"] 1000 @@ -221,19 +221,19 @@ Returns the minimum for a given column: .. code-block:: python >>> from piccolo.query import Min - >>> response = Band.select(Min(Band.popularity)).first().run_sync() + >>> response = await Band.select(Min(Band.popularity)).first() >>> response["min"] 500 Additional features ~~~~~~~~~~~~~~~~~~~ -You also can chain multiple different aggregate functions in one query: +You also can have multiple different aggregate functions in one query: .. code-block:: python >>> from piccolo.query import Avg, Sum - >>> response = Band.select(Avg(Band.popularity), Sum(Band.popularity)).first().run_sync() + >>> response = await Band.select(Avg(Band.popularity), Sum(Band.popularity)).first() >>> response {"avg": 750.0, "sum": 1500} @@ -241,13 +241,8 @@ And can use aliases for aggregate functions like this: .. code-block:: python - >>> from piccolo.query import Avg - >>> response = Band.select(Avg(Band.popularity, alias="popularity_avg")).first().run_sync() - >>> response["popularity_avg"] - 750.0 - # Alternatively, you can use the `as_alias` method. - >>> response = Band.select(Avg(Band.popularity).as_alias("popularity_avg")).first().run_sync() + >>> response = await Band.select(Avg(Band.popularity).as_alias("popularity_avg")).first() >>> response["popularity_avg"] 750.0 @@ -269,7 +264,7 @@ By default all columns are returned from the queried table. .. code-block:: python # Equivalent to SELECT * from band - Band.select().run_sync() + await Band.select() To restrict the returned columns, either pass in the columns into the ``select`` method, or use the ``columns`` method. @@ -277,51 +272,51 @@ To restrict the returned columns, either pass in the columns into the .. code-block:: python # Equivalent to SELECT name from band - Band.select(Band.name).run_sync() + await Band.select(Band.name) # Or alternatively: - Band.select().columns(Band.name).run_sync() + await Band.select().columns(Band.name) The ``columns`` method is additive, meaning you can chain it to add additional columns. .. code-block:: python - Band.select().columns(Band.name).columns(Band.manager).run_sync() + await Band.select().columns(Band.name).columns(Band.manager) # Or just define it one go: - Band.select().columns(Band.name, Band.manager).run_sync() + await Band.select().columns(Band.name, Band.manager) first ~~~~~ -See  :ref:`first`. +See :ref:`first`. group_by ~~~~~~~~ -See  :ref:`group_by`. +See :ref:`group_by`. limit ~~~~~ -See  :ref:`limit`. +See :ref:`limit`. offset ~~~~~~ -See  :ref:`offset`. +See :ref:`offset`. distinct ~~~~~~~~ -See  :ref:`distinct`. +See :ref:`distinct`. order_by ~~~~~~~~ -See  :ref:`order_by`. +See :ref:`order_by`. output ~~~~~~ @@ -331,4 +326,4 @@ See :ref:`output`. where ~~~~~ -See  :ref:`where`. +See :ref:`where`. diff --git a/docs/src/piccolo/query_types/update.rst b/docs/src/piccolo/query_types/update.rst index 1118eb874..6b46ace67 100644 --- a/docs/src/piccolo/query_types/update.rst +++ b/docs/src/piccolo/query_types/update.rst @@ -7,11 +7,11 @@ This is used to update any rows in the table which match the criteria. .. code-block:: python - >>> Band.update({ + >>> await Band.update({ >>> Band.name: 'Pythonistas 2' >>> }).where( >>> Band.name == 'Pythonistas' - >>> ).run_sync() + >>> ) [] As well as replacing values with new ones, you can also modify existing values, for @@ -30,29 +30,29 @@ You can add / subtract / multiply / divide values: .. code-block:: python # Add 100 to the popularity of each band: - Band.update({ + await Band.update({ Band.popularity: Band.popularity + 100 - }).run_sync() + }) # Decrease the popularity of each band by 100. - Band.update({ + await Band.update({ Band.popularity: Band.popularity - 100 - }).run_sync() + }) # Multiply the popularity of each band by 10. - Band.update({ + await Band.update({ Band.popularity: Band.popularity * 10 - }).run_sync() + }) # Divide the popularity of each band by 10. - Band.update({ + await Band.update({ Band.popularity: Band.popularity / 10 - }).run_sync() + }) # You can also use the operators in reverse: - Band.update({ + await Band.update({ Band.popularity: 2000 - Band.popularity - }).run_sync() + }) Varchar / Text columns ~~~~~~~~~~~~~~~~~~~~~~ @@ -62,19 +62,19 @@ You can concatenate values: .. code-block:: python # Append "!!!" to each band name. - Band.update({ + await Band.update({ Band.name: Band.name + "!!!" - }).run_sync() + }) # Concatenate the values in each column: - Band.update({ + await Band.update({ Band.name: Band.name + Band.name - }).run_sync() + }) # Prepend "!!!" to each band name. - Band.update({ + await Band.update({ Band.popularity: "!!!" + Band.popularity - }).run_sync() + }) You can currently only combine two values together at a time. diff --git a/docs/src/piccolo/schema/advanced.rst b/docs/src/piccolo/schema/advanced.rst index 42aea1b93..067df78c2 100644 --- a/docs/src/piccolo/schema/advanced.rst +++ b/docs/src/piccolo/schema/advanced.rst @@ -32,7 +32,7 @@ tooling - you can also use it your own queries. .. code-block:: python - Band.select(Band.get_readable()).run_sync() + await Band.select(Band.get_readable()) Here is an example of a more complex ``Readable``. @@ -111,9 +111,9 @@ We can then use the ``Enum`` in our queries. .. code-block:: python - >>> Shirt(size=Shirt.Size.large).save().run_sync() + >>> await Shirt(size=Shirt.Size.large).save() - >>> Shirt.select().run_sync() + >>> await Shirt.select() [{'id': 1, 'size': 'l'}] Note how the value stored in the database is the ``Enum`` value (in this case ``'l'``). @@ -123,12 +123,12 @@ where a query requires a value. .. code-block:: python - >>> Shirt.insert( + >>> await Shirt.insert( >>> Shirt(size=Shirt.Size.small), >>> Shirt(size=Shirt.Size.medium) - >>> ).run_sync() + >>> ) - >>> Shirt.select().where(Shirt.size == Shirt.Size.small).run_sync() + >>> await Shirt.select().where(Shirt.size == Shirt.Size.small) [{'id': 1, 'size': 's'}] Advantages @@ -184,7 +184,7 @@ Then you can use them like your normal ``Table`` classes: .. code-block:: python - >>> Band.select().run_sync() + >>> await Band.select() [{'id': 1, 'name': 'Pythonistas', 'manager': 1}, ...] diff --git a/docs/src/piccolo/schema/column_types.rst b/docs/src/piccolo/schema/column_types.rst index 4b86b01ec..fc2d3e63c 100644 --- a/docs/src/piccolo/schema/column_types.rst +++ b/docs/src/piccolo/schema/column_types.rst @@ -204,23 +204,23 @@ a subset of the JSON data, and for filtering in a where clause. class Booking(Table): data = JSONB() - Booking.create_table().run_sync() + await Booking.create_table() # Example data: - Booking.insert( + await Booking.insert( Booking(data='{"name": "Alison"}'), Booking(data='{"name": "Bob"}') - ).run_sync() + ) # Example queries - >>> Booking.select( + >>> await Booking.select( >>> Booking.id, Booking.data.arrow('name').as_alias('name') - >>> ).run_sync() + >>> ) [{'id': 1, 'name': '"Alison"'}, {'id': 2, 'name': '"Bob"'}] - >>> Booking.select(Booking.id).where( + >>> await Booking.select(Booking.id).where( >>> Booking.data.arrow('name') == '"Alison"' - >>> ).run_sync() + >>> ) [{'id': 1}] ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/serialization/index.rst b/docs/src/piccolo/serialization/index.rst index a55ce333c..c1f7f3f9b 100644 --- a/docs/src/piccolo/serialization/index.rst +++ b/docs/src/piccolo/serialization/index.rst @@ -40,14 +40,12 @@ We can then create model instances from data we fetch from the database: .. code-block:: python # If using objects: - model = BandModel( - **Band.objects().get(Band.name == 'Pythonistas').run_sync().to_dict() - ) + band = await Band.objects().get(Band.name == 'Pythonistas') + model = BandModel(**band.to_dict()) # If using select: - model = BandModel( - **Band.select().where(Band.name == 'Pythonistas').first().run_sync() - ) + band = await Band.select().where(Band.name == 'Pythonistas').first() + model = BandModel(**band) >>> model.name 'Pythonistas' @@ -103,21 +101,19 @@ To populate a nested Pydantic model with data from the database: .. code-block:: python # If using objects: - model = BandModel( - **Band.objects(Band.manager).get(Band.name == 'Pythonistas').run_sync().to_dict() - ) + band = await Band.objects(Band.manager).get(Band.name == 'Pythonistas') + model = BandModel(**band.to_dict()) # If using select: - model = BandModel( - **Band.select( - Band.all_columns(), - Band.manager.all_columns() - ).where( - Band.name == 'Pythonistas' - ).first().output( - nested=True - ).run_sync() + band = await Band.select( + Band.all_columns(), + Band.manager.all_columns() + ).where( + Band.name == 'Pythonistas' + ).first().output( + nested=True ) + model = BandModel(**band) >>> model.manager.name 'Guido' diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 30282ad50..195686cd6 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -385,7 +385,7 @@ class Column(Selectable): class MyTable(Table): class_ = Varchar(db_column_name="class") - >>> MyTable.select(MyTable.class_).run_sync() + >>> await MyTable.select(MyTable.class_) [{'id': 1, 'class': 'test'}] This is an advanced feature which you should only need in niche @@ -402,7 +402,7 @@ class Band(Table): name = Varchar() net_worth = Integer(secret=True) - >>> Property.select(exclude_secrets=True).run_sync() + >>> await Band.select(exclude_secrets=True) [{'name': 'Pythonistas'}] """ @@ -598,7 +598,7 @@ def _equals(self, column: Column, including_joins: bool = False) -> bool: .. code-block:: python - Band.select().where(Band.name == 'Pythonistas').run_sync() + await Band.select().where(Band.name == 'Pythonistas') But this means that comparisons such as this can give unexpected results: diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index c933e95c6..7e28ab094 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -173,10 +173,10 @@ class Band(Table): name = Varchar(length=100) # Create - >>> Band(name='Pythonistas').save().run_sync() + >>> await Band(name='Pythonistas').save() # Query - >>> Band.select(Band.name).run_sync() + >>> await Band.select(Band.name) {'name': 'Pythonistas'} :param length: @@ -283,10 +283,10 @@ class Band(Table): name = Text() # Create - >>> Band(name='Pythonistas').save().run_sync() + >>> await Band(name='Pythonistas').save() # Query - >>> Band.select(Band.name).run_sync() + >>> await Band.select(Band.name) {'name': 'Pythonistas'} """ @@ -354,10 +354,10 @@ class Band(Table): uuid = UUID() # Create - >>> DiscountCode(code=uuid.uuid4()).save().run_sync() + >>> await DiscountCode(code=uuid.uuid4()).save() # Query - >>> DiscountCode.select(DiscountCode.code).run_sync() + >>> await DiscountCode.select(DiscountCode.code) {'code': UUID('09c4c17d-af68-4ce7-9955-73dcd892e462')} """ @@ -416,10 +416,10 @@ class Band(Table): popularity = Integer() # Create - >>> Band(popularity=1000).save().run_sync() + >>> await Band(popularity=1000).save() # Query - >>> Band.select(Band.popularity).run_sync() + >>> await Band.select(Band.popularity) {'popularity': 1000} """ @@ -540,10 +540,10 @@ class Band(Table): value = BigInt() # Create - >>> Band(popularity=1000000).save().run_sync() + >>> await Band(popularity=1000000).save() # Query - >>> Band.select(Band.popularity).run_sync() + >>> await Band.select(Band.popularity) {'popularity': 1000000} """ @@ -588,10 +588,10 @@ class Band(Table): value = SmallInt() # Create - >>> Band(popularity=1000).save().run_sync() + >>> await Band(popularity=1000).save() # Query - >>> Band.select(Band.popularity).run_sync() + >>> await Band.select(Band.popularity) {'popularity': 1000} """ @@ -754,12 +754,12 @@ class Concert(Table): starts = Timestamp() # Create - >>> Concert( + >>> await Concert( >>> starts=datetime.datetime(year=2050, month=1, day=1) - >>> ).save().run_sync() + >>> ).save() # Query - >>> Concert.select(Concert.starts).run_sync() + >>> await Concert.select(Concert.starts) {'starts': datetime.datetime(2050, 1, 1, 0, 0)} """ @@ -820,14 +820,14 @@ class Concert(Table): starts = Timestamptz() # Create - >>> Concert( + >>> await Concert( >>> starts=datetime.datetime( >>> year=2050, month=1, day=1, tzinfo=datetime.timezone.tz >>> ) - >>> ).save().run_sync() + >>> ).save() # Query - >>> Concert.select(Concert.starts).run_sync() + >>> await Concert.select(Concert.starts) { 'starts': datetime.datetime( 2050, 1, 1, 0, 0, tzinfo=datetime.timezone.utc @@ -887,12 +887,12 @@ class Concert(Table): starts = Date() # Create - >>> Concert( + >>> await Concert( >>> starts=datetime.date(year=2020, month=1, day=1) - >>> ).save().run_sync() + >>> ).save() # Query - >>> Concert.select(Concert.starts).run_sync() + >>> await Concert.select(Concert.starts) {'starts': datetime.date(2020, 1, 1)} """ @@ -944,12 +944,12 @@ class Concert(Table): starts = Time() # Create - >>> Concert( + >>> await Concert( >>> starts=datetime.time(hour=20, minute=0, second=0) - >>> ).save().run_sync() + >>> ).save() # Query - >>> Concert.select(Concert.starts).run_sync() + >>> await Concert.select(Concert.starts) {'starts': datetime.time(20, 0, 0)} """ @@ -998,12 +998,12 @@ class Concert(Table): duration = Interval() # Create - >>> Concert( + >>> await Concert( >>> duration=timedelta(hours=2) - >>> ).save().run_sync() + >>> ).save() # Query - >>> Concert.select(Concert.duration).run_sync() + >>> await Concert.select(Concert.duration) {'duration': datetime.timedelta(seconds=7200)} """ @@ -1067,10 +1067,10 @@ class Band(Table): has_drummer = Boolean() # Create - >>> Band(has_drummer=True).save().run_sync() + >>> await Band(has_drummer=True).save() # Query - >>> Band.select(Band.has_drummer).run_sync() + >>> await Band.select(Band.has_drummer) {'has_drummer': True} """ @@ -1096,7 +1096,7 @@ def eq(self, value) -> Where: await MyTable.select().where( MyTable.some_boolean_column == True - ).run() + ) It's more Pythonic to use ``is True`` rather than ``== True``, which is why linters complain. The work around is to do the following instead: @@ -1105,7 +1105,7 @@ def eq(self, value) -> Where: await MyTable.select().where( MyTable.some_boolean_column.__eq__(True) - ).run() + ) Using the ``__eq__`` magic method is a bit untidy, which is why this ``eq`` method exists. @@ -1114,7 +1114,7 @@ def eq(self, value) -> Where: await MyTable.select().where( MyTable.some_boolean_column.eq(True) - ).run() + ) The ``ne`` method exists for the same reason, for ``!=``. @@ -1163,10 +1163,10 @@ class Ticket(Table): price = Numeric(digits=(5,2)) # Create - >>> Ticket(price=Decimal('50.0')).save().run_sync() + >>> await Ticket(price=Decimal('50.0')).save() # Query - >>> Ticket.select(Ticket.price).run_sync() + >>> await Ticket.select(Ticket.price) {'price': Decimal('50.0')} :param digits: @@ -1282,10 +1282,10 @@ class Concert(Table): rating = Real() # Create - >>> Concert(rating=7.8).save().run_sync() + >>> await Concert(rating=7.8).save() # Query - >>> Concert.select(Concert.rating).run_sync() + >>> await Concert.select(Concert.rating) {'rating': 7.8} """ @@ -1391,14 +1391,14 @@ class Band(Table): manager = ForeignKey(references=Manager) # Create - >>> Band(manager=1).save().run_sync() + >>> await Band(manager=1).save() # Query - >>> Band.select(Band.manager).run_sync() + >>> await Band.select(Band.manager) {'manager': 1} # Query object - >>> band = await Band.objects().first().run() + >>> band = await Band.objects().first() >>> band.manager 1 @@ -1408,14 +1408,14 @@ class Band(Table): .. code-block:: python - >>> await Band.select(Band.name, Band.manager.name).first().run() + >>> await Band.select(Band.name, Band.manager.name).first() {'name': 'Pythonistas', 'manager.name': 'Guido'} To retrieve all of the columns in the related table: .. code-block:: python - >>> await Band.select(Band.name, *Band.manager.all_columns()).first().run() + >>> await Band.select(Band.name, *Band.manager.all_columns()).first() {'name': 'Pythonistas', 'manager.id': 1, 'manager.name': 'Guido'} To get a referenced row as an object: @@ -1424,21 +1424,21 @@ class Band(Table): manager = await Manager.objects().where( Manager.id == some_band.manager - ).run() + ) Or use either of the following, which are just a proxy to the above: .. code-block:: python - manager = await band.get_related('manager').run() - manager = await band.get_related(Band.manager).run() + manager = await band.get_related('manager') + manager = await band.get_related(Band.manager) To change the manager: .. code-block:: python band.manager = some_manager_id - await band.save().run() + await band.save() :param references: The ``Table`` being referenced. @@ -1710,25 +1710,25 @@ def all_columns( .. code-block:: python - Band.select(Band.name, Band.manager.all_columns()).run_sync() + await Band.select(Band.name, Band.manager.all_columns()) # Equivalent to: - Band.select( + await Band.select( Band.name, Band.manager.id, Band.manager.name - ).run_sync() + ) To exclude certain columns: .. code-block:: python - Band.select( + await Band.select( Band.name, Band.manager.all_columns( exclude=[Band.manager.id] ) - ).run_sync() + ) :param exclude: Columns to exclude - can be the name of a column, or a column @@ -1771,14 +1771,14 @@ class Tour(Table): name = Varchar() concert = ForeignKey(Concert) - Tour.objects(Tour.concert, Tour.concert.all_related()).run_sync() + await Tour.objects(Tour.concert, Tour.concert.all_related()) # Equivalent to - Tour.objects( + await Tour.objects( Tour.concert, Tour.concert.band_1, Tour.concert.band_2 - ).run_sync() + ) :param exclude: Columns to exclude - can be the name of a column, or a @@ -2036,10 +2036,10 @@ class Token(Table): token = Bytea(default=b'token123') # Create - >>> Token(token=b'my-token').save().run_sync() + >>> await Token(token=b'my-token').save() # Query - >>> Token.select(Token.token).run_sync() + >>> await Token.select(Token.token) {'token': b'my-token'} """ @@ -2132,10 +2132,10 @@ class Ticket(Table): seat_numbers = Array(base_column=Integer()) # Create - >>> Ticket(seat_numbers=[34, 35, 36]).save().run_sync() + >>> await Ticket(seat_numbers=[34, 35, 36]).save() # Query - >>> Ticket.select(Ticket.seat_numbers).run_sync() + >>> await Ticket.select(Ticket.seat_numbers) {'seat_numbers': [34, 35, 36]} """ @@ -2186,7 +2186,7 @@ def __getitem__(self, value: int) -> Array: .. code-block:: python - >>> Ticket.select(Ticket.seat_numbers[0]).first().run_sync + >>> await Ticket.select(Ticket.seat_numbers[0]).first() {'seat_numbers': 325} @@ -2226,7 +2226,7 @@ def any(self, value: t.Any) -> Where: .. code-block:: python - >>> Ticket.select().where(Ticket.seat_numbers.any(510)).run_sync() + >>> await Ticket.select().where(Ticket.seat_numbers.any(510)) """ engine_type = self._meta.table._meta.db.engine_type @@ -2244,7 +2244,7 @@ def all(self, value: t.Any) -> Where: .. code-block:: python - >>> Ticket.select().where(Ticket.seat_numbers.all(510)).run_sync() + >>> await Ticket.select().where(Ticket.seat_numbers.all(510)) """ engine_type = self._meta.table._meta.db.engine_type diff --git a/piccolo/columns/combination.py b/piccolo/columns/combination.py index e03862c00..cb8520975 100644 --- a/piccolo/columns/combination.py +++ b/piccolo/columns/combination.py @@ -138,18 +138,18 @@ def clean_value(self, value: t.Any) -> t.Any: .. code-block:: python - manager = Manager.objects.where( + manager = await Manager.objects.where( Manager.name == 'Guido' - ).first().run_sync() + ).first() # The where clause should be: - Band.select().where(Band.manager.id == guido.id).run_sync() + await Band.select().where(Band.manager.id == guido.id) # Or - Band.select().where(Band.manager == guido.id).run_sync() + await Band.select().where(Band.manager == guido.id) # If the object is passed in, i.e. `guido` instead of `guido.id`, # it should still work. - Band.select().where(Band.manager == guido).run_sync() + await Band.select().where(Band.manager == guido) Also, convert Enums to their underlying values, and serialise any JSON. diff --git a/piccolo/conf/apps.py b/piccolo/conf/apps.py index e3f9e123d..c3921f640 100644 --- a/piccolo/conf/apps.py +++ b/piccolo/conf/apps.py @@ -187,10 +187,11 @@ def get_table_with_name(self, table_class_name: str) -> t.Type[Table]: @dataclass class AppRegistry: """ - Records all of the Piccolo apps in your project. Kept in piccolo_conf.py. + Records all of the Piccolo apps in your project. Kept in + ``piccolo_conf.py``. :param apps: - A list of paths to Piccolo apps, e.g. ['blog.piccolo_app'] + A list of paths to Piccolo apps, e.g. ``['blog.piccolo_app']``. """ diff --git a/piccolo/query/base.py b/piccolo/query/base.py index fc6382d9b..9b0d228f5 100644 --- a/piccolo/query/base.py +++ b/piccolo/query/base.py @@ -264,7 +264,7 @@ def freeze(self) -> FrozenQuery: # In the corresponding view/endpoint of whichever web framework # you're using: async def top_bands(self, request): - return await TOP_BANDS.run() + return await TOP_BANDS It means that Piccolo doesn't have to work as hard each time the query is run to generate the corresponding SQL - some of it is cached. If the diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 7e19f8b8e..12bf4d2db 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -79,18 +79,18 @@ class Count(Selectable): .. code-block:: python - Band.select(Band.name, Count()).group_by(Band.name).run() + await Band.select(Band.name, Count()).group_by(Band.name) # We can use an alias. These two are equivalent: - Band.select( + await Band.select( Band.name, Count(alias="total") - ).group_by(Band.name).run() + ).group_by(Band.name) - Band.select( + await Band.select( Band.name, Count().as_alias("total") - ).group_by(Band.name).run() + ).group_by(Band.name) """ diff --git a/piccolo/table.py b/piccolo/table.py index 4edef51d2..6f263f998 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -333,9 +333,9 @@ def save( .. code-block:: python - band = Band.objects().first().run_sync() + band = await Band.objects().first() band.popularity = 2000 - band.save(columns=[Band.popularity]).run_sync() + await band.save(columns=[Band.popularity]) If ``columns=None`` (the default) then all columns will be synced back to the database. diff --git a/piccolo/utils/dictionary.py b/piccolo/utils/dictionary.py index e5b26aed8..e15e477c3 100644 --- a/piccolo/utils/dictionary.py +++ b/piccolo/utils/dictionary.py @@ -12,7 +12,7 @@ def make_nested(dictionary: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]: .. code-block:: python - response = Band.select(Band.name, Band.manager.name).run_sync() + response = await Band.select(Band.name, Band.manager.name) >>> print(response) [{'name': 'Pythonistas', 'band.name': 'Guido'}] From c013e72d172b7e513cf309f3dc848c3b43436830 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 28 Jan 2022 09:48:09 +0000 Subject: [PATCH 243/727] tweak async docs --- docs/src/piccolo/getting_started/sync_and_async.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/src/piccolo/getting_started/sync_and_async.rst b/docs/src/piccolo/getting_started/sync_and_async.rst index a1f981cc5..12b7e67a7 100644 --- a/docs/src/piccolo/getting_started/sync_and_async.rst +++ b/docs/src/piccolo/getting_started/sync_and_async.rst @@ -22,14 +22,16 @@ You can await a query to run it: >>> await Band.select(Band.name) [{'name': 'Pythonistas'}] -If you need more control over how the query is executed, you can await the -``run`` method of a query: +Alternatively, you can await a query's ``run`` method: .. code-block:: python - >>> await Band.select(Band.name).run(in_pool=False) - [{'name': 'Pythonistas'}] + # This makes it extra explicit that a database query is being made: + >>> await Band.select(Band.name).run() + # It also gives you more control over how the query is run. + # For example, if we wanted to bypass the connection pool for some reason: + >>> await Band.select(Band.name).run(in_pool=False) Using the async version is useful for applications which require high throughput. Piccolo makes building an ASGI web app really simple - see From c15eb0ec0bc04ee69b75b7627bb1ef7eb5507e17 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 28 Jan 2022 10:01:46 +0000 Subject: [PATCH 244/727] bumped version --- CHANGES.rst | 42 ++++++++++++++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index cd018aaf0..852ad563b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,48 @@ Changes ======= +0.67.0 +------ + +create_user +~~~~~~~~~~~ + +``BaseUser`` now has a ``create_user`` method, which adds some extra password +validation vs just instantiating and saving ``BaseUser`` directly. + +.. code-block:: python + + >>> await BaseUser.create_user(username='bob', password='abc123XYZ') + + +We check that passwords are a reasonable length, and aren't already hashed. +See `PR 402 `_. + +async first +~~~~~~~~~~~ + +All of the docs have been updated to show the async version of queries. + +For example: + +.. code-block:: python + + # Previous: + Band.select().run_sync() + + # Now: + await Band.select() + +Most people use Piccolo in async apps, and the playground supports top level +await, so you can just paste in ``await Band.select()`` and it will still work. +See `PR 407 `_. + +We decided to use ``await Band.select()`` instead of ``await Band.select().run()``. +Both work, and have their merits, but the simpler version is probably easier +for newcomers. + +------------------------------------------------------------------------------- + 0.66.1 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index bdba0ec49..4b2618364 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.66.1" +__VERSION__ = "0.67.0" From ee38ccd27ae099eca42a62389694eb60bc548c62 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 31 Jan 2022 22:02:22 +0000 Subject: [PATCH 245/727] add `force` argument to `Update` queries (#412) * added `force` argument to `Update` queries * added docs about `force` and kwarg values for `Update` queries * bump mypy * fix black errors --- docs/src/piccolo/query_types/update.rst | 37 ++++++++++++++- piccolo/query/methods/delete.py | 3 +- piccolo/query/methods/update.py | 27 +++++++---- piccolo/table.py | 11 ++++- requirements/dev-requirements.txt | 4 +- tests/columns/test_bigint.py | 2 +- tests/columns/test_db_column_name.py | 4 +- tests/columns/test_json.py | 8 +++- tests/columns/test_reserved_column_names.py | 2 +- tests/columns/test_smallint.py | 2 +- tests/table/test_update.py | 50 ++++++++++++++------- 11 files changed, 112 insertions(+), 38 deletions(-) diff --git a/docs/src/piccolo/query_types/update.rst b/docs/src/piccolo/query_types/update.rst index 6b46ace67..64f6fa0ff 100644 --- a/docs/src/piccolo/query_types/update.rst +++ b/docs/src/piccolo/query_types/update.rst @@ -14,14 +14,31 @@ This is used to update any rows in the table which match the criteria. >>> ) [] -As well as replacing values with new ones, you can also modify existing values, for -instance by adding to an integer. +------------------------------------------------------------------------------- + +force +----- + +Piccolo won't let you run an update query without a where clause, unless you +explicitly tell it to do so. This is to prevent accidentally overwriting +the data in a table. + +.. code-block:: python + + >>> await Band.update() + Raises: UpdateError + + # Works fine: + >>> await Band.update({Band.popularity: 0}, force=True) ------------------------------------------------------------------------------- Modifying values ---------------- +As well as replacing values with new ones, you can also modify existing values, for +instance by adding to an integer. + Integer columns ~~~~~~~~~~~~~~~ @@ -81,6 +98,22 @@ You can currently only combine two values together at a time. ------------------------------------------------------------------------------- +Kwarg values +------------ + +Rather than passing in a dictionary of values, you can use kwargs instead if +you prefer: + +.. code-block:: python + + >>> await Band.update( + >>> name='Pythonistas 2' + >>> ).where( + >>> Band.name == 'Pythonistas' + >>> ) + +------------------------------------------------------------------------------- + Query clauses ------------- diff --git a/piccolo/query/methods/delete.py b/piccolo/query/methods/delete.py index f9e816e64..017bd17d2 100644 --- a/piccolo/query/methods/delete.py +++ b/piccolo/query/methods/delete.py @@ -37,7 +37,8 @@ def _validate(self): classname = self.table.__name__ raise DeletionError( "Do you really want to delete all the data from " - f"{classname}? If so, use {classname}.delete(force=True)." + f"{classname}? If so, use {classname}.delete(force=True). " + "Otherwise, add a where clause." ) @property diff --git a/piccolo/query/methods/update.py b/piccolo/query/methods/update.py index d2f029998..a6b0fbce0 100644 --- a/piccolo/query/methods/update.py +++ b/piccolo/query/methods/update.py @@ -12,12 +12,17 @@ from piccolo.table import Table +class UpdateError(Exception): + pass + + class Update(Query): - __slots__ = ("values_delegate", "where_delegate") + __slots__ = ("force", "values_delegate", "where_delegate") - def __init__(self, table: t.Type[Table], **kwargs): + def __init__(self, table: t.Type[Table], force: bool = False, **kwargs): super().__init__(table, **kwargs) + self.force = force self.values_delegate = ValuesDelegate(table=table) self.where_delegate = WhereDelegate() @@ -32,22 +37,26 @@ def where(self, *where: Combinable) -> Update: self.where_delegate.where(*where) return self - def validate(self): + def _validate(self): if len(self.values_delegate._values) == 0: - raise ValueError( - "No values were specified to update - please use .values" - ) + raise ValueError("No values were specified to update.") for column, _ in self.values_delegate._values.items(): if len(column._meta.call_chain) > 0: raise ValueError( - "Related values can't be updated via an update" + "Related values can't be updated via an update." ) + if (not self.where_delegate._where) and (not self.force): + classname = self.table.__name__ + raise UpdateError( + "Do you really want to update all rows in " + f"{classname}? If so, use pass `force=True` into " + f"`{classname}.update`. Otherwise, add a where clause." + ) + @property def default_querystrings(self) -> t.Sequence[QueryString]: - self.validate() - columns_str = ", ".join( f'"{col._meta.db_column_name}" = {{}}' for col, _ in self.values_delegate._values.items() diff --git a/piccolo/table.py b/piccolo/table.py index 6f263f998..a26f47e1f 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -955,7 +955,10 @@ def table_exists(cls) -> TableExists: @classmethod def update( - cls, values: t.Dict[t.Union[Column, str], t.Any] = {}, **kwargs + cls, + values: t.Dict[t.Union[Column, str], t.Any] = {}, + force: bool = False, + **kwargs, ) -> Update: """ Update rows. @@ -982,9 +985,13 @@ def update( Band.name=="Pythonistas" ).run() + :param force: + Unless set to ``True``, updates aren't allowed without a + ``where`` clause, to prevent accidental mass overriding of data. + """ values = dict(values, **kwargs) - return Update(table=cls).values(values) + return Update(table=cls, force=force).values(values) @classmethod def indexes(cls) -> Indexes: diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt index 281a72943..0eda59547 100644 --- a/requirements/dev-requirements.txt +++ b/requirements/dev-requirements.txt @@ -1,9 +1,9 @@ -black>=21.7b0 +black==22.1.0 ipdb==0.13.9 ipython==7.31.1 flake8==4.0.1 isort==5.10.1 twine==3.7.1 -mypy==0.930 +mypy==0.931 pip-upgrader==1.4.15 wheel==0.37.1 diff --git a/tests/columns/test_bigint.py b/tests/columns/test_bigint.py index 41dc71a2a..e7a54eb72 100644 --- a/tests/columns/test_bigint.py +++ b/tests/columns/test_bigint.py @@ -25,7 +25,7 @@ def tearDown(self): def _test_length(self): # Can store 8 bytes, but split between positive and negative values. - max_value = int(2 ** 64 / 2) - 1 + max_value = int(2**64 / 2) - 1 min_value = max_value * -1 print("Testing max value") diff --git a/tests/columns/test_db_column_name.py b/tests/columns/test_db_column_name.py index 1355186c6..b8da77156 100644 --- a/tests/columns/test_db_column_name.py +++ b/tests/columns/test_db_column_name.py @@ -114,7 +114,7 @@ def test_update(self): """ Band.objects().create(name="Pythonistas", popularity=1000).run_sync() - Band.update({Band.name: "Pythonistas 2"}).run_sync() + Band.update({Band.name: "Pythonistas 2"}, force=True).run_sync() bands = Band.select().run_sync() self.assertEqual( @@ -128,7 +128,7 @@ def test_update(self): ], ) - Band.update({"name": "Pythonistas 3"}).run_sync() + Band.update({"name": "Pythonistas 3"}, force=True).run_sync() bands = Band.select().run_sync() self.assertEqual( diff --git a/tests/columns/test_json.py b/tests/columns/test_json.py index 16663f56b..932c244d3 100644 --- a/tests/columns/test_json.py +++ b/tests/columns/test_json.py @@ -138,7 +138,9 @@ def test_json_update_string(self): Test updating a JSON field using a string. """ self.add_row() - MyTable.update({MyTable.json: '{"message": "updated"}'}).run_sync() + MyTable.update( + {MyTable.json: '{"message": "updated"}'}, force=True + ).run_sync() self.check_response() def test_json_update_object(self): @@ -146,5 +148,7 @@ def test_json_update_object(self): Test updating a JSON field using an object. """ self.add_row() - MyTable.update({MyTable.json: {"message": "updated"}}).run_sync() + MyTable.update( + {MyTable.json: {"message": "updated"}}, force=True + ).run_sync() self.check_response() diff --git a/tests/columns/test_reserved_column_names.py b/tests/columns/test_reserved_column_names.py index 77a56b9ae..71cedacc8 100644 --- a/tests/columns/test_reserved_column_names.py +++ b/tests/columns/test_reserved_column_names.py @@ -46,7 +46,7 @@ def test_common_operations(self): ) # Update - Concert.update({Concert.order: 3}).run_sync() + Concert.update({Concert.order: 3}, force=True).run_sync() self.assertEqual( Concert.select(Concert.order).run_sync(), [{"order": 3}], diff --git a/tests/columns/test_smallint.py b/tests/columns/test_smallint.py index 45b8ff683..229d66571 100644 --- a/tests/columns/test_smallint.py +++ b/tests/columns/test_smallint.py @@ -25,7 +25,7 @@ def tearDown(self): def _test_length(self): # Can store 2 bytes, but split between positive and negative values. - max_value = int(2 ** 16 / 2) - 1 + max_value = int(2**16 / 2) - 1 min_value = max_value * -1 print("Testing max value") diff --git a/tests/table/test_update.py b/tests/table/test_update.py index d2b2e3280..a57966357 100644 --- a/tests/table/test_update.py +++ b/tests/table/test_update.py @@ -93,7 +93,9 @@ class TestIntUpdateOperators(DBTestCase): def test_add(self): self.insert_row() - Band.update({Band.popularity: Band.popularity + 10}).run_sync() + Band.update( + {Band.popularity: Band.popularity + 10}, force=True + ).run_sync() response = Band.select(Band.popularity).first().run_sync() @@ -103,7 +105,7 @@ def test_add_column(self): self.insert_row() Band.update( - {Band.popularity: Band.popularity + Band.popularity} + {Band.popularity: Band.popularity + Band.popularity}, force=True ).run_sync() response = Band.select(Band.popularity).first().run_sync() @@ -113,7 +115,9 @@ def test_add_column(self): def test_radd(self): self.insert_row() - Band.update({Band.popularity: 10 + Band.popularity}).run_sync() + Band.update( + {Band.popularity: 10 + Band.popularity}, force=True + ).run_sync() response = Band.select(Band.popularity).first().run_sync() @@ -122,7 +126,9 @@ def test_radd(self): def test_sub(self): self.insert_row() - Band.update({Band.popularity: Band.popularity - 10}).run_sync() + Band.update( + {Band.popularity: Band.popularity - 10}, force=True + ).run_sync() response = Band.select(Band.popularity).first().run_sync() @@ -131,7 +137,9 @@ def test_sub(self): def test_rsub(self): self.insert_row() - Band.update({Band.popularity: 1100 - Band.popularity}).run_sync() + Band.update( + {Band.popularity: 1100 - Band.popularity}, force=True + ).run_sync() response = Band.select(Band.popularity).first().run_sync() @@ -140,7 +148,9 @@ def test_rsub(self): def test_mul(self): self.insert_row() - Band.update({Band.popularity: Band.popularity * 2}).run_sync() + Band.update( + {Band.popularity: Band.popularity * 2}, force=True + ).run_sync() response = Band.select(Band.popularity).first().run_sync() @@ -149,7 +159,9 @@ def test_mul(self): def test_rmul(self): self.insert_row() - Band.update({Band.popularity: 2 * Band.popularity}).run_sync() + Band.update( + {Band.popularity: 2 * Band.popularity}, force=True + ).run_sync() response = Band.select(Band.popularity).first().run_sync() @@ -158,7 +170,9 @@ def test_rmul(self): def test_div(self): self.insert_row() - Band.update({Band.popularity: Band.popularity / 10}).run_sync() + Band.update( + {Band.popularity: Band.popularity / 10}, force=True + ).run_sync() response = Band.select(Band.popularity).first().run_sync() @@ -167,7 +181,9 @@ def test_div(self): def test_rdiv(self): self.insert_row() - Band.update({Band.popularity: 1000 / Band.popularity}).run_sync() + Band.update( + {Band.popularity: 1000 / Band.popularity}, force=True + ).run_sync() response = Band.select(Band.popularity).first().run_sync() @@ -178,7 +194,7 @@ class TestVarcharUpdateOperators(DBTestCase): def test_add(self): self.insert_row() - Band.update({Band.name: Band.name + "!!!"}).run_sync() + Band.update({Band.name: Band.name + "!!!"}, force=True).run_sync() response = Band.select(Band.name).first().run_sync() @@ -187,7 +203,7 @@ def test_add(self): def test_add_column(self): self.insert_row() - Band.update({Band.name: Band.name + Band.name}).run_sync() + Band.update({Band.name: Band.name + Band.name}, force=True).run_sync() response = Band.select(Band.name).first().run_sync() @@ -196,7 +212,7 @@ def test_add_column(self): def test_radd(self): self.insert_row() - Band.update({Band.name: "!!!" + Band.name}).run_sync() + Band.update({Band.name: "!!!" + Band.name}, force=True).run_sync() response = Band.select(Band.name).first().run_sync() @@ -209,7 +225,9 @@ def setUp(self): Poster(content="Join us for this amazing show").save().run_sync() def test_add(self): - Poster.update({Poster.content: Poster.content + "!!!"}).run_sync() + Poster.update( + {Poster.content: Poster.content + "!!!"}, force=True + ).run_sync() response = Poster.select(Poster.content).first().run_sync() @@ -221,7 +239,7 @@ def test_add_column(self): self.insert_row() Poster.update( - {Poster.content: Poster.content + Poster.content} + {Poster.content: Poster.content + Poster.content}, force=True ).run_sync() response = Poster.select(Poster.content).first().run_sync() @@ -234,7 +252,9 @@ def test_add_column(self): def test_radd(self): self.insert_row() - Poster.update({Poster.content: "!!!" + Poster.content}).run_sync() + Poster.update( + {Poster.content: "!!!" + Poster.content}, force=True + ).run_sync() response = Poster.select(Poster.content).first().run_sync() From a6542546b8f7a5806618b493e33807c431ec5b2a Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 1 Feb 2022 22:04:59 +0000 Subject: [PATCH 246/727] fix `JSONB` column where null (#413) * fix JSON column where null * improve JSONB tests * bump dev requirements and fix black * Update doc-requirements.txt * change ipython version * make None always mean SQL null with JSON columns * improve JSON docs * added test for raw queries with JSON --- docs/src/piccolo/engines/index.rst | 2 +- .../getting_started/example_schema.rst | 31 +++- docs/src/piccolo/query_clauses/output.rst | 2 + .../piccolo/query_types/django_comparison.rst | 2 +- docs/src/piccolo/schema/column_types.rst | 114 ++++++++++++-- piccolo/apps/playground/commands/run.py | 2 +- piccolo/columns/base.py | 4 +- piccolo/columns/column_types.py | 17 +- piccolo/columns/combination.py | 8 +- piccolo/query/base.py | 5 + piccolo/utils/sql_values.py | 5 +- requirements/dev-requirements.txt | 2 +- requirements/doc-requirements.txt | 2 +- tests/columns/test_jsonb.py | 147 +++++++++++++++--- 14 files changed, 291 insertions(+), 52 deletions(-) diff --git a/docs/src/piccolo/engines/index.rst b/docs/src/piccolo/engines/index.rst index 1690dc6c6..2c6f191c4 100644 --- a/docs/src/piccolo/engines/index.rst +++ b/docs/src/piccolo/engines/index.rst @@ -75,7 +75,7 @@ In your terminal: export PICCOLO_CONF=piccolo_conf_test -Or at the entypoint of your app, before any other imports: +Or at the entrypoint of your app, before any other imports: .. code-block:: python diff --git a/docs/src/piccolo/getting_started/example_schema.rst b/docs/src/piccolo/getting_started/example_schema.rst index d55fa39f3..1a33a418c 100644 --- a/docs/src/piccolo/getting_started/example_schema.rst +++ b/docs/src/piccolo/getting_started/example_schema.rst @@ -3,7 +3,10 @@ Example Schema ============== -This is the schema used by the example queries throughout the docs. +This is the schema used by the example queries throughout the docs, and also +in the :ref:`playground`. + +``Manager`` and ``Band`` are most commonly used: .. code-block:: python @@ -20,4 +23,30 @@ This is the schema used by the example queries throughout the docs. manager = ForeignKey(references=Manager) popularity = Integer() +We sometimes use these other tables in the examples too: + +.. code-block:: python + + class Venue(Table): + name = Varchar() + capacity = Integer() + + + class Concert(Table): + band_1 = ForeignKey(references=Band) + band_2 = ForeignKey(references=Band) + venue = ForeignKey(references=Venue) + starts = Timestamp() + duration = Interval() + + + class Ticket(Table): + concert = ForeignKey(references=Concert) + price = Numeric() + + + class RecordingStudio(Table): + name = Varchar() + facilities = JSONB() + To understand more about defining your own schemas, see :ref:`DefiningSchema`. diff --git a/docs/src/piccolo/query_clauses/output.rst b/docs/src/piccolo/query_clauses/output.rst index 6c3a422c3..e4eacefb6 100644 --- a/docs/src/piccolo/query_clauses/output.rst +++ b/docs/src/piccolo/query_clauses/output.rst @@ -54,6 +54,8 @@ Output any data from related tables in nested dictionaries. Select and Objects queries -------------------------- +.. _load_json: + load_json ~~~~~~~~~ diff --git a/docs/src/piccolo/query_types/django_comparison.rst b/docs/src/piccolo/query_types/django_comparison.rst index 7f5b3df69..bf707c14a 100644 --- a/docs/src/piccolo/query_types/django_comparison.rst +++ b/docs/src/piccolo/query_types/django_comparison.rst @@ -173,7 +173,7 @@ Piccolo has something similar: .. code-block:: python # Piccolo - band = Band.objects(Band.manager).get(name='Pythonistas') + band = Band.objects(Band.manager).get(Band.name == 'Pythonistas').run_sync() >>> band.manager diff --git a/docs/src/piccolo/schema/column_types.rst b/docs/src/piccolo/schema/column_types.rst index fc2d3e63c..89db19443 100644 --- a/docs/src/piccolo/schema/column_types.rst +++ b/docs/src/piccolo/schema/column_types.rst @@ -192,36 +192,116 @@ JSONB .. autoclass:: JSONB +Serialising +=========== + +Piccolo automatically converts Python values into JSON strings: + +.. code-block:: python + + studio = RecordingStudio( + name="Abbey Road", + facilities={"restaurant": True, "mixing_desk": True} # Automatically serialised + ) + await studio.save() + +You can also pass in a JSON string if you prefer: + +.. code-block:: python + + studio = RecordingStudio( + name="Abbey Road", + facilities='{"restaurant": true, "mixing_desk": true}' + ) + await studio.save() + +Deserialising +============= + +The contents of a ``JSON`` / ``JSONB`` column are returned as a string by +default: + +.. code-block:: python + + >>> await RecordingStudio.select(RecordingStudio.facilities) + [{facilities: '{"restaurant": true, "mixing_desk": true}'}] + +However, we can ask Piccolo to deserialise the JSON automatically (see :ref:`load_json`): + +.. code-block:: python + + >>> await RecordingStudio.select( + >>> RecordingStudio.facilities + >>> ).output( + >>> load_json=True + >>> ) + [facilities: {"restaurant": True, "mixing_desk": True}}] + +With ``objects`` queries, we can modify the returned JSON, and then save it: + +.. code-block:: python + + studio = await RecordingStudio.objects().get( + RecordingStudio.name == 'Abbey Road' + ).output(load_json=True) + + studio['facilities']['restaurant'] = False + await studio.save() + arrow ===== ``JSONB`` columns have an ``arrow`` function, which is useful for retrieving -a subset of the JSON data, and for filtering in a where clause. +a subset of the JSON data: .. code-block:: python - # Example schema: - class Booking(Table): - data = JSONB() + >>> await RecordingStudio.select( + >>> RecordingStudio.name, + >>> RecordingStudio.facilities.arrow('mixing_desk').as_alias('mixing_desk') + >>> ).output(load_json=True) + [{'name': 'Abbey Road', 'mixing_desk': True}] - await Booking.create_table() +It can also be used for filtering in a where clause: - # Example data: - await Booking.insert( - Booking(data='{"name": "Alison"}'), - Booking(data='{"name": "Bob"}') - ) +.. code-block:: python + + >>> await RecordingStudio.select(RecordingStudio.name).where( + >>> RecordingStudio.facilities.arrow('mixing_desk') == True + >>> ) + [{'name': 'Abbey Road'}] - # Example queries - >>> await Booking.select( - >>> Booking.id, Booking.data.arrow('name').as_alias('name') +Handling null +============= + +When assigning a value of ``None`` to a ``JSON`` or ``JSONB`` column, this is +treated as null in the database. + +.. code-block:: python + + await RecordingStudio(name="ABC Studios", facilities=None).save() + + >>> await RecordingStudio.select( + >>> RecordingStudio.facilities + >>> ).where( + >>> RecordingStudio.name == "ABC Studios" >>> ) - [{'id': 1, 'name': '"Alison"'}, {'id': 2, 'name': '"Bob"'}] + [{'facilities': None}] + + +If instead you want to store JSON null in the database, assign a value of ``'null'`` +instead. + +.. code-block:: python + + await RecordingStudio(name="ABC Studios", facilities='null').save() - >>> await Booking.select(Booking.id).where( - >>> Booking.data.arrow('name') == '"Alison"' + >>> await RecordingStudio.select( + >>> RecordingStudio.facilities + >>> ).where( + >>> RecordingStudio.name == "ABC Studios" >>> ) - [{'id': 1}] + [{'facilities': 'null'}] ------------------------------------------------------------------------------- diff --git a/piccolo/apps/playground/commands/run.py b/piccolo/apps/playground/commands/run.py index 31a28d631..38965da76 100644 --- a/piccolo/apps/playground/commands/run.py +++ b/piccolo/apps/playground/commands/run.py @@ -58,7 +58,7 @@ class DiscountCode(Table): class RecordingStudio(Table): name = Varchar(length=100) - facilities = JSON() + facilities = JSON(null=True) TABLES = (Manager, Band, Venue, Concert, Ticket, DiscountCode, RecordingStudio) diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 195686cd6..d6ee2a372 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -668,14 +668,14 @@ def __hash__(self): def is_null(self) -> Where: """ Can be used instead of ``MyTable.column != None``, because some linters - don't like a comparison to None. + don't like a comparison to ``None``. """ return Where(column=self, operator=IsNull) def is_not_null(self) -> Where: """ Can be used instead of ``MyTable.column == None``, because some linters - don't like a comparison to None. + don't like a comparison to ``None``. """ return Where(column=self, operator=IsNotNull) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 7e28ab094..60d841524 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -1996,13 +1996,28 @@ def get_select_string(self, engine_type: str, just_alias=False) -> str: just_alias=just_alias, include_quotes=True ) if self.json_operator is None: - return select_string + if self.alias is None: + return select_string + else: + return f"{select_string} AS {self.alias}" else: if self.alias is None: return f"{select_string} {self.json_operator}" else: return f"{select_string} {self.json_operator} AS {self.alias}" + def eq(self, value) -> Where: + """ + See ``Boolean.eq`` for more details. + """ + return self.__eq__(value) + + def ne(self, value) -> Where: + """ + See ``Boolean.ne`` for more details. + """ + return self.__ne__(value) + ########################################################################### # Descriptors diff --git a/piccolo/columns/combination.py b/piccolo/columns/combination.py index cb8520975..f98346afd 100644 --- a/piccolo/columns/combination.py +++ b/piccolo/columns/combination.py @@ -122,7 +122,11 @@ def __init__( omitted, vs None, which is a valid value for a where clause. """ self.column = column - self.value = self.clean_value(value) + + if value == UNDEFINED: + self.value = value + else: + self.value = self.clean_value(value) if values == UNDEFINED: self.values = values @@ -133,7 +137,7 @@ def __init__( def clean_value(self, value: t.Any) -> t.Any: """ - If a where clause contains a Table instance, we should convert that + If a where clause contains a ``Table`` instance, we should convert that to a column reference. For example: .. code-block:: python diff --git a/piccolo/query/base.py b/piccolo/query/base.py index 9b0d228f5..1573b3405 100644 --- a/piccolo/query/base.py +++ b/piccolo/query/base.py @@ -80,6 +80,11 @@ async def _process_results(self, results): # noqa: C901 for column in json_columns: if column.alias is not None: json_column_names.append(column.alias) + elif column.json_operator is not None: + # If no alias is specified, then the default column name + # that Postgres gives when using the `->` operator is + # `?column?`. + json_column_names.append("?column?") elif len(column._meta.call_chain) > 0: json_column_names.append( column.get_select_string( diff --git a/piccolo/utils/sql_values.py b/piccolo/utils/sql_values.py index 766d59e00..b61be6fd1 100644 --- a/piccolo/utils/sql_values.py +++ b/piccolo/utils/sql_values.py @@ -34,6 +34,9 @@ def convert_to_sql_value(value: t.Any, column: Column) -> t.Any: elif isinstance(value, Enum): return value.value elif isinstance(column, (JSON, JSONB)) and not isinstance(value, str): - return dump_json(value) + if value is None: + return None + else: + return dump_json(value) else: return value diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt index 0eda59547..7c812f452 100644 --- a/requirements/dev-requirements.txt +++ b/requirements/dev-requirements.txt @@ -1,6 +1,6 @@ black==22.1.0 ipdb==0.13.9 -ipython==7.31.1 +ipython>=7.31.1 flake8==4.0.1 isort==5.10.1 twine==3.7.1 diff --git a/requirements/doc-requirements.txt b/requirements/doc-requirements.txt index 453dac60c..cfc1d6a37 100644 --- a/requirements/doc-requirements.txt +++ b/requirements/doc-requirements.txt @@ -1,3 +1,3 @@ -Sphinx==4.3.1 +Sphinx==4.4.0 sphinx-rtd-theme==1.0.0 livereload==2.6.3 diff --git a/tests/columns/test_jsonb.py b/tests/columns/test_jsonb.py index b5bfe923b..fab2b7d10 100644 --- a/tests/columns/test_jsonb.py +++ b/tests/columns/test_jsonb.py @@ -1,76 +1,177 @@ from unittest import TestCase -from piccolo.columns.column_types import JSONB +from piccolo.columns.column_types import JSONB, Varchar from piccolo.table import Table +from tests.base import postgres_only -from ..base import postgres_only - -class MyTable(Table): - json = JSONB() +class RecordingStudio(Table): + name = Varchar() + facilities = JSONB(null=True) @postgres_only class TestJSONB(TestCase): def setUp(self): - MyTable.create_table().run_sync() + RecordingStudio.create_table().run_sync() def tearDown(self): - MyTable.alter().drop_table().run_sync() + RecordingStudio.alter().drop_table().run_sync() def test_json(self): """ Test storing a valid JSON string. """ - row = MyTable(json='{"a": 1}') + row = RecordingStudio( + name="Abbey Road", facilities='{"mixing_desk": true}' + ) row.save().run_sync() - self.assertEqual(row.json, '{"a": 1}') + self.assertEqual(row.facilities, '{"mixing_desk": true}') + + def test_raw(self): + """ + Make sure raw queries convert the Python value into a JSON string. + """ + RecordingStudio.raw( + "INSERT INTO recording_studio (name, facilities) VALUES ({}, {})", + "Abbey Road", + '{"mixing_desk": true}', + ).run_sync() + + self.assertEqual( + RecordingStudio.select().run_sync(), + [ + { + "id": 1, + "name": "Abbey Road", + "facilities": '{"mixing_desk": true}', + } + ], + ) + + def test_where(self): + """ + Test using the where clause to match a subset of rows. + """ + RecordingStudio.insert( + RecordingStudio( + name="Abbey Road", facilities={"mixing_desk": True} + ), + RecordingStudio(name="ABC Studio", facilities=None), + ).run_sync() + + self.assertEqual( + RecordingStudio.select(RecordingStudio.name) + .where(RecordingStudio.facilities == {"mixing_desk": True}) + .run_sync(), + [{"name": "Abbey Road"}], + ) + + self.assertEqual( + RecordingStudio.select(RecordingStudio.name) + .where(RecordingStudio.facilities == '{"mixing_desk": true}') + .run_sync(), + [{"name": "Abbey Road"}], + ) + + self.assertEqual( + RecordingStudio.select(RecordingStudio.name) + .where(RecordingStudio.facilities.is_null()) + .run_sync(), + [{"name": "ABC Studio"}], + ) + + self.assertEqual( + RecordingStudio.select(RecordingStudio.name) + .where(RecordingStudio.facilities.is_not_null()) + .run_sync(), + [{"name": "Abbey Road"}], + ) def test_arrow(self): """ Test using the arrow function to retrieve a subset of the JSON. """ - MyTable(json='{"a": 1}').save().run_sync() - row = MyTable.select(MyTable.json.arrow("a")).first().run_sync() - self.assertEqual(row["?column?"], "1") + RecordingStudio( + name="Abbey Road", facilities='{"mixing_desk": true}' + ).save().run_sync() + + row = ( + RecordingStudio.select( + RecordingStudio.facilities.arrow("mixing_desk") + ) + .first() + .run_sync() + ) + self.assertEqual(row["?column?"], "true") + + row = ( + RecordingStudio.select( + RecordingStudio.facilities.arrow("mixing_desk") + ) + .output(load_json=True) + .first() + .run_sync() + ) + self.assertEqual(row["?column?"], True) def test_arrow_as_alias(self): """ Test using the arrow function to retrieve a subset of the JSON. """ - MyTable(json='{"a": 1}').save().run_sync() + RecordingStudio( + name="Abbey Road", facilities='{"mixing_desk": true}' + ).save().run_sync() + row = ( - MyTable.select(MyTable.json.arrow("a").as_alias("a")) + RecordingStudio.select( + RecordingStudio.facilities.arrow("mixing_desk").as_alias( + "mixing_desk" + ) + ) .first() .run_sync() ) - self.assertEqual(row["a"], "1") + self.assertEqual(row["mixing_desk"], "true") def test_arrow_where(self): """ Make sure the arrow function can be used within a WHERE clause. """ - MyTable(json='{"a": 1}').save().run_sync() + RecordingStudio( + name="Abbey Road", facilities='{"mixing_desk": true}' + ).save().run_sync() + self.assertEqual( - MyTable.count().where(MyTable.json.arrow("a") == "1").run_sync(), 1 + RecordingStudio.count() + .where(RecordingStudio.facilities.arrow("mixing_desk").eq(True)) + .run_sync(), + 1, ) self.assertEqual( - MyTable.count().where(MyTable.json.arrow("a") == "2").run_sync(), 0 + RecordingStudio.count() + .where(RecordingStudio.facilities.arrow("mixing_desk").eq(False)) + .run_sync(), + 0, ) def test_arrow_first(self): """ Make sure the arrow function can be used with the first clause. """ - MyTable.insert( - MyTable(json='{"a": 1}'), - MyTable(json='{"b": 2}'), + RecordingStudio.insert( + RecordingStudio(facilities='{"mixing_desk": true}'), + RecordingStudio(facilities='{"mixing_desk": false}'), ).run_sync() self.assertEqual( - MyTable.select(MyTable.json.arrow("a").as_alias("json")) + RecordingStudio.select( + RecordingStudio.facilities.arrow("mixing_desk").as_alias( + "mixing_desk" + ) + ) .first() .run_sync(), - {"json": "1"}, + {"mixing_desk": "true"}, ) From 098cd4e99a82ed057f7fbf980e83b929e610784b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 2 Feb 2022 14:44:04 +0000 Subject: [PATCH 247/727] bumped version --- CHANGES.rst | 43 +++++++++++++++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 852ad563b..04082b299 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,49 @@ Changes ======= +0.68.0 +------ + +``Update`` queries without a ``where`` clause +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you try and perform an update query without a ``where`` clause you will now +get an error: + +.. code-block:: python + + >>> await Band.update({Band.name: 'New Band'}) + UpdateError + +If you want to update all rows in the table, you can still do so, but you must +pass ``force=True``. + +.. code-block:: python + + >>> await Band.update({Band.name: 'New Band'}, force=True) + +This is a similar to ``delete`` queries, which require a ``where`` clause or +``force=True``. + +It was pointed out by @theelderbeever that an accidental mass update is almost +as bad as a mass deletion, which is why this safety measure has been added. + +See `PR 412 `_. + +.. warning:: This is a breaking change. It you're doing update queries without + a where clause, you will need to add ``force=True``. + +``JSONB`` improvements +~~~~~~~~~~~~~~~~~~~~~~ + +Fixed some bugs with nullable ``JSONB`` columns. A value of ``None`` is now +stored as ``null`` in the database, instead of the JSON string ``'null'``. +Thanks to @theelderbeever for reporting this. + +See `PR 413 `_. + +------------------------------------------------------------------------------- + 0.67.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 4b2618364..ca10f407b 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.67.0" +__VERSION__ = "0.68.0" From a53f578358e01b5622c35cb6b89180fb68a41e5c Mon Sep 17 00:00:00 2001 From: sinisaos Date: Thu, 3 Feb 2022 17:22:25 +0100 Subject: [PATCH 248/727] asgi template for Xpresso framework (#415) * asgi template for Xpresso framework * add lifespan context managers * apply suggestions and update readme.md * add xpresso and blacksheep to integration test, and modernise older ASGI templates Co-authored-by: Daniel Townsend --- README.md | 2 +- docs/src/piccolo/asgi/index.rst | 3 +- piccolo/apps/asgi/commands/new.py | 2 +- .../templates/app/_blacksheep_app.py.jinja | 44 +++---- .../templates/app/_fastapi_app.py.jinja | 20 ++-- .../templates/app/_xpresso_app.py.jinja | 113 ++++++++++++++++++ .../asgi/commands/templates/app/app.py.jinja | 2 + .../app/home/_blacksheep_endpoints.py.jinja | 2 +- .../app/home/_xpresso_endpoints.py.jinja | 20 ++++ .../templates/app/home/endpoints.py.jinja | 2 + .../app/home/templates/home.html.jinja_raw | 5 + tests/apps/asgi/commands/test_new.py | 64 +++++----- 12 files changed, 212 insertions(+), 67 deletions(-) create mode 100644 piccolo/apps/asgi/commands/templates/app/_xpresso_app.py.jinja create mode 100644 piccolo/apps/asgi/commands/templates/app/home/_xpresso_endpoints.py.jinja diff --git a/README.md b/README.md index 6433ea0ac..bf53d082a 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ Let Piccolo scaffold you an ASGI web app, using Piccolo as the ORM: piccolo asgi new ``` -[Starlette](https://www.starlette.io/), [FastAPI](https://fastapi.tiangolo.com/), and [BlackSheep](https://www.neoteroi.dev/blacksheep/) are currently supported. +[Starlette](https://www.starlette.io/), [FastAPI](https://fastapi.tiangolo.com/), [BlackSheep](https://www.neoteroi.dev/blacksheep/) and [Xpresso](https://xpresso-api.dev/) are currently supported. ## Are you a Django user? diff --git a/docs/src/piccolo/asgi/index.rst b/docs/src/piccolo/asgi/index.rst index 4c00e340e..b930392d0 100644 --- a/docs/src/piccolo/asgi/index.rst +++ b/docs/src/piccolo/asgi/index.rst @@ -21,7 +21,8 @@ Routing frameworks ****************** Currently, `Starlette `_, `FastAPI `_, -and `BlackSheep `_ are supported. +`BlackSheep `_ and `Xpresso `_ +are supported. Other great ASGI routing frameworks exist, and may be supported in the future (`Quart `_ , diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index 3b7a2482d..8e66e0410 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -10,7 +10,7 @@ TEMPLATE_DIR = os.path.join(os.path.dirname(__file__), "templates/app/") SERVERS = ["uvicorn", "Hypercorn"] -ROUTERS = ["starlette", "fastapi", "blacksheep"] +ROUTERS = ["starlette", "fastapi", "blacksheep", "xpresso"] def print_instruction(message: str): diff --git a/piccolo/apps/asgi/commands/templates/app/_blacksheep_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_blacksheep_app.py.jinja index fd4db076f..7c20f05b4 100644 --- a/piccolo/apps/asgi/commands/templates/app/_blacksheep_app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/_blacksheep_app.py.jinja @@ -47,56 +47,56 @@ TaskModelPartial: t.Any = create_pydantic_model( @app.router.get("/tasks/") async def tasks() -> t.List[TaskModelOut]: - return await Task.select().order_by(Task.id).run() + return await Task.select().order_by(Task.id) @app.router.post("/tasks/") -async def create_task(task: FromJSON[TaskModelIn]) -> TaskModelOut: - task = Task(**task.value.__dict__) - await task.save().run() - return TaskModelOut(**task.__dict__) +async def create_task(task_model: FromJSON[TaskModelIn]) -> TaskModelOut: + task = Task(**task_model.value.dict()) + await task.save() + return TaskModelOut(**task.to_dict()) @app.router.put("/tasks/{task_id}/") async def put_task( - task_id: int, task: FromJSON[TaskModelIn] + task_id: int, task_model: FromJSON[TaskModelIn] ) -> TaskModelOut: - _task = await Task.objects().where(Task.id == task_id).first().run() - if not _task: + task = await Task.objects().get(Task.id == task_id) + if not task: return json({}, status=404) - for key, value in task.value.__dict__.items(): - setattr(_task, key, value) + for key, value in task_model.value.dict().items(): + setattr(task, key, value) - await _task.save().run() + await task.save() - return TaskModelOut(**_task.__dict__) + return TaskModelOut(**task.to_dict()) @app.router.patch("/tasks/{task_id}/") async def patch_task( - task_id: int, task: FromJSON[TaskModelPartial] + task_id: int, task_model: FromJSON[TaskModelPartial] ) -> TaskModelOut: - _task = await Task.objects().where(Task.id == task_id).first().run() - if not _task: + task = await Task.objects().get(Task.id == task_id) + if not task: return json({}, status=404) - for key, value in task.value.__dict__.items(): - if value is not None: - setattr(_task, key, value) + for key, value in task_model.value.dict().items(): + if value is not None: + setattr(task, key, value) - await _task.save().run() + await task.save() - return TaskModelOut(**_task.__dict__) + return TaskModelOut(**task.to_dict()) @app.router.delete("/tasks/{task_id}/") async def delete_task(task_id: int): - task = await Task.objects().where(Task.id == task_id).first().run() + task = await Task.objects().get(Task.id == task_id) if not task: return json({}, status=404) - await task.remove().run() + await task.remove() return json({}) diff --git a/piccolo/apps/asgi/commands/templates/app/_fastapi_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_fastapi_app.py.jinja index 0015ebf79..c2408223b 100644 --- a/piccolo/apps/asgi/commands/templates/app/_fastapi_app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/_fastapi_app.py.jinja @@ -1,11 +1,11 @@ import typing as t +from fastapi import FastAPI +from fastapi.responses import JSONResponse from piccolo_admin.endpoints import create_admin from piccolo_api.crud.serializers import create_pydantic_model from piccolo.engine import engine_finder from starlette.routing import Route, Mount -from fastapi import FastAPI -from fastapi.responses import JSONResponse from starlette.staticfiles import StaticFiles from home.endpoints import HomeEndpoint @@ -42,37 +42,37 @@ TaskModelOut: t.Any = create_pydantic_model( @app.get("/tasks/", response_model=t.List[TaskModelOut]) async def tasks(): - return await Task.select().order_by(Task.id).run() + return await Task.select().order_by(Task.id) @app.post('/tasks/', response_model=TaskModelOut) async def create_task(task_model: TaskModelIn): - task = Task(**task_model.__dict__) - await task.save().run() + task = Task(**task_model.dict()) + await task.save() return task.to_dict() @app.put('/tasks/{task_id}/', response_model=TaskModelOut) async def update_task(task_id: int, task_model: TaskModelIn): - task = await Task.objects().where(Task.id == task_id).first().run() + task = await Task.objects().get(Task.id == task_id) if not task: return JSONResponse({}, status_code=404) - for key, value in task_model.__dict__.items(): + for key, value in task_model.dict().items(): setattr(task, key, value) - await task.save().run() + await task.save() return task.to_dict() @app.delete('/tasks/{task_id}/') async def delete_task(task_id: int): - task = await Task.objects().where(Task.id == task_id).first().run() + task = await Task.objects().get(Task.id == task_id) if not task: return JSONResponse({}, status_code=404) - await task.remove().run() + await task.remove() return JSONResponse({}) diff --git a/piccolo/apps/asgi/commands/templates/app/_xpresso_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_xpresso_app.py.jinja new file mode 100644 index 000000000..09010d199 --- /dev/null +++ b/piccolo/apps/asgi/commands/templates/app/_xpresso_app.py.jinja @@ -0,0 +1,113 @@ +import typing as t +from contextlib import asynccontextmanager + +from piccolo.engine import engine_finder +from piccolo.utils.pydantic import create_pydantic_model +from piccolo_admin.endpoints import create_admin +from starlette.staticfiles import StaticFiles +from xpresso import App, FromJson, FromPath, HTTPException, Operation, Path +from xpresso.routing.mount import Mount + +from home.endpoints import home +from home.piccolo_app import APP_CONFIG +from home.tables import Task + +TaskModelIn: t.Any = create_pydantic_model(table=Task, model_name="TaskModelIn") +TaskModelOut: t.Any = create_pydantic_model( + table=Task, include_default_columns=True, model_name="TaskModelOut" +) + + +async def tasks() -> t.List[TaskModelOut]: + return await Task.select().order_by(Task.id) + + +async def create_task(task_model: FromJson[TaskModelIn]) -> TaskModelOut: + task = Task(**task_model.dict()) + await task.save() + return task.to_dict() + + +async def update_task( + task_id: FromPath[int], task_model: FromJson[TaskModelIn] +) -> TaskModelOut: + task = await Task.objects().get(Task.id == task_id) + if not task: + raise HTTPException(status_code=404) + + for key, value in task_model.dict().items(): + setattr(task, key, value) + + await task.save() + + return task.to_dict() + + +async def delete_task(task_id: FromPath[int]): + task = await Task.objects().get(Task.id == task_id) + if not task: + raise HTTPException(status_code=404) + + await task.remove() + + return {} + + +@asynccontextmanager +async def lifespan(): + await open_database_connection_pool() + try: + yield + finally: + await close_database_connection_pool() + + +app = App( + routes=[ + Path( + "/", + get=Operation( + home, + include_in_schema=False, + ), + ), + Mount( + "/admin/", + create_admin( + tables=APP_CONFIG.table_classes, + # Required when running under HTTPS: + # allowed_hosts=['my_site.com'] + ), + ), + Path( + "/tasks/", + get=tasks, + post=create_task, + tags=["Task"], + ), + Path( + "/tasks/{task_id}/", + put=update_task, + delete=delete_task, + tags=["Task"], + ), + Mount("/static/", StaticFiles(directory="static")), + ], + lifespan=lifespan, +) + + +async def open_database_connection_pool(): + try: + engine = engine_finder() + await engine.start_connection_pool() + except Exception: + print("Unable to connect to the database") + + +async def close_database_connection_pool(): + try: + engine = engine_finder() + await engine.close_connection_pool() + except Exception: + print("Unable to connect to the database") diff --git a/piccolo/apps/asgi/commands/templates/app/app.py.jinja b/piccolo/apps/asgi/commands/templates/app/app.py.jinja index c21b3e029..18e45a23b 100644 --- a/piccolo/apps/asgi/commands/templates/app/app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/app.py.jinja @@ -4,4 +4,6 @@ {% include '_starlette_app.py.jinja' %} {% elif router == 'blacksheep' %} {% include '_blacksheep_app.py.jinja' %} +{% elif router == 'xpresso' %} + {% include '_xpresso_app.py.jinja' %} {% endif %} diff --git a/piccolo/apps/asgi/commands/templates/app/home/_blacksheep_endpoints.py.jinja b/piccolo/apps/asgi/commands/templates/app/home/_blacksheep_endpoints.py.jinja index c303b7061..daf650cb6 100644 --- a/piccolo/apps/asgi/commands/templates/app/home/_blacksheep_endpoints.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/home/_blacksheep_endpoints.py.jinja @@ -12,7 +12,7 @@ ENVIRONMENT = jinja2.Environment( ) -def home(): +async def home(request): template = ENVIRONMENT.get_template("home.html.jinja") content = template.render(title="Piccolo + ASGI",) return Response( diff --git a/piccolo/apps/asgi/commands/templates/app/home/_xpresso_endpoints.py.jinja b/piccolo/apps/asgi/commands/templates/app/home/_xpresso_endpoints.py.jinja new file mode 100644 index 000000000..2068880ac --- /dev/null +++ b/piccolo/apps/asgi/commands/templates/app/home/_xpresso_endpoints.py.jinja @@ -0,0 +1,20 @@ +import os + +import jinja2 +from xpresso.responses import HTMLResponse + +ENVIRONMENT = jinja2.Environment( + loader=jinja2.FileSystemLoader( + searchpath=os.path.join(os.path.dirname(__file__), "templates") + ) +) + + +async def home(): + template = ENVIRONMENT.get_template("home.html.jinja") + + content = template.render( + title="Piccolo + ASGI", + ) + + return HTMLResponse(content) diff --git a/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja b/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja index 47e908805..36e4eaf49 100644 --- a/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja @@ -2,4 +2,6 @@ {% include '_starlette_endpoints.py.jinja' %} {% elif router == 'blacksheep' %} {% include '_blacksheep_endpoints.py.jinja' %} +{% elif router == 'xpresso' %} + {% include '_xpresso_endpoints.py.jinja' %} {% endif %} diff --git a/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw b/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw index 726bcc612..614f75ba6 100644 --- a/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw +++ b/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw @@ -51,6 +51,11 @@

  • Admin
  • Swagger API
    • +

    Xpresso

    +
    {% endblock content %} diff --git a/tests/apps/asgi/commands/test_new.py b/tests/apps/asgi/commands/test_new.py index 7ec147bd0..fa4a99cab 100644 --- a/tests/apps/asgi/commands/test_new.py +++ b/tests/apps/asgi/commands/test_new.py @@ -56,38 +56,40 @@ def test_new(self): """ Test that the ASGI app actually runs. """ - router = "fastapi" - - with patch( - "piccolo.apps.asgi.commands.new.get_routing_framework", - return_value=router, - ), patch( - "piccolo.apps.asgi.commands.new.get_server", - return_value=SERVERS[0], - ): - root = os.path.join(tempfile.gettempdir(), "asgi_app") + for router in ROUTERS: + with patch( + "piccolo.apps.asgi.commands.new.get_routing_framework", + return_value=router, + ), patch( + "piccolo.apps.asgi.commands.new.get_server", + return_value=SERVERS[0], + ): + root = os.path.join(tempfile.gettempdir(), "asgi_app") - if os.path.exists(root): - shutil.rmtree(root) + if os.path.exists(root): + shutil.rmtree(root) - os.mkdir(root) - new(root=root) + os.mkdir(root) + new(root=root) - # Copy a dummy ASGI server, so we can test that the server works. - shutil.copyfile( - os.path.join( - os.path.dirname(__file__), - "files", - "dummy_server.py", - ), - os.path.join(root, "dummy_server.py"), - ) + # Copy a dummy ASGI server, so we can test that the server + # works. + shutil.copyfile( + os.path.join( + os.path.dirname(__file__), + "files", + "dummy_server.py", + ), + os.path.join(root, "dummy_server.py"), + ) - response = subprocess.run( - f"cd {root} && " - "python -m venv venv && " - "./venv/bin/pip install -r requirements.txt && " - "./venv/bin/python dummy_server.py", - shell=True, - ) - self.assertEqual(response.returncode, 0, msg=f"{router} failed") + response = subprocess.run( + f"cd {root} && " + "python -m venv venv && " + "./venv/bin/pip install -r requirements.txt && " + "./venv/bin/python dummy_server.py", + shell=True, + ) + self.assertEqual( + response.returncode, 0, msg=f"{router} failed" + ) From c5c382681d5ca05acd3e43a4ed53827ed7ad4cfb Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 3 Feb 2022 16:55:58 +0000 Subject: [PATCH 249/727] bumped version --- CHANGES.rst | 15 ++++++++++++++- piccolo/__init__.py | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 04082b299..045f39969 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,19 @@ Changes ======= +0.69.0 +------ + +Added `Xpresso `_ as a supported ASGI framework when +using ``piccolo asgi new`` to generate a web app. + +Thanks to @sinisaos for adding this template, and @adriangb for reviewing. + +We also took this opportunity to update our FastAPI and BlackSheep ASGI +templates. + +------------------------------------------------------------------------------- + 0.68.0 ------ @@ -1240,7 +1253,7 @@ example: class RecordingStudio(Table): pk = UUID(primary_key=True) -The BlackSheep template generated by `piccolo asgi new` now supports mounting +The BlackSheep template generated by ``piccolo asgi new`` now supports mounting of the Piccolo Admin (courtesy @sinisaos). ------------------------------------------------------------------------------- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index ca10f407b..9c1785465 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.68.0" +__VERSION__ = "0.69.0" From 94313d8c402215cf135f74edebc92b05c8c0c111 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 4 Feb 2022 10:47:25 +0000 Subject: [PATCH 250/727] fix bug with migrations which rename columns (#423) --- piccolo/apps/migrations/auto/schema_snapshot.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/piccolo/apps/migrations/auto/schema_snapshot.py b/piccolo/apps/migrations/auto/schema_snapshot.py index 1b51ccd91..9543576b5 100644 --- a/piccolo/apps/migrations/auto/schema_snapshot.py +++ b/piccolo/apps/migrations/auto/schema_snapshot.py @@ -102,5 +102,8 @@ def get_snapshot(self) -> t.List[DiffableTable]: for column in table.columns: if column._meta.name == rename_column.old_column_name: column._meta.name = rename_column.new_column_name + column._meta.db_column_name = ( + rename_column.new_db_column_name + ) return tables From 87b231c6d19734deb92d18d0fe796baece3f13c9 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 4 Feb 2022 10:51:57 +0000 Subject: [PATCH 251/727] bumped version --- CHANGES.rst | 9 +++++++++ piccolo/__init__.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 045f39969..0af6521b4 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,15 @@ Changes ======= +0.69.1 +------ + +Fixed a bug with auto migrations which rename columns - see +`PR 423 `_. Thanks to +@theelderbeever for reporting this, and @sinisaos for help investigating. + +------------------------------------------------------------------------------- + 0.69.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 9c1785465..087ccac95 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.69.0" +__VERSION__ = "0.69.1" From 97b34536da8971f38e61875829c2a6c6810c0fca Mon Sep 17 00:00:00 2001 From: sinisaos Date: Mon, 7 Feb 2022 17:57:06 +0100 Subject: [PATCH 252/727] add docs for a custom BaseUser (#422) * add docs for a custom BaseUser * mention copying the user app Co-authored-by: Daniel Townsend --- docs/src/piccolo/authentication/baseuser.rst | 23 ++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/src/piccolo/authentication/baseuser.rst b/docs/src/piccolo/authentication/baseuser.rst index 11b0fd3af..74a3a4372 100644 --- a/docs/src/piccolo/authentication/baseuser.rst +++ b/docs/src/piccolo/authentication/baseuser.rst @@ -141,6 +141,29 @@ sufficiently long for most use cases. ------------------------------------------------------------------------------- +Extending ``BaseUser`` +---------------------- + +If you want to extend ``BaseUser`` with additional fields, we recommend creating +a ``Profile`` table with a ``ForeignKey`` to ``BaseUser``, which can include +any custom fields. + +.. code-block:: python + + from piccolo.apps.user.tables import BaseUser + from piccolo.columns import ForeignKey, Text, Varchar + from piccolo.table import Table + + class Profile(Table): + custom_user = ForeignKey(BaseUser) + phone_number = Varchar() + bio = Text() + +Alternatively, you can copy the entire `user app `_ into your +project, and customise it to fit your needs. + +---------------------------------------------------------------------------------- + Source ------ From 2d70ba95e661e92bf1b1ea0335d1636da32cd544 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 7 Feb 2022 16:57:22 +0000 Subject: [PATCH 253/727] add missing param tags (#425) --- piccolo/utils/pydantic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piccolo/utils/pydantic.py b/piccolo/utils/pydantic.py index bc9a09259..8c3aa8c68 100644 --- a/piccolo/utils/pydantic.py +++ b/piccolo/utils/pydantic.py @@ -125,9 +125,9 @@ def create_pydantic_model( By default, the values of any Piccolo ``JSON`` or ``JSONB`` columns are returned as strings. By setting this parameter to True, they will be returned as objects. - :recursion_depth: + :param recursion_depth: Not to be set by the user - used internally to track recursion. - :max_recursion_depth: + :param max_recursion_depth: If using nested models, this specifies the max amount of recursion. :param schema_extra_kwargs: This can be used to add additional fields to the schema. This is From e84aa1fb480f9a678925b997acc307abf1b29ada Mon Sep 17 00:00:00 2001 From: Arie Bovenberg Date: Thu, 10 Feb 2022 10:47:29 +0100 Subject: [PATCH 254/727] Fix overlapping, missing slots (#426) * fix overlapping, missing slots * add `__slots__` definition for `WhereRaw` * add slotscheck to linting Co-authored-by: Daniel Townsend --- piccolo/columns/combination.py | 5 +++++ piccolo/query/methods/alter.py | 4 ++-- requirements/dev-requirements.txt | 1 + scripts/lint.sh | 7 ++++++- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/piccolo/columns/combination.py b/piccolo/columns/combination.py index f98346afd..82668c3d8 100644 --- a/piccolo/columns/combination.py +++ b/piccolo/columns/combination.py @@ -12,6 +12,9 @@ class CombinableMixin(object): + + __slots__ = () + def __and__(self, value: Combinable) -> "And": return And(self, value) # type: ignore @@ -86,6 +89,8 @@ class Undefined: class WhereRaw(CombinableMixin): + __slots__ = ("querystring",) + def __init__(self, sql: str, *args: t.Any) -> None: """ Execute raw SQL queries in your where clause. Use with caution! diff --git a/piccolo/query/methods/alter.py b/piccolo/query/methods/alter.py index 0768c27c0..518eb3b81 100644 --- a/piccolo/query/methods/alter.py +++ b/piccolo/query/methods/alter.py @@ -72,7 +72,7 @@ def ddl(self) -> str: @dataclass class AddColumn(AlterColumnStatement): - __slots__ = ("column", "name") + __slots__ = ("name",) column: Column name: str @@ -120,7 +120,7 @@ def ddl(self) -> str: @dataclass class SetDefault(AlterColumnStatement): - __slots__ = ("column", "value") + __slots__ = ("value",) column: Column value: t.Any diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt index 7c812f452..c4e438143 100644 --- a/requirements/dev-requirements.txt +++ b/requirements/dev-requirements.txt @@ -3,6 +3,7 @@ ipdb==0.13.9 ipython>=7.31.1 flake8==4.0.1 isort==5.10.1 +slotscheck==0.12.0 twine==3.7.1 mypy==0.931 pip-upgrader==1.4.15 diff --git a/scripts/lint.sh b/scripts/lint.sh index 3908d8377..14582f903 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -1,7 +1,8 @@ #!/bin/bash set -e -SOURCES="piccolo tests" +MODULES="piccolo" +SOURCES="$MODULES tests" echo "Running isort..." isort --check $SOURCES @@ -19,4 +20,8 @@ echo "Running mypy..." mypy $SOURCES echo "-----" +echo "Running slotscheck..." +python -m slotscheck $MODULES +echo "-----" + echo "All passed!" From 0c48ed628148707a45f4e04dcfeda59a4eb9ff36 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 10 Feb 2022 09:51:10 +0000 Subject: [PATCH 255/727] improve docstring for `piccolo migrations new` command (#428) --- piccolo/apps/migrations/commands/new.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piccolo/apps/migrations/commands/new.py b/piccolo/apps/migrations/commands/new.py index 6a50608a5..49174e61a 100644 --- a/piccolo/apps/migrations/commands/new.py +++ b/piccolo/apps/migrations/commands/new.py @@ -199,8 +199,8 @@ async def new( :param auto: Auto create the migration contents. :param desc: - A description of what the migration does, for example 'adding name - column'. + A description of what the migration does, for example --desc='adding + name column'. :param auto_input: If provided, all prompts for user input will automatically have this entered. For example, --auto_input='y'. From 8500c81a5e3bdc226d91d3356fc688a4038e0299 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 13 Feb 2022 19:21:02 +0000 Subject: [PATCH 256/727] documentation improvements (#432) * add missing backticks * use RST links to refer to other methods * fix list indentation in docs * link directly to Piccolo Admin tag in docs * remove extra space * restructure Piccolo apps docs * nicer refs * simplify engine link * added api reference page * add line breaks to example code, so it's easier to read * improve ref link * add `LazyTableReference` to API reference * add missing docstrings to `Table` class * fix duplicate reference error * fix typos in `update` docstring * add more line breaks to code snippets in testing docs * fix typo * add links to docstring * fix hint about using sync * add missing bracket * simplify playground link, and wrap text * add missing backticks * add missing 'the' * fix list indentation * move warning text into an admonition --- docs/src/index.rst | 1 + docs/src/piccolo/api_reference/index.rst | 20 ++++ docs/src/piccolo/authentication/baseuser.rst | 5 +- docs/src/piccolo/contributing/index.rst | 30 ++--- docs/src/piccolo/ecosystem/index.rst | 10 +- docs/src/piccolo/engines/index.rst | 2 +- docs/src/piccolo/features/syntax.rst | 4 +- .../piccolo/getting_started/playground.rst | 6 +- .../projects_and_apps/piccolo_apps.rst | 105 +++++++++--------- .../projects_and_apps/piccolo_projects.rst | 4 +- docs/src/piccolo/query_clauses/where.rst | 5 +- docs/src/piccolo/query_types/index.rst | 2 +- docs/src/piccolo/query_types/select.rst | 6 +- docs/src/piccolo/schema/advanced.rst | 16 +-- docs/src/piccolo/schema/defining.rst | 6 +- docs/src/piccolo/schema/m2m.rst | 4 +- docs/src/piccolo/testing/index.rst | 20 +++- piccolo/apps/user/tables.py | 6 +- piccolo/columns/column_types.py | 3 +- piccolo/columns/reference.py | 9 +- piccolo/engine/postgres.py | 12 +- piccolo/table.py | 41 +++++-- piccolo/utils/pydantic.py | 4 +- 23 files changed, 194 insertions(+), 127 deletions(-) create mode 100644 docs/src/piccolo/api_reference/index.rst diff --git a/docs/src/index.rst b/docs/src/index.rst index 137b6bc8c..6af89899b 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -26,6 +26,7 @@ batteries included. piccolo/contributing/index piccolo/changes/index piccolo/help/index + piccolo/api_reference/index ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/api_reference/index.rst b/docs/src/piccolo/api_reference/index.rst new file mode 100644 index 000000000..9ec9cbb8c --- /dev/null +++ b/docs/src/piccolo/api_reference/index.rst @@ -0,0 +1,20 @@ +API reference +============= + +Table +----- + +.. currentmodule:: piccolo.table + +.. autoclass:: Table + :members: + +------------------------------------------------------------------------------- + +LazyTableReference +------------------ + +.. currentmodule:: piccolo.columns + +.. autoclass:: LazyTableReference + :members: diff --git a/docs/src/piccolo/authentication/baseuser.rst b/docs/src/piccolo/authentication/baseuser.rst index 74a3a4372..31fa90e68 100644 --- a/docs/src/piccolo/authentication/baseuser.rst +++ b/docs/src/piccolo/authentication/baseuser.rst @@ -40,8 +40,9 @@ script), you can pass all of the arguments in as follows: piccolo user create --username=bob --password=bob123 --email=foo@bar.com --is_admin=t --is_superuser=t --is_active=t -If you choose this approach then be careful, as the password will be in the -shell's history. +.. warning:: + If you choose this approach then be careful, as the password will be in the + shell's history. change_password ~~~~~~~~~~~~~~~ diff --git a/docs/src/piccolo/contributing/index.rst b/docs/src/piccolo/contributing/index.rst index f7efbd4a7..1f4565de0 100644 --- a/docs/src/piccolo/contributing/index.rst +++ b/docs/src/piccolo/contributing/index.rst @@ -11,16 +11,16 @@ instructions. Get the tests running --------------------- - * Create a new virtualenv - * Clone the `Git repo `_ - * ``cd piccolo`` - * Install default dependencies: ``pip install -r requirements/requirements.txt`` - * Install development dependencies: ``pip install -r requirements/dev-requirements.txt`` - * Install test dependencies: ``pip install -r requirements/test-requirements.txt`` - * Setup Postgres - * Run the automated code linting/formatting tools: ``./scripts/lint.sh`` - * Run the test suite with Postgres: ``./scripts/test-postgres.sh`` - * Run the test suite with Sqlite: ``./scripts/test-sqlite.sh`` +* Create a new virtualenv +* Clone the `Git repo `_ +* ``cd piccolo`` +* Install default dependencies: ``pip install -r requirements/requirements.txt`` +* Install development dependencies: ``pip install -r requirements/dev-requirements.txt`` +* Install test dependencies: ``pip install -r requirements/test-requirements.txt`` +* Setup Postgres +* Run the automated code linting/formatting tools: ``./scripts/lint.sh`` +* Run the test suite with Postgres: ``./scripts/test-postgres.sh`` +* Run the test suite with Sqlite: ``./scripts/test-sqlite.sh`` ------------------------------------------------------------------------------- @@ -29,11 +29,11 @@ Contributing to the docs The docs are written using Sphinx. To get them running locally: - * Install the requirements: ``pip install -r requirements/doc-requirements.txt`` - * ``cd docs`` - * Do an initial build of the docs: ``make html`` - * Serve the docs: ``python serve_docs.py`` - * The docs will auto rebuild as you make changes. +* Install the requirements: ``pip install -r requirements/doc-requirements.txt`` +* ``cd docs`` +* Do an initial build of the docs: ``make html`` +* Serve the docs: ``python serve_docs.py`` +* The docs will auto rebuild as you make changes. ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/ecosystem/index.rst b/docs/src/piccolo/ecosystem/index.rst index 6b414dc66..08efe4512 100644 --- a/docs/src/piccolo/ecosystem/index.rst +++ b/docs/src/piccolo/ecosystem/index.rst @@ -9,16 +9,18 @@ Piccolo API Provides some handy utilities for creating an API around your Piccolo tables. Examples include: - * Easily creating CRUD endpoints for ASGI apps, based on Piccolo tables. - * Automatically creating Pydantic models from your Piccolo tables. - * Great FastAPI integration. - * Authentication and rate limiting. +* Easily creating CRUD endpoints for ASGI apps, based on Piccolo tables. +* Automatically creating Pydantic models from your Piccolo tables. +* Great FastAPI integration. +* Authentication and rate limiting. `See the docs `_ for more information. ------------------------------------------------------------------------------- +.. _PiccoloAdmin: + Piccolo Admin ------------- diff --git a/docs/src/piccolo/engines/index.rst b/docs/src/piccolo/engines/index.rst index 2c6f191c4..c39777da7 100644 --- a/docs/src/piccolo/engines/index.rst +++ b/docs/src/piccolo/engines/index.rst @@ -4,7 +4,7 @@ Engines ======= Engines are what execute the SQL queries. Each supported backend has its own -engine (see  :ref:`EngineTypes`). +:ref:`engine `. It's important that each ``Table`` class knows which engine to use. There are two ways of doing this - setting it explicitly via the ``db`` argument, or diff --git a/docs/src/piccolo/features/syntax.rst b/docs/src/piccolo/features/syntax.rst index 2bcedf89a..82deeae95 100644 --- a/docs/src/piccolo/features/syntax.rst +++ b/docs/src/piccolo/features/syntax.rst @@ -9,8 +9,8 @@ closely as possible. For example: - * In other ORMs, you define models - in Piccolo you define tables. - * Rather than using a filter method, you use a `where` method like in SQL. +* In other ORMs, you define models - in Piccolo you define tables. +* Rather than using a filter method, you use a `where` method like in SQL. ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/getting_started/playground.rst b/docs/src/piccolo/getting_started/playground.rst index 971f6e354..f24838e42 100644 --- a/docs/src/piccolo/getting_started/playground.rst +++ b/docs/src/piccolo/getting_started/playground.rst @@ -10,7 +10,8 @@ to learn the basics. piccolo playground run -It will create an example schema for you (see :ref:`ExampleSchema`) , populates it with data, and launches an `iPython `_ shell. +It will create an :ref:`example schema ` for you, populates it +with data, and launches an `iPython `_ shell. You can follow along with the tutorials without first learning advanced concepts like migrations. @@ -19,7 +20,8 @@ It's a nice place to experiment with querying / inserting / deleting data using Piccolo, no matter how experienced you are. .. warning:: - Each time you launch the playground it flushes out the existing tables and rebuilds them, so don't use it for anything permanent! + Each time you launch the playground it flushes out the existing tables and + rebuilds them, so don't use it for anything permanent! ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/projects_and_apps/piccolo_apps.rst b/docs/src/piccolo/projects_and_apps/piccolo_apps.rst index 63e585ad3..fe13f9e1a 100644 --- a/docs/src/piccolo/projects_and_apps/piccolo_apps.rst +++ b/docs/src/piccolo/projects_and_apps/piccolo_apps.rst @@ -1,8 +1,7 @@ .. _PiccoloApps: -############ Piccolo Apps -############ +============ By leveraging Piccolo apps you can: @@ -12,9 +11,8 @@ By leveraging Piccolo apps you can: ------------------------------------------------------------------------------- -*************** Creating an app -*************** +--------------- Run the following command within your project: @@ -49,9 +47,8 @@ operations on your app, such as :ref:`Migrations`. ------------------------------------------------------------------------------- -********* AppConfig -********* +--------- Inside your app's ``piccolo_app.py`` file is an ``AppConfig`` instance. This is how you customise your app's settings. @@ -75,14 +72,17 @@ how you customise your app's settings. APP_CONFIG = AppConfig( app_name='blog', - migrations_folder_path=os.path.join(CURRENT_DIRECTORY, 'piccolo_migrations'), + migrations_folder_path=os.path.join( + CURRENT_DIRECTORY, + 'piccolo_migrations' + ), table_classes=[Author, Post, Category, CategoryToPost], migration_dependencies=[], commands=[] ) app_name -======== +~~~~~~~~ This is used to identify your app, when using the ``piccolo`` CLI, for example: @@ -91,62 +91,28 @@ This is used to identify your app, when using the ``piccolo`` CLI, for example: piccolo migrations forwards blog migrations_folder_path -====================== +~~~~~~~~~~~~~~~~~~~~~~ Specifies where your app's migrations are stored. By default, a folder called ``piccolo_migrations`` is used. table_classes -============= +~~~~~~~~~~~~~ Use this to register your app's ``Table`` subclasses. This is important for -auto migrations (see :ref:`Migrations`). - -You can register them manually, see the example above, or can use -``table_finder``. - -.. _TableFinder: - -table_finder ------------- - -Instead of manually registering ``Table`` subclasses, you can use -``table_finder`` to automatically import any ``Table`` subclasses from a given -list of modules. - -.. code-block:: python - - from piccolo.conf.apps import table_finder - - APP_CONFIG = AppConfig( - app_name='blog', - migrations_folder_path=os.path.join(CURRENT_DIRECTORY, 'piccolo_migrations'), - table_classes=table_finder(modules=['blog.tables']), - migration_dependencies=[], - commands=[] - ) - -The module path should be from the root of the project (the same directory as -your ``piccolo_conf.py`` file, rather than a relative path). - -You can filter the ``Table`` subclasses returned using tags (see :ref:`TableTags`). - -Source -~~~~~~ - -.. currentmodule:: piccolo.conf.apps - -.. autofunction:: table_finder +:ref:`auto migrations `. +You can register them manually (see the example above), or can use +:ref:`table_finder `. migration_dependencies -====================== +~~~~~~~~~~~~~~~~~~~~~~ Used to specify other Piccolo apps whose migrations need to be run before the current app's migrations. commands -======== +~~~~~~~~ You can register functions and coroutines, which are automatically added to the ``piccolo`` CLI. @@ -210,9 +176,46 @@ for inspiration. ------------------------------------------------------------------------------- -************ +.. _TableFinder: + +table_finder +------------ + +Instead of manually registering ``Table`` subclasses, you can use +``table_finder`` to automatically import any ``Table`` subclasses from a given +list of modules. + +.. code-block:: python + + from piccolo.conf.apps import table_finder + + APP_CONFIG = AppConfig( + app_name='blog', + migrations_folder_path=os.path.join( + CURRENT_DIRECTORY, + 'piccolo_migrations' + ), + table_classes=table_finder(modules=['blog.tables']), + migration_dependencies=[], + commands=[] + ) + +The module path should be from the root of the project (the same directory as +your ``piccolo_conf.py`` file, rather than a relative path). + +You can filter the ``Table`` subclasses returned using :ref:`tags `. + +Source +~~~~~~ + +.. currentmodule:: piccolo.conf.apps + +.. autofunction:: table_finder + +------------------------------------------------------------------------------- + Sharing Apps -************ +------------ By breaking up your project into apps, the project becomes more maintainable. You can also share these apps between projects, and they can even be installed diff --git a/docs/src/piccolo/projects_and_apps/piccolo_projects.rst b/docs/src/piccolo/projects_and_apps/piccolo_projects.rst index 5fb784386..31e27bd4a 100644 --- a/docs/src/piccolo/projects_and_apps/piccolo_projects.rst +++ b/docs/src/piccolo/projects_and_apps/piccolo_projects.rst @@ -18,8 +18,8 @@ A project requires a ``piccolo_conf.py`` file. To create this, use the following The file serves two important purposes: - * Contains your database settings. - * Is used for registering :ref:`PiccoloApps`. +* Contains your database settings. +* Is used for registering :ref:`PiccoloApps`. Location ~~~~~~~~ diff --git a/docs/src/piccolo/query_clauses/where.rst b/docs/src/piccolo/query_clauses/where.rst index dd6c50225..29e683430 100644 --- a/docs/src/piccolo/query_clauses/where.rst +++ b/docs/src/piccolo/query_clauses/where.rst @@ -67,7 +67,7 @@ The percentage operator is required to designate where the match should occur. ) await Band.select().where( - Band.name.like('%is%') # Matches anywhere in string + Band.name.like('%is%') # Matches anywhere in the string ) await Band.select().where( @@ -126,7 +126,8 @@ with None: Band.manager == None ) -To avoid the linter errors, you can use `is_null` and `is_not_null` instead. +To avoid the linter errors, you can use ``is_null`` and ``is_not_null`` +instead. .. code-block:: python diff --git a/docs/src/piccolo/query_types/index.rst b/docs/src/piccolo/query_types/index.rst index a07bf63e6..5c054f64e 100644 --- a/docs/src/piccolo/query_types/index.rst +++ b/docs/src/piccolo/query_types/index.rst @@ -4,7 +4,7 @@ Query Types There are many different queries you can perform using Piccolo. The main ways to query data are with :ref:`Select`, which returns data as -dictionaries, and :ref:`Objects` , which returns data as class instances, like a +dictionaries, and :ref:`Objects`, which returns data as class instances, like a typical ORM. .. toctree:: diff --git a/docs/src/piccolo/query_types/select.rst b/docs/src/piccolo/query_types/select.rst index 5f2509287..9d61a1eed 100644 --- a/docs/src/piccolo/query_types/select.rst +++ b/docs/src/piccolo/query_types/select.rst @@ -29,7 +29,9 @@ Or use an alias to make it shorter: [{'id': 1, 'name': 'Pythonistas', 'manager': 1, 'popularity': 1000}, {'id': 2, 'name': 'Rustaceans', 'manager': 2, 'popularity': 500}] -.. hint:: All of these examples also work with async by using .run() inside coroutines - see :ref:`SyncAndAsync`. +.. hint:: + All of these examples also work synchronously using ``run_sync`` - + see :ref:`SyncAndAsync`. ------------------------------------------------------------------------------- @@ -98,7 +100,7 @@ You can exclude some columns if you like: >>> await Band.select( >>> Band.name, - >>> Band.manager.all_columns(exclude=[Band.manager.id) + >>> Band.manager.all_columns(exclude=[Band.manager.id]) >>> ) [ {'name': 'Pythonistas', 'manager.name': 'Guido'}, diff --git a/docs/src/piccolo/schema/advanced.rst b/docs/src/piccolo/schema/advanced.rst index 067df78c2..246f0780e 100644 --- a/docs/src/piccolo/schema/advanced.rst +++ b/docs/src/piccolo/schema/advanced.rst @@ -7,7 +7,7 @@ Readable -------- Sometimes Piccolo needs a succinct representation of a row - for example, when -displaying a link in the Piccolo Admin GUI (see :ref:`Ecosystem`). Rather than +displaying a link in the :ref:`Piccolo Admin `. Rather than just displaying the row ID, we can specify something more user friendly using ``Readable``. @@ -56,7 +56,7 @@ Table Tags ---------- ``Table`` subclasses can be given tags. The tags can be used for filtering, -for example with ``table_finder`` (see :ref:`TableFinder`). +for example with :ref:`table_finder `. .. code-block:: python @@ -136,9 +136,9 @@ Advantages By using choices, you get the following benefits: - * Signalling to other programmers what values are acceptable for the column. - * Improved storage efficiency (we can store ``'l'`` instead of ``'large'``). - * Piccolo Admin support +* Signalling to other programmers what values are acceptable for the column. +* Improved storage efficiency (we can store ``'l'`` instead of ``'large'``). +* Piccolo Admin support ------------------------------------------------------------------------------- @@ -155,9 +155,9 @@ can dynamically create them at run time, by inspecting the database. These Some example use cases: - * You have a very dynamic database, where new tables are being created - constantly, so updating a ``tables.py`` is impractical. - * You use Piccolo on the command line to explore databases. +* You have a very dynamic database, where new tables are being created + constantly, so updating a ``tables.py`` is impractical. +* You use Piccolo on the command line to explore databases. Full reflection ~~~~~~~~~~~~~~~ diff --git a/docs/src/piccolo/schema/defining.rst b/docs/src/piccolo/schema/defining.rst index 63683321c..a110e36e8 100644 --- a/docs/src/piccolo/schema/defining.rst +++ b/docs/src/piccolo/schema/defining.rst @@ -3,8 +3,8 @@ Defining a Schema ================= -The schema is usually defined within the ``tables.py`` file of your Piccolo -app (see :ref:`PiccoloApps`). +The schema is usually defined within the ``tables.py`` file of your +:ref:`Piccolo app `. This reflects the tables in your database. Each table consists of several columns. Here's a very simple schema: @@ -19,7 +19,7 @@ columns. Here's a very simple schema: class Band(Table): name = Varchar(length=100) -For a full list of columns, see :ref:`ColumnTypes`. +For a full list of columns, see :ref:`column types `. .. hint:: If you're using an existing database, see Piccolo's :ref:`auto schema generation command`, which will save you some diff --git a/docs/src/piccolo/schema/m2m.rst b/docs/src/piccolo/schema/m2m.rst index c93bb097b..c8536b2ba 100644 --- a/docs/src/piccolo/schema/m2m.rst +++ b/docs/src/piccolo/schema/m2m.rst @@ -142,17 +142,19 @@ add_m2m .. currentmodule:: piccolo.table .. automethod:: Table.add_m2m - + :noindex: get_m2m ------- .. automethod:: Table.get_m2m + :noindex: remove_m2m ---------- .. automethod:: Table.remove_m2m + :noindex: .. hint:: All of these methods can be run synchronously as well - for example, ``band.get_m2m(Band.genres).run_sync()``. diff --git a/docs/src/piccolo/testing/index.rst b/docs/src/piccolo/testing/index.rst index c7d2240dd..6904f5d47 100644 --- a/docs/src/piccolo/testing/index.rst +++ b/docs/src/piccolo/testing/index.rst @@ -34,7 +34,8 @@ You can build a random ``Band`` which will also build and save a random ``Manage from piccolo.testing.model_builder import ModelBuilder - band = await ModelBuilder.build(Band) # Band instance with random values persisted + # Band instance with random values persisted: + band = await ModelBuilder.build(Band) .. note:: ``ModelBuilder.build(Band)`` persists the record into the database by default. @@ -51,11 +52,17 @@ To specify any attribute, pass the ``defaults`` dictionary to the ``build`` meth manager = ModelBuilder.build(Manager) - # Using table columns - band = await ModelBuilder.build(Band, defaults={Band.name: "Guido", Band.manager: manager}) + # Using table columns: + band = await ModelBuilder.build( + Band, + defaults={Band.name: "Guido", Band.manager: manager} + ) - # Or using strings as keys - band = await ModelBuilder.build(Band, defaults={"name": "Guido", "manager": manager}) + # Or using strings as keys: + band = await ModelBuilder.build( + Band, + defaults={"name": "Guido", "manager": manager} + ) To build objects without persisting them into the database: @@ -67,7 +74,8 @@ To build object with minimal attributes, leaving nullable fields empty: .. code-block:: python - band = await ModelBuilder.build(Band, minimal=True) # Leaves manager empty + # Leaves manager empty: + band = await ModelBuilder.build(Band, minimal=True) ------------------------------------------------------------------------------- diff --git a/piccolo/apps/user/tables.py b/piccolo/apps/user/tables.py index 7fc084c91..10fb4b1e4 100644 --- a/piccolo/apps/user/tables.py +++ b/piccolo/apps/user/tables.py @@ -74,7 +74,7 @@ def get_readable(cls) -> Readable: @classmethod def update_password_sync(cls, user: t.Union[str, int], password: str): """ - A sync equivalent of ``update_password``. + A sync equivalent of :meth:`update_password`. """ return run_sync(cls.update_password(user, password)) @@ -145,7 +145,7 @@ def split_stored_password(cls, password: str) -> t.List[str]: @classmethod def login_sync(cls, username: str, password: str) -> t.Optional[int]: """ - A sync equivalent of ``login``. + A sync equivalent of :meth:`login`. """ return run_sync(cls.login(username, password)) @@ -201,7 +201,7 @@ def create_user_sync( cls, username: str, password: str, **extra_params ) -> BaseUser: """ - A sync equivalent of ``create_user``. + A sync equivalent of :meth:`create_user`. """ return run_sync( cls.create_user( diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 60d841524..c0be30e04 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -1057,7 +1057,8 @@ def __set__(self, obj, value: t.Union[timedelta, None]): class Boolean(Column): """ - Used for storing True / False values. Uses the ``bool`` type for values. + Used for storing ``True`` / ``False`` values. Uses the ``bool`` type for + values. **Example** diff --git a/piccolo/columns/reference.py b/piccolo/columns/reference.py index c4991aae4..841545eeb 100644 --- a/piccolo/columns/reference.py +++ b/piccolo/columns/reference.py @@ -16,17 +16,18 @@ @dataclass class LazyTableReference: """ - Holds a reference to a ``Table`` subclass. Used to avoid circular - dependencies in the ``references`` argument of ``ForeignKey`` columns. + Holds a reference to a :class:`Table ` subclass. Used + to avoid circular dependencies in the ``references`` argument of + :class:`ForeignKey ` columns. :param table_class_name: - The name of the ``Table`` subclass. For example, 'Manager'. + The name of the ``Table`` subclass. For example, ``'Manager'``. :param app_name: If specified, the ``Table`` subclass is imported from a Piccolo app with the given name. :param module_path: If specified, the ``Table`` subclass is imported from this path. - For example, 'my_app.tables'. + For example, ``'my_app.tables'``. """ diff --git a/piccolo/engine/postgres.py b/piccolo/engine/postgres.py index 63e3e988d..c9bcb95e6 100644 --- a/piccolo/engine/postgres.py +++ b/piccolo/engine/postgres.py @@ -206,17 +206,17 @@ class PostgresEngine(Engine): The config dictionary is passed to the underlying database adapter, asyncpg. Common arguments you're likely to need are: - * host - * port - * user - * password - * database + * host + * port + * user + * password + * database For example, ``{'host': 'localhost', 'port': 5432}``. To see all available options: - * https://magicstack.github.io/asyncpg/current/api/index.html#connection + * https://magicstack.github.io/asyncpg/current/api/index.html#connection :param extensions: When the engine starts, it will try and create these extensions diff --git a/piccolo/table.py b/piccolo/table.py index a26f47e1f..5bbfc0881 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -140,6 +140,10 @@ def __str__(cls): class Table(metaclass=TableMetaclass): + """ + The class represents a database table. An instance represents a row. + """ + # These are just placeholder values, so type inference isn't confused - the # actual values are set in __init_subclass__. _meta = TableMeta() @@ -271,6 +275,13 @@ def __init__( ): """ Assigns any default column values to the class. + + :param ignore_missing: + If ``False`` a ``ValueError`` will be raised if any column values + haven't been provided. + :param exists_in_db: + Used internally to track whether this row exists in the database. + """ self._exists_in_db = exists_in_db @@ -742,7 +753,9 @@ def ref(cls, column_name: str) -> Column: ever need to do this, but other libraries built on top of Piccolo may need this functionality. - Example: Band.ref('manager.name') + .. code-block:: python + + Band.ref('manager.name') """ local_column_name, reference_column_name = column_name.split(".") @@ -766,9 +779,14 @@ def ref(cls, column_name: str) -> Column: @classmethod def insert(cls, *rows: "Table") -> Insert: """ - await Band.insert( - Band(name="Pythonistas", popularity=500, manager=1) - ).run() + Insert rows into the database. + + .. code-block:: python + + await Band.insert( + Band(name="Pythonistas", popularity=500, manager=1) + ).run() + """ query = Insert(table=cls) if rows: @@ -780,11 +798,16 @@ def raw(cls, sql: str, *args: t.Any) -> Raw: """ Execute raw SQL queries on the underlying engine - use with caution! - await Band.raw('select * from band').run() + .. code-block:: python + + await Band.raw('select * from band').run() Or passing in parameters: - await Band.raw("select * from band where name = {}", 'Pythonistas') + .. code-block:: python + + await Band.raw("select * from band where name = {}", 'Pythonistas') + """ return Raw(table=cls, querystring=QueryString(sql, *args)) @@ -970,19 +993,19 @@ def update( await Band.update( {Band.name: "Spamalot"} ).where( - Band.name=="Pythonistas" + Band.name == "Pythonistas" ).run() await Band.update( {"name": "Spamalot"} ).where( - Band.name=="Pythonistas" + Band.name == "Pythonistas" ).run() await Band.update( name="Spamalot" ).where( - Band.name=="Pythonistas" + Band.name == "Pythonistas" ).run() :param force: diff --git a/piccolo/utils/pydantic.py b/piccolo/utils/pydantic.py index 8c3aa8c68..5f270faf9 100644 --- a/piccolo/utils/pydantic.py +++ b/piccolo/utils/pydantic.py @@ -123,8 +123,8 @@ def create_pydantic_model( same Piccolo table. :param deserialize_json: By default, the values of any Piccolo ``JSON`` or ``JSONB`` columns are - returned as strings. By setting this parameter to True, they will be - returned as objects. + returned as strings. By setting this parameter to ``True``, they will + be returned as objects. :param recursion_depth: Not to be set by the user - used internally to track recursion. :param max_recursion_depth: From 5f59841a746efc56b18abda9f1bc95663c55b260 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 13 Feb 2022 19:59:58 +0000 Subject: [PATCH 257/727] Create index fix (#433) * update table docstrings - remove `run` * quote column names when creating an index --- piccolo/query/methods/create_index.py | 4 +- piccolo/table.py | 54 +++++++++++++-------------- tests/table/test_indexes.py | 39 ++++++++++++++++++- 3 files changed, 66 insertions(+), 31 deletions(-) diff --git a/piccolo/query/methods/create_index.py b/piccolo/query/methods/create_index.py index 4a1822ccb..594fd63b6 100644 --- a/piccolo/query/methods/create_index.py +++ b/piccolo/query/methods/create_index.py @@ -44,7 +44,7 @@ def postgres_ddl(self) -> t.Sequence[str]: index_name = self.table._get_index_name(column_names) tablename = self.table._meta.tablename method_name = self.method.value - column_names_str = ", ".join(column_names) + column_names_str = ", ".join([f'"{i}"' for i in self.column_names]) return [ ( f"{self.prefix} {index_name} ON {tablename} USING " @@ -62,7 +62,7 @@ def sqlite_ddl(self) -> t.Sequence[str]: if method_name != "btree": raise ValueError("SQLite only support btree indexes.") - column_names_str = ", ".join(column_names) + column_names_str = ", ".join([f'"{i}"' for i in self.column_names]) return [ ( f"{self.prefix} {index_name} ON {tablename} " diff --git a/piccolo/table.py b/piccolo/table.py index 5bbfc0881..dcc0efd46 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -404,8 +404,8 @@ def get_related(self, foreign_key: t.Union[ForeignKey, str]) -> Objects: .. code-block:: python - band = await Band.objects().first().run() - manager = await band.get_related(Band.manager).run() + band = await Band.objects().first() + manager = await band.get_related(Band.manager) >>> print(manager.name) 'Guido' @@ -534,7 +534,7 @@ def to_dict(self, *columns: Column) -> t.Dict[str, t.Any]: instance = await Manager.objects().get( Manager.name == 'Guido' - ).run() + ) >>> instance.to_dict() {'id': 1, 'name': 'Guido'} @@ -677,7 +677,7 @@ def all_related( concert = await Concert.objects( Concert.all_related() - ).run() + ) >>> concert.band_1 @@ -695,7 +695,7 @@ def all_related( Concert.venue, Concert.band_1, Concert.band_2 - ).run() + ) :param exclude: You can request all columns, except these. @@ -726,7 +726,7 @@ def all_columns( await Band.select( Band.all_columns(), Band.manager.all_columns() - ).run() + ) This is mostly useful when the table has a lot of columns, and typing them out by hand would be tedious. @@ -785,7 +785,7 @@ def insert(cls, *rows: "Table") -> Insert: await Band.insert( Band(name="Pythonistas", popularity=500, manager=1) - ).run() + ) """ query = Insert(table=cls) @@ -800,7 +800,7 @@ def raw(cls, sql: str, *args: t.Any) -> Raw: .. code-block:: python - await Band.raw('select * from band').run() + await Band.raw('select * from band') Or passing in parameters: @@ -839,9 +839,9 @@ def select( .. code-block:: python - await Band.select().columns(Band.name).run() - await Band.select(Band.name).run() - await Band.select('name').run() + await Band.select().columns(Band.name) + await Band.select(Band.name) + await Band.select('name') :param exclude_secrets: If ``True``, any password fields are omitted from the response. @@ -861,7 +861,7 @@ def delete(cls, force=False) -> Delete: .. code-block:: python - await Band.delete().where(Band.name == 'Pythonistas').run() + await Band.delete().where(Band.name == 'Pythonistas') :param force: Unless set to ``True``, deletions aren't allowed without a @@ -879,7 +879,7 @@ def create_table( .. code-block:: python - await Band.create_table().run() + await Band.create_table() """ return Create( @@ -895,7 +895,7 @@ def alter(cls) -> Alter: .. code-block:: python - await Band.alter().rename_column(Band.popularity, 'rating').run() + await Band.alter().rename_column(Band.popularity, 'rating') """ return Alter(table=cls) @@ -912,11 +912,11 @@ def objects( pythonistas = await Band.objects().where( Band.name == 'Pythonistas' - ).first().run() + ).first() pythonistas.name = 'Pythonistas Reborn' - await pythonistas.save().run() + await pythonistas.save() # Or to remove it from the database: await pythonistas.remove() @@ -928,12 +928,12 @@ def objects( .. code-block:: python # Without nested - band = await Band.objects().first().run() + band = await Band.objects().first() >>> band.manager 1 # With nested - band = await Band.objects(Band.manager).first().run() + band = await Band.objects(Band.manager).first() >>> band.manager @@ -947,7 +947,7 @@ def count(cls) -> Count: .. code-block:: python - await Band.count().where(Band.popularity > 1000).run() + await Band.count().where(Band.popularity > 1000) """ return Count(table=cls) @@ -959,7 +959,7 @@ def exists(cls) -> Exists: .. code-block:: python - await Band.exists().where(Band.name == 'Pythonistas').run() + await Band.exists().where(Band.name == 'Pythonistas') """ return Exists(table=cls) @@ -971,7 +971,7 @@ def table_exists(cls) -> TableExists: .. code-block:: python - await Band.table_exists().run() + await Band.table_exists() """ return TableExists(table=cls) @@ -994,19 +994,19 @@ def update( {Band.name: "Spamalot"} ).where( Band.name == "Pythonistas" - ).run() + ) await Band.update( {"name": "Spamalot"} ).where( Band.name == "Pythonistas" - ).run() + ) await Band.update( name="Spamalot" ).where( Band.name == "Pythonistas" - ).run() + ) :param force: Unless set to ``True``, updates aren't allowed without a @@ -1023,7 +1023,7 @@ def indexes(cls) -> Indexes: .. code-block:: python - await Band.indexes().run() + await Band.indexes() """ return Indexes(table=cls) @@ -1041,7 +1041,7 @@ def create_index( .. code-block:: python - await Band.create_index([Band.name]).run() + await Band.create_index([Band.name]) """ return CreateIndex( @@ -1061,7 +1061,7 @@ def drop_index( .. code-block:: python - await Band.drop_index([Band.name]).run() + await Band.drop_index([Band.name]) """ return DropIndex(table=cls, columns=columns, if_exists=if_exists) diff --git a/tests/table/test_indexes.py b/tests/table/test_indexes.py index 95275a4f5..383dc06f6 100644 --- a/tests/table/test_indexes.py +++ b/tests/table/test_indexes.py @@ -1,10 +1,17 @@ from unittest import TestCase -from tests.base import DBTestCase +from piccolo.columns.column_types import Integer +from piccolo.table import Table from tests.example_apps.music.tables import Manager -class TestIndexes(DBTestCase): +class TestIndexes(TestCase): + def setUp(self): + Manager.create_table().run_sync() + + def tearDown(self): + Manager.alter().drop_table().run_sync() + def test_create_index(self): """ Test single column and multi column indexes. @@ -27,6 +34,34 @@ def test_create_index(self): self.assertTrue(index_name not in index_names) +class Concert(Table): + order = Integer() + + +class TestProblematicColumnName(TestCase): + def setUp(self): + Concert.create_table().run_sync() + + def tearDown(self): + Concert.alter().drop_table().run_sync() + + def test_problematic_name(self): + """ + Make sure we can add an index to a column with a problematic name + (which clashes with a SQL keyword). + """ + columns = [Concert.order] + Concert.create_index(columns=columns).run_sync() + index_name = Concert._get_index_name([i._meta.name for i in columns]) + + index_names = Concert.indexes().run_sync() + self.assertTrue(index_name in index_names) + + Concert.drop_index(columns).run_sync() + index_names = Concert.indexes().run_sync() + self.assertTrue(index_name not in index_names) + + class TestIndexName(TestCase): def test_index_name(self): self.assertEqual( From 4904761cc1a346e9bb40993a05aeca3c4e413bfe Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 13 Feb 2022 20:08:49 +0000 Subject: [PATCH 258/727] bumped version --- CHANGES.rst | 13 +++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 0af6521b4..2b7352ad3 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,19 @@ Changes ======= +0.69.2 +------ + + * Lots of documentation improvements, including how to customise ``BaseUser`` + (courtesy @sinisaos). + * Fixed a bug with creating indexes when the column name clashes with a SQL + keyword (e.g. ``'order'``). See `Pr 433 `_. + Thanks to @wmshort for reporting this issue. + * Fixed an issue where some slots which incorrectly configured (courtesy + @ariebovenberg). See `PR 426 `_. + +------------------------------------------------------------------------------- + 0.69.1 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 087ccac95..d5fc96a55 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.69.1" +__VERSION__ = "0.69.2" From d3976294a75ad075566f705875d27d81b66b328d Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 13 Feb 2022 20:23:43 +0000 Subject: [PATCH 259/727] fix typo --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2b7352ad3..4fd4d8f2b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -9,7 +9,7 @@ Changes * Fixed a bug with creating indexes when the column name clashes with a SQL keyword (e.g. ``'order'``). See `Pr 433 `_. Thanks to @wmshort for reporting this issue. - * Fixed an issue where some slots which incorrectly configured (courtesy + * Fixed an issue where some slots were incorrectly configured (courtesy @ariebovenberg). See `PR 426 `_. ------------------------------------------------------------------------------- From 275172784c5cb6d4cc018e60978c82f28326ee42 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 14 Feb 2022 19:04:47 +0000 Subject: [PATCH 260/727] More doc improvements (#434) * add `OnDelete` and `OnUpdate` to the API docs * add `IndexMethod` to the API docs * simplify link * add `DateOffset` to the API docs * change autodoc settings * remove `githubpages` Sphinx extension * add intersphinx extension * make code examples look closer to doctest format * fix typo in `Array` docs --- docs/src/conf.py | 10 ++++-- docs/src/piccolo/api_reference/index.rst | 40 +++++++++++++++++++++ docs/src/piccolo/migrations/create.rst | 2 +- docs/src/piccolo/query_clauses/group_by.rst | 10 +++--- docs/src/piccolo/query_types/select.rst | 18 +++++----- docs/src/piccolo/query_types/update.rst | 16 ++++----- docs/src/piccolo/schema/advanced.rst | 6 ++-- docs/src/piccolo/schema/column_types.rst | 38 ++++++++++---------- piccolo/columns/base.py | 10 ++++++ piccolo/columns/column_types.py | 24 ++++++------- piccolo/columns/defaults/date.py | 19 ++++++++++ piccolo/columns/indexes.py | 5 +++ piccolo/table.py | 12 +++---- 13 files changed, 145 insertions(+), 65 deletions(-) diff --git a/docs/src/conf.py b/docs/src/conf.py index f0e40a355..1b2352aa6 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -42,10 +42,8 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - "sphinx.ext.autodoc", "sphinx.ext.todo", "sphinx.ext.coverage", - "sphinx.ext.githubpages", ] # Add any paths that contain templates here, relative to this directory. @@ -75,9 +73,17 @@ # The name of the Pygments (syntax highlighting) style to use. pygments_style = None +# -- Intersphinx ------------------------------------------------------------- + +intersphinx_mapping = {"python": ("https://docs.python.org/3", None)} +extensions += ["sphinx.ext.intersphinx"] + # -- Autodoc ----------------------------------------------------------------- +extensions += ["sphinx.ext.autodoc"] autodoc_typehints = "signature" +autodoc_typehints_format = "short" +autoclass_content = "both" # -- Options for HTML output ------------------------------------------------- diff --git a/docs/src/piccolo/api_reference/index.rst b/docs/src/piccolo/api_reference/index.rst index 9ec9cbb8c..22bce6688 100644 --- a/docs/src/piccolo/api_reference/index.rst +++ b/docs/src/piccolo/api_reference/index.rst @@ -18,3 +18,43 @@ LazyTableReference .. autoclass:: LazyTableReference :members: + +------------------------------------------------------------------------------- + +Enums +----- + +Foreign Keys +~~~~~~~~~~~~ + +.. currentmodule:: piccolo.columns + +.. autoclass:: OnDelete + :members: + :undoc-members: + +.. autoclass:: OnUpdate + :members: + :undoc-members: + +.. currentmodule:: piccolo.columns.indexes + +Indexes +~~~~~~~ + +.. autoclass:: IndexMethod + :members: + :undoc-members: + +------------------------------------------------------------------------------- + +Column defaults +--------------- + +.. currentmodule:: piccolo.columns.defaults + +Date +~~~~ + +.. autoclass:: DateOffset + :members: diff --git a/docs/src/piccolo/migrations/create.rst b/docs/src/piccolo/migrations/create.rst index 74119873f..ceeb326d3 100644 --- a/docs/src/piccolo/migrations/create.rst +++ b/docs/src/piccolo/migrations/create.rst @@ -2,7 +2,7 @@ Creating migrations =================== Migrations are Python files which are used to modify the database schema in a -controlled way. Each migration belongs to a Piccolo app (see :ref:`PiccoloApps`). +controlled way. Each migration belongs to a :ref:`Piccolo app `. You can either manually populate migrations, or allow Piccolo to do it for you automatically. To create an empty migration: diff --git a/docs/src/piccolo/query_clauses/group_by.rst b/docs/src/piccolo/query_clauses/group_by.rst index 5853126cd..984a124f0 100644 --- a/docs/src/piccolo/query_clauses/group_by.rst +++ b/docs/src/piccolo/query_clauses/group_by.rst @@ -22,11 +22,11 @@ In the following query, we get a count of the number of bands per manager: >>> from piccolo.query.methods.select import Count >>> await Band.select( - >>> Band.manager.name, - >>> Count(Band.manager) - >>> ).group_by( - >>> Band.manager - >>> ) + ... Band.manager.name, + ... Count(Band.manager) + ... ).group_by( + ... Band.manager + ... ) [ {"manager.name": "Graydon", "count": 1}, diff --git a/docs/src/piccolo/query_types/select.rst b/docs/src/piccolo/query_types/select.rst index 9d61a1eed..d1f30d973 100644 --- a/docs/src/piccolo/query_types/select.rst +++ b/docs/src/piccolo/query_types/select.rst @@ -99,9 +99,9 @@ You can exclude some columns if you like: .. code-block:: python >>> await Band.select( - >>> Band.name, - >>> Band.manager.all_columns(exclude=[Band.manager.id]) - >>> ) + ... Band.name, + ... Band.manager.all_columns(exclude=[Band.manager.id]) + ... ) [ {'name': 'Pythonistas', 'manager.name': 'Guido'}, {'name': 'Rustaceans', 'manager.name': 'Graydon'} @@ -113,9 +113,9 @@ Strings are supported too if you prefer: .. code-block:: python >>> await Band.select( - >>> Band.name, - >>> Band.manager.all_columns(exclude=['id']) - >>> ) + ... Band.name, + ... Band.manager.all_columns(exclude=['id']) + ... ) [ {'name': 'Pythonistas', 'manager.name': 'Guido'}, {'name': 'Rustaceans', 'manager.name': 'Graydon'} @@ -127,9 +127,9 @@ you have lots of columns. It works identically to related tables: .. code-block:: python >>> await Band.select( - >>> Band.all_columns(exclude=[Band.id]), - >>> Band.manager.all_columns(exclude=[Band.manager.id]) - >>> ) + ... Band.all_columns(exclude=[Band.id]), + ... Band.manager.all_columns(exclude=[Band.manager.id]) + ... ) [ {'name': 'Pythonistas', 'popularity': 1000, 'manager.name': 'Guido'}, {'name': 'Rustaceans', 'popularity': 500, 'manager.name': 'Graydon'} diff --git a/docs/src/piccolo/query_types/update.rst b/docs/src/piccolo/query_types/update.rst index 64f6fa0ff..820a8e1ce 100644 --- a/docs/src/piccolo/query_types/update.rst +++ b/docs/src/piccolo/query_types/update.rst @@ -8,10 +8,10 @@ This is used to update any rows in the table which match the criteria. .. code-block:: python >>> await Band.update({ - >>> Band.name: 'Pythonistas 2' - >>> }).where( - >>> Band.name == 'Pythonistas' - >>> ) + ... Band.name: 'Pythonistas 2' + ... }).where( + ... Band.name == 'Pythonistas' + ... ) [] ------------------------------------------------------------------------------- @@ -107,10 +107,10 @@ you prefer: .. code-block:: python >>> await Band.update( - >>> name='Pythonistas 2' - >>> ).where( - >>> Band.name == 'Pythonistas' - >>> ) + ... name='Pythonistas 2' + ... ).where( + ... Band.name == 'Pythonistas' + ... ) ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/schema/advanced.rst b/docs/src/piccolo/schema/advanced.rst index 246f0780e..0532aebfd 100644 --- a/docs/src/piccolo/schema/advanced.rst +++ b/docs/src/piccolo/schema/advanced.rst @@ -124,9 +124,9 @@ where a query requires a value. .. code-block:: python >>> await Shirt.insert( - >>> Shirt(size=Shirt.Size.small), - >>> Shirt(size=Shirt.Size.medium) - >>> ) + ... Shirt(size=Shirt.Size.small), + ... Shirt(size=Shirt.Size.medium) + ... ) >>> await Shirt.select().where(Shirt.size == Shirt.Size.small) [{'id': 1, 'size': 's'}] diff --git a/docs/src/piccolo/schema/column_types.rst b/docs/src/piccolo/schema/column_types.rst index 89db19443..e327a4acf 100644 --- a/docs/src/piccolo/schema/column_types.rst +++ b/docs/src/piccolo/schema/column_types.rst @@ -231,10 +231,10 @@ However, we can ask Piccolo to deserialise the JSON automatically (see :ref:`loa .. code-block:: python >>> await RecordingStudio.select( - >>> RecordingStudio.facilities - >>> ).output( - >>> load_json=True - >>> ) + ... RecordingStudio.facilities + ... ).output( + ... load_json=True + ... ) [facilities: {"restaurant": True, "mixing_desk": True}}] With ``objects`` queries, we can modify the returned JSON, and then save it: @@ -257,9 +257,9 @@ a subset of the JSON data: .. code-block:: python >>> await RecordingStudio.select( - >>> RecordingStudio.name, - >>> RecordingStudio.facilities.arrow('mixing_desk').as_alias('mixing_desk') - >>> ).output(load_json=True) + ... RecordingStudio.name, + ... RecordingStudio.facilities.arrow('mixing_desk').as_alias('mixing_desk') + ... ).output(load_json=True) [{'name': 'Abbey Road', 'mixing_desk': True}] It can also be used for filtering in a where clause: @@ -267,8 +267,8 @@ It can also be used for filtering in a where clause: .. code-block:: python >>> await RecordingStudio.select(RecordingStudio.name).where( - >>> RecordingStudio.facilities.arrow('mixing_desk') == True - >>> ) + ... RecordingStudio.facilities.arrow('mixing_desk') == True + ... ) [{'name': 'Abbey Road'}] Handling null @@ -282,10 +282,10 @@ treated as null in the database. await RecordingStudio(name="ABC Studios", facilities=None).save() >>> await RecordingStudio.select( - >>> RecordingStudio.facilities - >>> ).where( - >>> RecordingStudio.name == "ABC Studios" - >>> ) + ... RecordingStudio.facilities + ... ).where( + ... RecordingStudio.name == "ABC Studios" + ... ) [{'facilities': None}] @@ -297,10 +297,10 @@ instead. await RecordingStudio(name="ABC Studios", facilities='null').save() >>> await RecordingStudio.select( - >>> RecordingStudio.facilities - >>> ).where( - >>> RecordingStudio.name == "ABC Studios" - >>> ) + ... RecordingStudio.facilities + ... ).where( + ... RecordingStudio.name == "ABC Studios" + ... ) [{'facilities': 'null'}] ------------------------------------------------------------------------------- @@ -309,8 +309,8 @@ instead. Array ***** -Arrays of data can be stored, which can be useful when you want store lots of -values without using foreign keys. +Arrays of data can be stored, which can be useful when you want to store lots +of values without using foreign keys. .. autoclass:: Array diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index d6ee2a372..c7d31e5c5 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -40,6 +40,11 @@ class OnDelete(str, Enum): + """ + Used by :class:`ForeignKey ` to + specify the behaviour when a related row is deleted. + """ + cascade = "CASCADE" restrict = "RESTRICT" no_action = "NO ACTION" @@ -54,6 +59,11 @@ def __repr__(self): class OnUpdate(str, Enum): + """ + Used by :class:`ForeignKey ` to + specify the behaviour when a related row is updated. + """ + cascade = "CASCADE" restrict = "RESTRICT" no_action = "NO ACTION" diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index c0be30e04..fe3f8a6ca 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -755,8 +755,8 @@ class Concert(Table): # Create >>> await Concert( - >>> starts=datetime.datetime(year=2050, month=1, day=1) - >>> ).save() + ... starts=datetime.datetime(year=2050, month=1, day=1) + ... ).save() # Query >>> await Concert.select(Concert.starts) @@ -821,10 +821,10 @@ class Concert(Table): # Create >>> await Concert( - >>> starts=datetime.datetime( - >>> year=2050, month=1, day=1, tzinfo=datetime.timezone.tz - >>> ) - >>> ).save() + ... starts=datetime.datetime( + ... year=2050, month=1, day=1, tzinfo=datetime.timezone.tz + ... ) + ... ).save() # Query >>> await Concert.select(Concert.starts) @@ -888,8 +888,8 @@ class Concert(Table): # Create >>> await Concert( - >>> starts=datetime.date(year=2020, month=1, day=1) - >>> ).save() + ... starts=datetime.date(year=2020, month=1, day=1) + ... ).save() # Query >>> await Concert.select(Concert.starts) @@ -945,8 +945,8 @@ class Concert(Table): # Create >>> await Concert( - >>> starts=datetime.time(hour=20, minute=0, second=0) - >>> ).save() + ... starts=datetime.time(hour=20, minute=0, second=0) + ... ).save() # Query >>> await Concert.select(Concert.starts) @@ -999,8 +999,8 @@ class Concert(Table): # Create >>> await Concert( - >>> duration=timedelta(hours=2) - >>> ).save() + ... duration=timedelta(hours=2) + ... ).save() # Query >>> await Concert.select(Concert.duration) diff --git a/piccolo/columns/defaults/date.py b/piccolo/columns/defaults/date.py index b6fecf86f..27fa3327b 100644 --- a/piccolo/columns/defaults/date.py +++ b/piccolo/columns/defaults/date.py @@ -8,7 +8,26 @@ class DateOffset(Default): + """ + This makes the default value for a + :class:`Date ` column the current date, + but offset by a number of days. + + For example, if you wanted the default to be tomorrow, you can specify + ``DateOffset(days=1)``: + + .. code-block:: python + + class DiscountCode(Table): + expires = Date(default=DateOffset(days=1)) + + """ + def __init__(self, days: int): + """ + :param days: + The number of days to offset. + """ self.days = days @property diff --git a/piccolo/columns/indexes.py b/piccolo/columns/indexes.py index 9b25a1f27..79060277f 100644 --- a/piccolo/columns/indexes.py +++ b/piccolo/columns/indexes.py @@ -2,6 +2,11 @@ class IndexMethod(str, Enum): + """ + Used to specify the index method for a + :class:`Column `. + """ + btree = "btree" hash = "hash" gist = "gist" diff --git a/piccolo/table.py b/piccolo/table.py index dcc0efd46..8bff8e6b9 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -468,9 +468,9 @@ def add_m2m( >>> band = await Band.objects().get(Band.name == "Pythonistas") >>> await band.add_m2m( - >>> Genre(name="Punk rock"), - >>> m2m=Band.genres - >>> ) + ... Genre(name="Punk rock"), + ... m2m=Band.genres + ... ) [{'id': 1}] :param extra_column_values: @@ -514,9 +514,9 @@ def remove_m2m(self, *rows: Table, m2m: M2M) -> M2MRemoveRelated: >>> band = await Band.objects().get(Band.name == "Pythonistas") >>> genre = await Genre.objects().get(Genre.name == "Rock") >>> await band.remove_m2m( - >>> genre, - >>> m2m=Band.genres - >>> ) + ... genre, + ... m2m=Band.genres + ... ) """ return M2MRemoveRelated( From ef54858b86ea936b8bbb85b76225156de650d983 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 14 Feb 2022 19:42:57 +0000 Subject: [PATCH 261/727] Refactor test assertions using `teyit` (#435) * ran teyit * run black --- tests/columns/test_base.py | 4 +- tests/columns/test_db_column_name.py | 6 +- tests/columns/test_double_precision.py | 2 +- tests/columns/test_interval.py | 8 +- tests/columns/test_m2m.py | 12 +-- tests/columns/test_numeric.py | 4 +- tests/columns/test_primary_key.py | 6 +- tests/columns/test_readable.py | 2 +- tests/columns/test_real.py | 2 +- tests/columns/test_time.py | 6 +- tests/columns/test_timestamp.py | 6 +- tests/columns/test_timestamptz.py | 2 +- tests/conf/test_apps.py | 4 +- tests/engine/test_nested_transaction.py | 4 +- tests/engine/test_pool.py | 2 +- tests/query/test_freeze.py | 6 +- tests/table/test_alter.py | 26 +++--- tests/table/test_batch.py | 20 ++--- tests/table/test_count.py | 2 +- tests/table/test_create.py | 2 +- tests/table/test_create_table_class.py | 2 +- tests/table/test_exists.py | 2 +- tests/table/test_indexes.py | 8 +- tests/table/test_insert.py | 6 +- tests/table/test_objects.py | 22 ++--- tests/table/test_output.py | 6 +- tests/table/test_raw.py | 2 +- tests/table/test_ref.py | 2 +- tests/table/test_select.py | 110 +++++++++++------------- tests/table/test_table_exists.py | 2 +- tests/utils/test_pydantic.py | 14 +-- 31 files changed, 146 insertions(+), 156 deletions(-) diff --git a/tests/columns/test_base.py b/tests/columns/test_base.py index 309e558b8..db111f6f1 100644 --- a/tests/columns/test_base.py +++ b/tests/columns/test_base.py @@ -32,7 +32,7 @@ def test_help_text(self): """ help_text = "This is some important help text for users." column = Varchar(help_text=help_text) - self.assertTrue(column._meta.help_text == help_text) + self.assertEqual(column._meta.help_text, help_text) class TestSecretParameter(TestCase): @@ -42,7 +42,7 @@ def test_secret_parameter(self): """ secret = False column = Varchar(secret=secret) - self.assertTrue(column._meta.secret == secret) + self.assertEqual(column._meta.secret, secret) class TestChoices(TestCase): diff --git a/tests/columns/test_db_column_name.py b/tests/columns/test_db_column_name.py index b8da77156..e18d391a3 100644 --- a/tests/columns/test_db_column_name.py +++ b/tests/columns/test_db_column_name.py @@ -48,7 +48,7 @@ def test_save(self): band.save().run_sync() band_from_db = Band.objects().first().run_sync() - self.assertTrue(band_from_db.name == "Pythonistas") + self.assertEqual(band_from_db.name, "Pythonistas") def test_create(self): """ @@ -59,10 +59,10 @@ def test_create(self): .create(name="Pythonistas", popularity=1000) .run_sync() ) - self.assertTrue(band.name == "Pythonistas") + self.assertEqual(band.name, "Pythonistas") band_from_db = Band.objects().first().run_sync() - self.assertTrue(band_from_db.name == "Pythonistas") + self.assertEqual(band_from_db.name, "Pythonistas") def test_select(self): """ diff --git a/tests/columns/test_double_precision.py b/tests/columns/test_double_precision.py index 0b008bc73..20d63331b 100644 --- a/tests/columns/test_double_precision.py +++ b/tests/columns/test_double_precision.py @@ -20,5 +20,5 @@ def test_creation(self): row.save().run_sync() _row = MyTable.objects().first().run_sync() - self.assertTrue(type(_row.column_a) == float) + self.assertEqual(type(_row.column_a), float) self.assertAlmostEqual(_row.column_a, 1.23) diff --git a/tests/columns/test_interval.py b/tests/columns/test_interval.py index 253fd09db..11f30c670 100644 --- a/tests/columns/test_interval.py +++ b/tests/columns/test_interval.py @@ -64,7 +64,7 @@ def test_interval_where_clause(self): .first() .run_sync() ) - self.assertTrue(result is not None) + self.assertIsNotNone(result) result = ( MyTable.objects() @@ -72,7 +72,7 @@ def test_interval_where_clause(self): .first() .run_sync() ) - self.assertTrue(result is not None) + self.assertIsNotNone(result) result = ( MyTable.objects() @@ -80,7 +80,7 @@ def test_interval_where_clause(self): .first() .run_sync() ) - self.assertTrue(result is not None) + self.assertIsNotNone(result) result = ( MyTable.exists() @@ -102,4 +102,4 @@ def test_interval(self): row.save().run_sync() result = MyTableDefault.objects().first().run_sync() - self.assertTrue(result.interval.days == 1) + self.assertEqual(result.interval.days, 1) diff --git a/tests/columns/test_m2m.py b/tests/columns/test_m2m.py index f4a2de4c0..47f034e3a 100644 --- a/tests/columns/test_m2m.py +++ b/tests/columns/test_m2m.py @@ -600,10 +600,10 @@ def test_select_all(self): SmallTable.varchar_col, SmallTable.mega_rows(load_json=True) ).run_sync() - self.assertTrue(len(response) == 1) + self.assertEqual(len(response), 1) mega_rows = response[0]["mega_rows"] - self.assertTrue(len(mega_rows) == 1) + self.assertEqual(len(mega_rows), 1) mega_row = mega_rows[0] for key, value in mega_row.items(): @@ -631,9 +631,7 @@ def test_select_single(self): returned_value = data[column_name] if type(column) == UUID: - self.assertTrue( - type(returned_value) in (uuid.UUID, asyncpgUUID) - ) + self.assertIn(type(returned_value), (uuid.UUID, asyncpgUUID)) else: self.assertEqual( type(original_value), @@ -657,9 +655,7 @@ def test_select_single(self): returned_value = response[0]["mega_rows"][0] if type(column) == UUID: - self.assertTrue( - type(returned_value) in (uuid.UUID, asyncpgUUID) - ) + self.assertIn(type(returned_value), (uuid.UUID, asyncpgUUID)) self.assertEqual(str(original_value), str(returned_value)) else: self.assertEqual( diff --git a/tests/columns/test_numeric.py b/tests/columns/test_numeric.py index 1482dc8ad..0c35e50ec 100644 --- a/tests/columns/test_numeric.py +++ b/tests/columns/test_numeric.py @@ -23,8 +23,8 @@ def test_creation(self): _row = MyTable.objects().first().run_sync() - self.assertTrue(type(_row.column_a) == Decimal) - self.assertTrue(type(_row.column_b) == Decimal) + self.assertEqual(type(_row.column_a), Decimal) + self.assertEqual(type(_row.column_b), Decimal) self.assertAlmostEqual(_row.column_a, Decimal(1.23)) self.assertEqual(_row.column_b, Decimal("1.23")) diff --git a/tests/columns/test_primary_key.py b/tests/columns/test_primary_key.py index 30ba19ccc..98d1f5d4c 100644 --- a/tests/columns/test_primary_key.py +++ b/tests/columns/test_primary_key.py @@ -139,7 +139,7 @@ def test_primary_key_queries(self): ["pk", "name"], ) - self.assertTrue(isinstance(manager_dict["pk"], uuid.UUID)) + self.assertIsInstance(manager_dict["pk"], uuid.UUID) ####################################################################### # Make sure we can create rows with foreign keys to tables with a @@ -155,8 +155,8 @@ def test_primary_key_queries(self): self.assertEqual( [i for i in band_dict.keys()], ["pk", "name", "manager"] ) - self.assertTrue(isinstance(band_dict["pk"], uuid.UUID)) - self.assertTrue(isinstance(band_dict["manager"], uuid.UUID)) + self.assertIsInstance(band_dict["pk"], uuid.UUID) + self.assertIsInstance(band_dict["manager"], uuid.UUID) ####################################################################### # Make sure foreign key values can be specified as the primary key's diff --git a/tests/columns/test_readable.py b/tests/columns/test_readable.py index e7225fbd1..5b214896a 100644 --- a/tests/columns/test_readable.py +++ b/tests/columns/test_readable.py @@ -23,7 +23,7 @@ def setUp(self): def test_readable(self): response = MyTable.select(MyTable.get_readable()).run_sync() - self.assertTrue(response[0]["readable"] == "Guido van Rossum") + self.assertEqual(response[0]["readable"], "Guido van Rossum") def tearDown(self): MyTable.alter().drop_table().run_sync() diff --git a/tests/columns/test_real.py b/tests/columns/test_real.py index 09bcdeb40..30dc4338d 100644 --- a/tests/columns/test_real.py +++ b/tests/columns/test_real.py @@ -20,5 +20,5 @@ def test_creation(self): row.save().run_sync() _row = MyTable.objects().first().run_sync() - self.assertTrue(type(_row.column_a) == float) + self.assertEqual(type(_row.column_a), float) self.assertAlmostEqual(_row.column_a, 1.23) diff --git a/tests/columns/test_time.py b/tests/columns/test_time.py index 56282b262..d018bbcce 100644 --- a/tests/columns/test_time.py +++ b/tests/columns/test_time.py @@ -46,7 +46,7 @@ def test_timestamp(self): _datetime = partial(datetime.datetime, year=2020, month=1, day=1) result = MyTableDefault.objects().first().run_sync() - self.assertTrue( + self.assertLess( _datetime( hour=result.created_on.hour, minute=result.created_on.minute, @@ -56,6 +56,6 @@ def test_timestamp(self): hour=created_on.hour, minute=created_on.minute, second=created_on.second, - ) - < datetime.timedelta(seconds=1) + ), + datetime.timedelta(seconds=1), ) diff --git a/tests/columns/test_timestamp.py b/tests/columns/test_timestamp.py index 70ba3fa28..2c79728e9 100644 --- a/tests/columns/test_timestamp.py +++ b/tests/columns/test_timestamp.py @@ -61,7 +61,7 @@ def test_timestamp(self): row.save().run_sync() result = MyTableDefault.objects().first().run_sync() - self.assertTrue( - result.created_on - created_on < datetime.timedelta(seconds=1) + self.assertLess( + result.created_on - created_on, datetime.timedelta(seconds=1) ) - self.assertTrue(result.created_on.tzinfo is None) + self.assertIsNone(result.created_on.tzinfo) diff --git a/tests/columns/test_timestamptz.py b/tests/columns/test_timestamptz.py index 3f66ab4dc..09755e340 100644 --- a/tests/columns/test_timestamptz.py +++ b/tests/columns/test_timestamptz.py @@ -94,5 +94,5 @@ def test_timestamptz_default(self): result = MyTableDefault.objects().first().run_sync() delta = result.created_on - created_on - self.assertTrue(delta < datetime.timedelta(seconds=1)) + self.assertLess(delta, datetime.timedelta(seconds=1)) self.assertEqual(result.created_on.tzinfo, datetime.timezone.utc) diff --git a/tests/conf/test_apps.py b/tests/conf/test_apps.py index 0435b6d5e..717f9f857 100644 --- a/tests/conf/test_apps.py +++ b/tests/conf/test_apps.py @@ -19,12 +19,12 @@ class TestAppRegistry(TestCase): def test_get_app_config(self): app_registry = AppRegistry(apps=["piccolo.apps.user.piccolo_app"]) app_config = app_registry.get_app_config(app_name="user") - self.assertTrue(isinstance(app_config, AppConfig)) + self.assertIsInstance(app_config, AppConfig) def test_get_table_classes(self): app_registry = AppRegistry(apps=["piccolo.apps.user.piccolo_app"]) table_classes = app_registry.get_table_classes(app_name="user") - self.assertTrue(BaseUser in table_classes) + self.assertIn(BaseUser, table_classes) with self.assertRaises(ValueError): app_registry.get_table_classes(app_name="Foo") diff --git a/tests/engine/test_nested_transaction.py b/tests/engine/test_nested_transaction.py index cfc08e35b..dd1347b7d 100644 --- a/tests/engine/test_nested_transaction.py +++ b/tests/engine/test_nested_transaction.py @@ -45,11 +45,11 @@ async def run_nested(self): self.assertTrue(await Musician.table_exists().run()) musician = await Musician.select("name").first().run() - self.assertTrue(musician["name"] == "Bob") + self.assertEqual(musician["name"], "Bob") self.assertTrue(await Roadie.table_exists().run()) roadie = await Roadie.select("name").first().run() - self.assertTrue(roadie["name"] == "Dave") + self.assertEqual(roadie["name"], "Dave") def test_nested(self): asyncio.run(self.run_nested()) diff --git a/tests/engine/test_pool.py b/tests/engine/test_pool.py index b061ef042..21c23870a 100644 --- a/tests/engine/test_pool.py +++ b/tests/engine/test_pool.py @@ -26,7 +26,7 @@ async def _make_query(self): await Manager(name="Bob").save().run() response = await Manager.select().run() - self.assertTrue("Bob" in [i["name"] for i in response]) + self.assertIn("Bob", [i["name"] for i in response]) await Manager._meta.db.close_connection_pool() diff --git a/tests/query/test_freeze.py b/tests/query/test_freeze.py index 255d27a9f..9b264012c 100644 --- a/tests/query/test_freeze.py +++ b/tests/query/test_freeze.py @@ -95,9 +95,9 @@ def test_frozen_performance(self): ) # Remove the outliers before comparing - self.assertTrue( - sum(sorted(query_duration)[5:-5]) - > sum(sorted(frozen_query_duration)[5:-5]) + self.assertGreater( + sum(sorted(query_duration)[5:-5]), + sum(sorted(frozen_query_duration)[5:-5]), ) def test_attribute_access(self): diff --git a/tests/table/test_alter.py b/tests/table/test_alter.py index 125b387f0..5b37c8c90 100644 --- a/tests/table/test_alter.py +++ b/tests/table/test_alter.py @@ -86,7 +86,7 @@ def _test_drop(self, column: str): response = Band.raw("SELECT * FROM band").run_sync() column_names = response[0].keys() - self.assertTrue("popularity" not in column_names) + self.assertNotIn("popularity", column_names) def test_drop_string(self): self._test_drop(Band.popularity) @@ -105,7 +105,7 @@ def _test_add_column( response = Band.raw("SELECT * FROM band").run_sync() column_names = response[0].keys() - self.assertTrue(column_name in column_names) + self.assertIn(column_name, column_names) self.assertEqual(response[0][column_name], expected_value) @@ -158,7 +158,7 @@ def test_unique(self): Manager(name="Bob").save().run_sync() response = Manager.select().run_sync() - self.assertTrue(len(response) == 2) + self.assertEqual(len(response), 2) # Now remove the constraint, and add a row. not_unique_query = Manager.alter().set_unique(Manager.name, False) @@ -188,8 +188,8 @@ def test_multiple(self): response = Band.raw("SELECT * FROM manager").run_sync() column_names = response[0].keys() - self.assertTrue("column_a" in column_names) - self.assertTrue("column_b" in column_names) + self.assertIn("column_a", column_names) + self.assertIn("column_b", column_names) # TODO - test more conversions. @@ -271,11 +271,11 @@ def test_set_null(self): Band.alter().set_null(Band.popularity, boolean=True).run_sync() response = Band.raw(query).run_sync() - self.assertTrue(response[0]["is_nullable"] == "YES") + self.assertEqual(response[0]["is_nullable"], "YES") Band.alter().set_null(Band.popularity, boolean=False).run_sync() response = Band.raw(query).run_sync() - self.assertTrue(response[0]["is_nullable"] == "NO") + self.assertEqual(response[0]["is_nullable"], "NO") @postgres_only @@ -291,7 +291,7 @@ def test_set_length(self): for length in (5, 20, 50): Band.alter().set_length(Band.name, length=length).run_sync() response = Band.raw(query).run_sync() - self.assertTrue(response[0]["character_maximum_length"] == length) + self.assertEqual(response[0]["character_maximum_length"], length) @postgres_only @@ -305,7 +305,7 @@ def test_set_default(self): ).run_sync() manager = Manager.objects().first().run_sync() - self.assertTrue(manager.name == "Pending") + self.assertEqual(manager.name, "Pending") ############################################################################### @@ -336,10 +336,10 @@ def test_set_digits(self): column=Ticket.price, digits=(6, 2) ).run_sync() response = Ticket.raw(query).run_sync() - self.assertTrue(response[0]["numeric_precision"] == 6) - self.assertTrue(response[0]["numeric_scale"] == 2) + self.assertEqual(response[0]["numeric_precision"], 6) + self.assertEqual(response[0]["numeric_scale"], 2) Ticket.alter().set_digits(column=Ticket.price, digits=None).run_sync() response = Ticket.raw(query).run_sync() - self.assertTrue(response[0]["numeric_precision"] is None) - self.assertTrue(response[0]["numeric_scale"] is None) + self.assertIsNone(response[0]["numeric_precision"]) + self.assertIsNone(response[0]["numeric_scale"]) diff --git a/tests/table/test_batch.py b/tests/table/test_batch.py index 114c683fa..a0a5b34ee 100644 --- a/tests/table/test_batch.py +++ b/tests/table/test_batch.py @@ -10,12 +10,12 @@ def _check_results(self, batch): """ Make sure the data is returned in the correct format. """ - self.assertTrue(type(batch) == list) + self.assertEqual(type(batch), list) if len(batch) > 0: row = batch[0] - self.assertTrue(type(row) == dict) - self.assertTrue("name" in row.keys()) - self.assertTrue("id" in row.keys()) + self.assertEqual(type(row), dict) + self.assertIn("name", row.keys()) + self.assertIn("id", row.keys()) async def run_batch(self, batch_size): row_count = 0 @@ -44,8 +44,8 @@ def test_batch(self): _iterations = math.ceil(row_count / batch_size) - self.assertTrue(_row_count == row_count) - self.assertTrue(iterations == _iterations) + self.assertEqual(_row_count, row_count) + self.assertEqual(iterations, _iterations) class TestBatchObjects(DBTestCase): @@ -53,10 +53,10 @@ def _check_results(self, batch): """ Make sure the data is returned in the correct format. """ - self.assertTrue(type(batch) == list) + self.assertEqual(type(batch), list) if len(batch) > 0: row = batch[0] - self.assertTrue(isinstance(row, Manager)) + self.assertIsInstance(row, Manager) async def run_batch(self, batch_size): row_count = 0 @@ -85,5 +85,5 @@ def test_batch(self): _iterations = math.ceil(row_count / batch_size) - self.assertTrue(_row_count == row_count) - self.assertTrue(iterations == _iterations) + self.assertEqual(_row_count, row_count) + self.assertEqual(iterations, _iterations) diff --git a/tests/table/test_count.py b/tests/table/test_count.py index a5943a51d..b6eafd11d 100644 --- a/tests/table/test_count.py +++ b/tests/table/test_count.py @@ -8,4 +8,4 @@ def test_exists(self): response = Band.count().where(Band.name == "Pythonistas").run_sync() - self.assertTrue(response == 1) + self.assertEqual(response, 1) diff --git a/tests/table/test_create.py b/tests/table/test_create.py index 521c4f7d6..6f95fa381 100644 --- a/tests/table/test_create.py +++ b/tests/table/test_create.py @@ -27,7 +27,7 @@ def tearDown(self): def test_create_table_with_indexes(self): index_names = BandMember.indexes().run_sync() index_name = BandMember._get_index_name(["name"]) - self.assertTrue(index_name in index_names) + self.assertIn(index_name, index_names) def test_create_if_not_exists_with_indexes(self): """ diff --git a/tests/table/test_create_table_class.py b/tests/table/test_create_table_class.py index 1c891e443..9c6f013d3 100644 --- a/tests/table/test_create_table_class.py +++ b/tests/table/test_create_table_class.py @@ -21,7 +21,7 @@ def test_create_table_class(self): _Table = create_table_class( class_name="MyTable", class_members={"name": column} ) - self.assertTrue(column in _Table._meta.columns) + self.assertIn(column, _Table._meta.columns) def test_protected_tablenames(self): """ diff --git a/tests/table/test_exists.py b/tests/table/test_exists.py index 7650ad85f..8d8f07cc0 100644 --- a/tests/table/test_exists.py +++ b/tests/table/test_exists.py @@ -8,4 +8,4 @@ def test_exists(self): response = Band.exists().where(Band.name == "Pythonistas").run_sync() - self.assertTrue(response is True) + self.assertTrue(response) diff --git a/tests/table/test_indexes.py b/tests/table/test_indexes.py index 383dc06f6..6aebd350c 100644 --- a/tests/table/test_indexes.py +++ b/tests/table/test_indexes.py @@ -27,11 +27,11 @@ def test_create_index(self): ) index_names = Manager.indexes().run_sync() - self.assertTrue(index_name in index_names) + self.assertIn(index_name, index_names) Manager.drop_index(columns).run_sync() index_names = Manager.indexes().run_sync() - self.assertTrue(index_name not in index_names) + self.assertNotIn(index_name, index_names) class Concert(Table): @@ -55,11 +55,11 @@ def test_problematic_name(self): index_name = Concert._get_index_name([i._meta.name for i in columns]) index_names = Concert.indexes().run_sync() - self.assertTrue(index_name in index_names) + self.assertIn(index_name, index_names) Concert.drop_index(columns).run_sync() index_names = Concert.indexes().run_sync() - self.assertTrue(index_name not in index_names) + self.assertNotIn(index_name, index_names) class TestIndexName(TestCase): diff --git a/tests/table/test_insert.py b/tests/table/test_insert.py index b2f9bf33a..7822811e6 100644 --- a/tests/table/test_insert.py +++ b/tests/table/test_insert.py @@ -11,7 +11,7 @@ def test_insert(self): response = Band.select(Band.name).run_sync() names = [i["name"] for i in response] - self.assertTrue("Rustaceans" in names) + self.assertIn("Rustaceans", names) def test_add(self): self.insert_rows() @@ -21,7 +21,7 @@ def test_add(self): response = Band.select(Band.name).run_sync() names = [i["name"] for i in response] - self.assertTrue("Rustaceans" in names) + self.assertIn("Rustaceans", names) def test_incompatible_type(self): """ @@ -41,4 +41,4 @@ def test_insert_curly_braces(self): response = Band.select(Band.name).run_sync() names = [i["name"] for i in response] - self.assertTrue("{}" in names) + self.assertIn("{}", names) diff --git a/tests/table/test_objects.py b/tests/table/test_objects.py index 0a0a6fbaa..4593c0ca3 100644 --- a/tests/table/test_objects.py +++ b/tests/table/test_objects.py @@ -8,21 +8,21 @@ def test_get_all(self): response = Band.objects().run_sync() - self.assertTrue(len(response) == 1) + self.assertEqual(len(response), 1) instance = response[0] - self.assertTrue(isinstance(instance, Band)) - self.assertTrue(instance.name == "Pythonistas") + self.assertIsInstance(instance, Band) + self.assertEqual(instance.name, "Pythonistas") # Now try changing the value and saving it. instance.name = "Rustaceans" save_query = instance.save() save_query.run_sync() - self.assertTrue( - Band.select(Band.name).output(as_list=True).run_sync()[0] - == "Rustaceans" + self.assertEqual( + Band.select(Band.name).output(as_list=True).run_sync()[0], + "Rustaceans", ) @postgres_only @@ -60,7 +60,7 @@ def test_get(self): band = Band.objects().get(Band.name == "Pythonistas").run_sync() - self.assertTrue(band.name == "Pythonistas") + self.assertEqual(band.name, "Pythonistas") def test_get__prefetch(self): self.insert_rows() @@ -96,8 +96,8 @@ def test_get_or_create(self): ) self.assertIsInstance(instance, Band) - self.assertTrue(instance.name == "Pink Floyd") - self.assertTrue(instance.popularity == 100) + self.assertEqual(instance.name, "Pink Floyd") + self.assertEqual(instance.popularity, 100) # When the row already exists in the db: Band.objects().get_or_create( @@ -109,8 +109,8 @@ def test_get_or_create(self): ) self.assertIsInstance(instance, Band) - self.assertTrue(instance.name == "Pink Floyd") - self.assertTrue(instance.popularity == 100) + self.assertEqual(instance.name, "Pink Floyd") + self.assertEqual(instance.popularity, 100) def test_get_or_create_complex(self): """ diff --git a/tests/table/test_output.py b/tests/table/test_output.py index e4d21ff12..683e9097c 100644 --- a/tests/table/test_output.py +++ b/tests/table/test_output.py @@ -10,7 +10,7 @@ def test_output_as_list(self): self.insert_row() response = Band.select(Band.name).output(as_list=True).run_sync() - self.assertTrue(response == ["Pythonistas"]) + self.assertEqual(response, ["Pythonistas"]) # Make sure that if no rows are found, an empty list is returned. empty_response = ( @@ -19,7 +19,7 @@ def test_output_as_list(self): .output(as_list=True) .run_sync() ) - self.assertTrue(empty_response == []) + self.assertEqual(empty_response, []) class TestOutputJSON(DBTestCase): @@ -28,7 +28,7 @@ def test_output_as_json(self): response = Band.select(Band.name).output(as_json=True).run_sync() - self.assertTrue(json.loads(response) == [{"name": "Pythonistas"}]) + self.assertEqual(json.loads(response), [{"name": "Pythonistas"}]) class TestOutputLoadJSON(TestCase): diff --git a/tests/table/test_raw.py b/tests/table/test_raw.py index 3e6ebdb41..12fa372fc 100644 --- a/tests/table/test_raw.py +++ b/tests/table/test_raw.py @@ -20,7 +20,7 @@ def test_raw_with_args(self): "select * from band where name = {}", "Pythonistas" ).run_sync() - self.assertTrue(len(response) == 1) + self.assertEqual(len(response), 1) self.assertDictEqual( response[0], {"id": 1, "name": "Pythonistas", "manager": 1, "popularity": 1000}, diff --git a/tests/table/test_ref.py b/tests/table/test_ref.py index fb973f086..0d6645eff 100644 --- a/tests/table/test_ref.py +++ b/tests/table/test_ref.py @@ -7,4 +7,4 @@ class TestRef(TestCase): def test_ref(self): column = Band.ref("manager.name") - self.assertTrue(isinstance(column, Varchar)) + self.assertIsInstance(column, Varchar) diff --git a/tests/table/test_select.py b/tests/table/test_select.py index f00421762..bb8308600 100644 --- a/tests/table/test_select.py +++ b/tests/table/test_select.py @@ -381,7 +381,7 @@ def test_multiple_where(self): response = query.run_sync() self.assertEqual(response, [{"name": "Rustaceans"}]) - self.assertTrue("AND" in query.__str__()) + self.assertIn("AND", query.__str__()) def test_complex_where(self): """ @@ -489,18 +489,18 @@ def test_distinct(self): self.insert_rows() query = Band.select(Band.name).where(Band.name == "Pythonistas") - self.assertTrue("DISTINCT" not in query.__str__()) + self.assertNotIn("DISTINCT", query.__str__()) response = query.run_sync() - self.assertTrue( - response == [{"name": "Pythonistas"}, {"name": "Pythonistas"}] + self.assertEqual( + response, [{"name": "Pythonistas"}, {"name": "Pythonistas"}] ) query = query.distinct() - self.assertTrue("DISTINCT" in query.__str__()) + self.assertIn("DISTINCT", query.__str__()) response = query.run_sync() - self.assertTrue(response == [{"name": "Pythonistas"}]) + self.assertEqual(response, [{"name": "Pythonistas"}]) def test_count_group_by(self): """ @@ -516,13 +516,13 @@ def test_count_group_by(self): .run_sync() ) - self.assertTrue( - response - == [ + self.assertEqual( + response, + [ {"name": "CSharps", "count": 2}, {"name": "Pythonistas", "count": 2}, {"name": "Rustaceans", "count": 2}, - ] + ], ) def test_count_with_alias_group_by(self): @@ -539,13 +539,13 @@ def test_count_with_alias_group_by(self): .run_sync() ) - self.assertTrue( - response - == [ + self.assertEqual( + response, + [ {"name": "CSharps", "total": 2}, {"name": "Pythonistas", "total": 2}, {"name": "Rustaceans", "total": 2}, - ] + ], ) def test_count_with_as_alias_group_by(self): @@ -562,13 +562,13 @@ def test_count_with_as_alias_group_by(self): .run_sync() ) - self.assertTrue( - response - == [ + self.assertEqual( + response, + [ {"name": "CSharps", "total": 2}, {"name": "Pythonistas", "total": 2}, {"name": "Rustaceans", "total": 2}, - ] + ], ) def test_count_column_group_by(self): @@ -602,14 +602,14 @@ def test_count_column_group_by(self): # differently when sorting. response = sorted(response, key=lambda x: x["manager.name"] or "") - self.assertTrue( - response - == [ + self.assertEqual( + response, + [ {"manager.name": None, "count": 0}, {"manager.name": "Graydon", "count": 2}, {"manager.name": "Guido", "count": 2}, {"manager.name": "Mads", "count": 2}, - ] + ], ) # This time the nulls should be counted, as we omit the column argument @@ -623,14 +623,14 @@ def test_count_column_group_by(self): response = sorted(response, key=lambda x: x["manager.name"] or "") - self.assertTrue( - response - == [ + self.assertEqual( + response, + [ {"manager.name": None, "count": 1}, {"manager.name": "Graydon", "count": 2}, {"manager.name": "Guido", "count": 2}, {"manager.name": "Mads", "count": 2}, - ] + ], ) def test_avg(self): @@ -638,7 +638,7 @@ def test_avg(self): response = Band.select(Avg(Band.popularity)).first().run_sync() - self.assertTrue(float(response["avg"]) == 1003.3333333333334) + self.assertEqual(float(response["avg"]), 1003.3333333333334) def test_avg_alias(self): self.insert_rows() @@ -649,9 +649,7 @@ def test_avg_alias(self): .run_sync() ) - self.assertTrue( - float(response["popularity_avg"]) == 1003.3333333333334 - ) + self.assertEqual(float(response["popularity_avg"]), 1003.3333333333334) def test_avg_as_alias_method(self): self.insert_rows() @@ -662,9 +660,7 @@ def test_avg_as_alias_method(self): .run_sync() ) - self.assertTrue( - float(response["popularity_avg"]) == 1003.3333333333334 - ) + self.assertEqual(float(response["popularity_avg"]), 1003.3333333333334) def test_avg_with_where_clause(self): self.insert_rows() @@ -676,7 +672,7 @@ def test_avg_with_where_clause(self): .run_sync() ) - self.assertTrue(response["avg"] == 1500) + self.assertEqual(response["avg"], 1500) def test_avg_alias_with_where_clause(self): """ @@ -692,7 +688,7 @@ def test_avg_alias_with_where_clause(self): .run_sync() ) - self.assertTrue(response["popularity_avg"] == 1500) + self.assertEqual(response["popularity_avg"], 1500) def test_avg_as_alias_method_with_where_clause(self): """ @@ -708,14 +704,14 @@ def test_avg_as_alias_method_with_where_clause(self): .run_sync() ) - self.assertTrue(response["popularity_avg"] == 1500) + self.assertEqual(response["popularity_avg"], 1500) def test_max(self): self.insert_rows() response = Band.select(Max(Band.popularity)).first().run_sync() - self.assertTrue(response["max"] == 2000) + self.assertEqual(response["max"], 2000) def test_max_alias(self): self.insert_rows() @@ -726,7 +722,7 @@ def test_max_alias(self): .run_sync() ) - self.assertTrue(response["popularity_max"] == 2000) + self.assertEqual(response["popularity_max"], 2000) def test_max_as_alias_method(self): self.insert_rows() @@ -737,14 +733,14 @@ def test_max_as_alias_method(self): .run_sync() ) - self.assertTrue(response["popularity_max"] == 2000) + self.assertEqual(response["popularity_max"], 2000) def test_min(self): self.insert_rows() response = Band.select(Min(Band.popularity)).first().run_sync() - self.assertTrue(response["min"] == 10) + self.assertEqual(response["min"], 10) def test_min_alias(self): self.insert_rows() @@ -755,7 +751,7 @@ def test_min_alias(self): .run_sync() ) - self.assertTrue(response["popularity_min"] == 10) + self.assertEqual(response["popularity_min"], 10) def test_min_as_alias_method(self): self.insert_rows() @@ -766,14 +762,14 @@ def test_min_as_alias_method(self): .run_sync() ) - self.assertTrue(response["popularity_min"] == 10) + self.assertEqual(response["popularity_min"], 10) def test_sum(self): self.insert_rows() response = Band.select(Sum(Band.popularity)).first().run_sync() - self.assertTrue(response["sum"] == 3010) + self.assertEqual(response["sum"], 3010) def test_sum_alias(self): self.insert_rows() @@ -784,7 +780,7 @@ def test_sum_alias(self): .run_sync() ) - self.assertTrue(response["popularity_sum"] == 3010) + self.assertEqual(response["popularity_sum"], 3010) def test_sum_as_alias_method(self): self.insert_rows() @@ -795,7 +791,7 @@ def test_sum_as_alias_method(self): .run_sync() ) - self.assertTrue(response["popularity_sum"] == 3010) + self.assertEqual(response["popularity_sum"], 3010) def test_sum_with_where_clause(self): self.insert_rows() @@ -807,7 +803,7 @@ def test_sum_with_where_clause(self): .run_sync() ) - self.assertTrue(response["sum"] == 3000) + self.assertEqual(response["sum"], 3000) def test_sum_alias_with_where_clause(self): """ @@ -823,7 +819,7 @@ def test_sum_alias_with_where_clause(self): .run_sync() ) - self.assertTrue(response["popularity_sum"] == 3000) + self.assertEqual(response["popularity_sum"], 3000) def test_sum_as_alias_method_with_where_clause(self): """ @@ -839,7 +835,7 @@ def test_sum_as_alias_method_with_where_clause(self): .run_sync() ) - self.assertTrue(response["popularity_sum"] == 3000) + self.assertEqual(response["popularity_sum"], 3000) def test_chain_different_functions(self): self.insert_rows() @@ -850,8 +846,8 @@ def test_chain_different_functions(self): .run_sync() ) - self.assertTrue(float(response["avg"]) == 1003.3333333333334) - self.assertTrue(response["sum"] == 3010) + self.assertEqual(float(response["avg"]), 1003.3333333333334) + self.assertEqual(response["sum"], 3010) def test_chain_different_functions_alias(self): self.insert_rows() @@ -865,10 +861,8 @@ def test_chain_different_functions_alias(self): .run_sync() ) - self.assertTrue( - float(response["popularity_avg"]) == 1003.3333333333334 - ) - self.assertTrue(response["popularity_sum"] == 3010) + self.assertEqual(float(response["popularity_avg"]), 1003.3333333333334) + self.assertEqual(response["popularity_sum"], 3010) def test_avg_validation(self): with self.assertRaises(ValueError): @@ -892,7 +886,7 @@ def test_columns(self): .first() .run_sync() ) - self.assertTrue(response == {"name": "Pythonistas"}) + self.assertEqual(response, {"name": "Pythonistas"}) # Multiple calls to 'columns' should be additive. response = ( @@ -903,7 +897,7 @@ def test_columns(self): .first() .run_sync() ) - self.assertTrue(response == {"id": 1, "name": "Pythonistas"}) + self.assertEqual(response, {"id": 1, "name": "Pythonistas"}) def test_call_chain(self): """ @@ -963,7 +957,7 @@ def test_secret(self): user.save().run_sync() user_dict = BaseUser.select(exclude_secrets=True).first().run_sync() - self.assertTrue("password" not in user_dict.keys()) + self.assertNotIn("password", user_dict.keys()) class TestSelectSecretParameter(TestCase): @@ -983,4 +977,4 @@ def test_secret_parameter(self): venue_dict = Venue.select(exclude_secrets=True).first().run_sync() self.assertTrue(venue_dict, {"id": 1, "name": "The Garage"}) - self.assertTrue("capacity" not in venue_dict.keys()) + self.assertNotIn("capacity", venue_dict.keys()) diff --git a/tests/table/test_table_exists.py b/tests/table/test_table_exists.py index e35af2288..8cdd18b76 100644 --- a/tests/table/test_table_exists.py +++ b/tests/table/test_table_exists.py @@ -9,7 +9,7 @@ def setUp(self): def test_table_exists(self): response = Manager.table_exists().run_sync() - self.assertTrue(response is True) + self.assertTrue(response) def tearDown(self): Manager.alter().drop_table().run_sync() diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index d56c83c42..cc891be37 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -494,7 +494,7 @@ class Concert(Table): self.assertEqual(ManagerModel.__qualname__, "Band.manager") AssistantManagerType = BandModel.__fields__["assistant_manager"].type_ - self.assertTrue(AssistantManagerType is int) + self.assertIs(AssistantManagerType, int) ####################################################################### # Test two levels deep @@ -511,7 +511,7 @@ class Concert(Table): self.assertEqual(ManagerModel.__qualname__, "Band.manager") AssistantManagerType = BandModel.__fields__["assistant_manager"].type_ - self.assertTrue(AssistantManagerType is int) + self.assertIs(AssistantManagerType, int) CountryModel = ManagerModel.__fields__["country"].type_ self.assertTrue(issubclass(CountryModel, pydantic.BaseModel)) @@ -526,7 +526,7 @@ class Concert(Table): ) VenueModel = ConcertModel.__fields__["venue"].type_ - self.assertTrue(VenueModel is int) + self.assertIs(VenueModel, int) BandModel = ConcertModel.__fields__["band_1"].type_ self.assertTrue(issubclass(BandModel, pydantic.BaseModel)) @@ -545,10 +545,10 @@ class Concert(Table): self.assertEqual(ManagerModel.__qualname__, "Concert.band_1.manager") AssistantManagerType = BandModel.__fields__["assistant_manager"].type_ - self.assertTrue(AssistantManagerType is int) + self.assertIs(AssistantManagerType, int) CountryModel = ManagerModel.__fields__["country"].type_ - self.assertTrue(CountryModel is int) + self.assertIs(CountryModel, int) ####################################################################### # Test with `model_name` arg @@ -638,7 +638,7 @@ class Concert(Table): # We should have hit the recursion depth: CountryModel = ManagerModel.__fields__["country"].type_ - self.assertTrue(CountryModel is int) + self.assertIs(CountryModel, int) class TestDBColumnName(TestCase): @@ -655,7 +655,7 @@ class Band(Table): model = BandModel(regrettable_column_name="test") - self.assertTrue(model.name == "test") + self.assertEqual(model.name, "test") class TestSchemaExtraKwargs(TestCase): From 988657d62e93ffa213a56ba81cdac772fe6ed96e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 16 Feb 2022 12:55:48 +0000 Subject: [PATCH 262/727] Docs for renaming migration files, and `is_in` / `not_in` (#437) * add better docs for `is_in` and `not_in` * fix indentation of list on Changes page * mention that migration files can be renamed --- CHANGES.rst | 14 +++++++------- docs/src/piccolo/migrations/create.rst | 7 ++++++- docs/src/piccolo/query_clauses/where.rst | 8 ++++++-- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 4fd4d8f2b..bee1a2835 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,13 +4,13 @@ Changes 0.69.2 ------ - * Lots of documentation improvements, including how to customise ``BaseUser`` - (courtesy @sinisaos). - * Fixed a bug with creating indexes when the column name clashes with a SQL - keyword (e.g. ``'order'``). See `Pr 433 `_. - Thanks to @wmshort for reporting this issue. - * Fixed an issue where some slots were incorrectly configured (courtesy - @ariebovenberg). See `PR 426 `_. +* Lots of documentation improvements, including how to customise ``BaseUser`` + (courtesy @sinisaos). +* Fixed a bug with creating indexes when the column name clashes with a SQL + keyword (e.g. ``'order'``). See `Pr 433 `_. + Thanks to @wmshort for reporting this issue. +* Fixed an issue where some slots were incorrectly configured (courtesy + @ariebovenberg). See `PR 426 `_. ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/migrations/create.rst b/docs/src/piccolo/migrations/create.rst index ceeb326d3..b836bf86e 100644 --- a/docs/src/piccolo/migrations/create.rst +++ b/docs/src/piccolo/migrations/create.rst @@ -12,13 +12,15 @@ automatically. To create an empty migration: piccolo migrations new my_app This creates a new migration file in the migrations folder of the app. The -migration filename is a timestamp, which also serves as the migration ID. +migration filename is a timestamp: .. code-block:: bash piccolo_migrations/ 2021-08-06T16-22-51-415781.py +.. hint:: You can rename this file if you like to make it more memorable. + The contents of an empty migration file looks like this: .. code-block:: python @@ -40,6 +42,9 @@ The contents of an empty migration file looks like this: manager.add_raw(run) return manager +The ``ID`` is very important - it uniquely identifies the migration, and +shouldn't be changed. + Replace the ``run`` function with whatever you want the migration to do - typically running some SQL. It can be a function or a coroutine. diff --git a/docs/src/piccolo/query_clauses/where.rst b/docs/src/piccolo/query_clauses/where.rst index 29e683430..cc2785729 100644 --- a/docs/src/piccolo/query_clauses/where.rst +++ b/docs/src/piccolo/query_clauses/where.rst @@ -94,16 +94,20 @@ Usage is the same as ``like`` excepts it excludes matching rows. is_in / not_in -------------- +You can get all rows with a value contained in the list: + .. code-block:: python await Band.select().where( - Band.name.is_in(['Pythonistas']) + Band.name.is_in(['Pythonistas', 'Rustaceans']) ) +And all rows with a value not contained in the list: + .. code-block:: python await Band.select().where( - Band.name.not_in(['Rustaceans']) + Band.name.not_in(['Terrible Band', 'Awful Band']) ) ------------------------------------------------------------------------------- From 9bd81c092bc67a56524636ca20fbb05f3954f138 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 16 Feb 2022 14:56:36 +0000 Subject: [PATCH 263/727] fix rst lists With RST, there is is no space before the bullet. If you add a space, then it gets wrapped in a blockquote. --- CHANGES.rst | 352 +++++++++--------- docs/src/piccolo/authentication/baseuser.rst | 4 +- .../projects_and_apps/included_apps.rst | 4 +- piccolo/conf/apps.py | 4 +- 4 files changed, 182 insertions(+), 182 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index bee1a2835..69cce5797 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -476,16 +476,16 @@ Similarly: 0.59.0 ------ - * When using ``piccolo asgi new`` to generate a FastAPI app, the generated code - is now cleaner. It also contains a ``conftest.py`` file, which encourages - people to use ``piccolo tester run`` rather than using ``pytest`` directly. - * Tidied up docs, and added logo. - * Clarified the use of the ``PICCOLO_CONF`` environment variable in the docs - (courtesy @theelderbeever). - * ``create_pydantic_model`` now accepts an ``include_columns`` argument, in - case you only want a few columns in your model, it's faster than using - ``exclude_columns`` (courtesy @sinisaos). - * Updated linters, and fixed new errors. +* When using ``piccolo asgi new`` to generate a FastAPI app, the generated code + is now cleaner. It also contains a ``conftest.py`` file, which encourages + people to use ``piccolo tester run`` rather than using ``pytest`` directly. +* Tidied up docs, and added logo. +* Clarified the use of the ``PICCOLO_CONF`` environment variable in the docs + (courtesy @theelderbeever). +* ``create_pydantic_model`` now accepts an ``include_columns`` argument, in + case you only want a few columns in your model, it's faster than using + ``exclude_columns`` (courtesy @sinisaos). +* Updated linters, and fixed new errors. ------------------------------------------------------------------------------- @@ -579,8 +579,8 @@ Fixed schema generation bug When using ``piccolo schema generate`` to auto generate Piccolo ``Table`` classes from an existing database, it would fail in this situation: - * A table has a column with an index. - * The column name clashed with a Postgres type. +* A table has a column with an index. +* The column name clashed with a Postgres type. For example, we couldn't auto generate this ``Table`` class: @@ -770,15 +770,15 @@ Here's an example: ------ Lots of improvements to ``piccolo schema generate``: - * Dramatically improved performance, by executing more queries in parallel - (courtesy @AliSayyah). - * If a table in the database has a foreign key to a table in another - schema, this will now work (courtesy @AliSayyah). - * The column defaults are now extracted from the database (courtesy @wmshort). - * The ``scale`` and ``precision`` values for ``Numeric`` / ``Decimal`` column - types are extracted from the database (courtesy @wmshort). - * The ``ON DELETE`` and ``ON UPDATE`` values for ``ForeignKey`` columns are - now extracted from the database (courtesy @wmshort). +* Dramatically improved performance, by executing more queries in parallel + (courtesy @AliSayyah). +* If a table in the database has a foreign key to a table in another + schema, this will now work (courtesy @AliSayyah). +* The column defaults are now extracted from the database (courtesy @wmshort). +* The ``scale`` and ``precision`` values for ``Numeric`` / ``Decimal`` column + types are extracted from the database (courtesy @wmshort). +* The ``ON DELETE`` and ``ON UPDATE`` values for ``ForeignKey`` columns are + now extracted from the database (courtesy @wmshort). Added ``BigSerial`` column type (courtesy @aliereno). @@ -1183,13 +1183,13 @@ Thanks to @wmshort for reporting this issue. 0.35.0 ------ - * Improved ``PrimaryKey`` deprecation warning (courtesy @tonybaloney). - * Added ``piccolo schema generate`` which creates a Piccolo schema from an - existing database. - * Added ``piccolo tester run`` which is a wrapper around pytest, and - temporarily sets ``PICCOLO_CONF``, so a test database is used. - * Added the ``get`` convenience method (courtesy @aminalaee). It returns the - first matching record, or ``None`` if there's no match. For example: +* Improved ``PrimaryKey`` deprecation warning (courtesy @tonybaloney). +* Added ``piccolo schema generate`` which creates a Piccolo schema from an + existing database. +* Added ``piccolo tester run`` which is a wrapper around pytest, and + temporarily sets ``PICCOLO_CONF``, so a test database is used. +* Added the ``get`` convenience method (courtesy @aminalaee). It returns the + first matching record, or ``None`` if there's no match. For example: .. code-block:: python @@ -1215,11 +1215,11 @@ usage: 0.33.1 ------ - * Bug fix, where ``compare_dicts`` was failing in migrations if any ``Column`` - had an unhashable type as an argument. For example: ``Array(default=[])``. - Thanks to @hipertracker for reporting this problem. - * Increased the minimum version of orjson, so binaries are available for Macs - running on Apple silicon (courtesy @hipertracker). +* Bug fix, where ``compare_dicts`` was failing in migrations if any ``Column`` + had an unhashable type as an argument. For example: ``Array(default=[])``. + Thanks to @hipertracker for reporting this problem. +* Increased the minimum version of orjson, so binaries are available for Macs + running on Apple silicon (courtesy @hipertracker). ------------------------------------------------------------------------------- @@ -1306,10 +1306,10 @@ if linters complain about using ``SomeTable.some_column == True``. 0.25.0 ------ - * Changed the migration IDs, so the timestamp now includes microseconds. This - is to make clashing migration IDs much less likely. - * Added a lot of end-to-end tests for migrations, which revealed some bugs - in ``Column`` defaults. +* Changed the migration IDs, so the timestamp now includes microseconds. This + is to make clashing migration IDs much less likely. +* Added a lot of end-to-end tests for migrations, which revealed some bugs + in ``Column`` defaults. ------------------------------------------------------------------------------- @@ -1366,16 +1366,16 @@ for more details). 0.22.0 ------ - * An error is now raised if a user tries to create a Piccolo app using - ``piccolo app new`` with the same name as a builtin Python module, as it - will cause strange bugs. - * Fixing a strange bug where using an expression such as - ``Concert.band_1.manager.id`` in a query would cause an error. It only - happened if multiple joins were involved, and the last column in the chain - was ``id``. - * ``where`` clauses can now accept ``Table`` instances. For example: - ``await Band.select().where(Band.manager == some_manager).run()``, instead - of having to explicity reference the ``id``. +* An error is now raised if a user tries to create a Piccolo app using + ``piccolo app new`` with the same name as a builtin Python module, as it + will cause strange bugs. +* Fixing a strange bug where using an expression such as + ``Concert.band_1.manager.id`` in a query would cause an error. It only + happened if multiple joins were involved, and the last column in the chain + was ``id``. +* ``where`` clauses can now accept ``Table`` instances. For example: + ``await Band.select().where(Band.manager == some_manager).run()``, instead + of having to explicity reference the ``id``. ------------------------------------------------------------------------------- @@ -1394,8 +1394,8 @@ Fix missing imports in FastAPI and Starlette app templates. 0.21.0 ------ - * Added a ``freeze`` method to ``Query``. - * Added BlackSheep as an option to ``piccolo asgi new``. +* Added a ``freeze`` method to ``Query``. +* Added BlackSheep as an option to ``piccolo asgi new``. ------------------------------------------------------------------------------- @@ -1407,8 +1407,8 @@ Added ``choices`` option to ``Column``. 0.19.1 ------ - * Added ``piccolo user change_permissions`` command. - * Added aliases for CLI commands. +* Added ``piccolo user change_permissions`` command. +* Added aliases for CLI commands. ------------------------------------------------------------------------------- @@ -1432,15 +1432,15 @@ following DDL statements: 0.18.4 ------ - * Fixed a bug when multiple tables inherit from the same mixin (thanks to - @brnosouza). - * Added a ``log_queries`` option to ``PostgresEngine``, which is useful during - debugging. - * Added the `inflection` library for converting ``Table`` class names to - database table names. Previously, a class called ``TableA`` would wrongly - have a table called ``table`` instead of ``table_a``. - * Fixed a bug with ``SerialisedBuiltin.__hash__`` not returning a number, - which could break migrations (thanks to @sinisaos). +* Fixed a bug when multiple tables inherit from the same mixin (thanks to + @brnosouza). +* Added a ``log_queries`` option to ``PostgresEngine``, which is useful during + debugging. +* Added the `inflection` library for converting ``Table`` class names to + database table names. Previously, a class called ``TableA`` would wrongly + have a table called ``table`` instead of ``table_a``. +* Fixed a bug with ``SerialisedBuiltin.__hash__`` not returning a number, + which could break migrations (thanks to @sinisaos). ------------------------------------------------------------------------------- @@ -1464,11 +1464,11 @@ Add the ``Array`` column type as a top level import in ``piccolo.columns``. 0.18.0 ------ - * Refactored ``forwards`` and ``backwards`` commands for migrations, to make - them easier to run programatically. - * Added a simple ``Array`` column type. - * ``table_finder`` now works if just a string is passed in, instead of having - to pass in an array of strings. +* Refactored ``forwards`` and ``backwards`` commands for migrations, to make + them easier to run programatically. +* Added a simple ``Array`` column type. +* ``table_finder`` now works if just a string is passed in, instead of having + to pass in an array of strings. ------------------------------------------------------------------------------- @@ -1496,10 +1496,10 @@ Piccolo Admin to show tooltips. 0.17.2 ------ - * Exposing ``index_type`` in the ``Column`` constructor. - * Fixing a typo with ``start_connection_pool` and ``close_connection_pool`` - - thanks to paolodina for finding this. - * Fixing a typo in the ``PostgresEngine`` docs - courtesy of paolodina. +* Exposing ``index_type`` in the ``Column`` constructor. +* Fixing a typo with ``start_connection_pool` and ``close_connection_pool`` - + thanks to paolodina for finding this. +* Fixing a typo in the ``PostgresEngine`` docs - courtesy of paolodina. ------------------------------------------------------------------------------- @@ -1512,11 +1512,11 @@ Fixing a bug with ``SchemaSnapshot`` if column types were changed in migrations 0.17.0 ------ - * Migrations now directly import ``Column`` classes - this allows users to - create custom ``Column`` subclasses. Migrations previously only worked with - the builtin column types. - * Migrations now detect if the column type has changed, and will try and - convert it automatically. +* Migrations now directly import ``Column`` classes - this allows users to + create custom ``Column`` subclasses. Migrations previously only worked with + the builtin column types. +* Migrations now detect if the column type has changed, and will try and + convert it automatically. ------------------------------------------------------------------------------- @@ -1529,16 +1529,16 @@ can now be configured. 0.16.4 ------ - * Fixed a bug with ``MyTable.column != None`` - * Added ``is_null`` and ``is_not_null`` methods, to avoid linting issues when - comparing with None. +* Fixed a bug with ``MyTable.column != None`` +* Added ``is_null`` and ``is_not_null`` methods, to avoid linting issues when + comparing with None. ------------------------------------------------------------------------------- 0.16.3 ------ - * Added ``WhereRaw``, so raw SQL can be used in where clauses. - * ``piccolo shell run`` now uses syntax highlighting - courtesy of Fingel. +* Added ``WhereRaw``, so raw SQL can be used in where clauses. +* ``piccolo shell run`` now uses syntax highlighting - courtesy of Fingel. ------------------------------------------------------------------------------- @@ -1558,10 +1558,10 @@ aware. 0.16.0 ------ - * Fixed a bug with creating a ``ForeignKey`` column with ``references="self"`` - in auto migrations. - * Changed migration file naming, so there are no characters in there which - are unsupported on Windows. +* Fixed a bug with creating a ``ForeignKey`` column with ``references="self"`` + in auto migrations. +* Changed migration file naming, so there are no characters in there which + are unsupported on Windows. ------------------------------------------------------------------------------- @@ -1586,11 +1586,11 @@ Fixing a bug with migrations which drop column defaults. 0.14.12 ------- - * Fixing a bug where re-running ``Table.create(if_not_exists=True)`` would - fail if it contained columns with indexes. - * Raising a ``ValueError`` if a relative path is provided to ``ForeignKey`` - ``references``. For example, ``.tables.Manager``. The paths must be absolute - for now. +* Fixing a bug where re-running ``Table.create(if_not_exists=True)`` would + fail if it contained columns with indexes. +* Raising a ``ValueError`` if a relative path is provided to ``ForeignKey`` + ``references``. For example, ``.tables.Manager``. The paths must be absolute + for now. ------------------------------------------------------------------------------- @@ -1603,14 +1603,14 @@ metaclass not being explicit enough when checking falsy values. 0.14.10 ------- - * The ``ForeignKey`` ``references`` argument can now be specified using a - string, or a ``LazyTableReference`` instance, rather than just a ``Table`` - subclass. This allows a ``Table`` to be specified which is in a Piccolo app, - or Python module. The ``Table`` is only loaded after imports have completed, - which prevents circular import issues. - * Faster column copying, which is important when specifying joins, e.g. - ``await Band.select(Band.manager.name).run()``. - * Fixed a bug with migrations and foreign key contraints. +* The ``ForeignKey`` ``references`` argument can now be specified using a + string, or a ``LazyTableReference`` instance, rather than just a ``Table`` + subclass. This allows a ``Table`` to be specified which is in a Piccolo app, + or Python module. The ``Table`` is only loaded after imports have completed, + which prevents circular import issues. +* Faster column copying, which is important when specifying joins, e.g. + ``await Band.select(Band.manager.name).run()``. +* Fixed a bug with migrations and foreign key contraints. ------------------------------------------------------------------------------- @@ -1623,76 +1623,76 @@ migrations are left to run / reverse. Otherwise build scripts may fail. 0.14.8 ------ - * Improved the method signature of the ``output`` query clause (explicitly - added args, instead of using ``**kwargs``). - * Fixed a bug where ``output(as_list=True)`` would fail if no rows were found. - * Made ``piccolo migrations forwards`` command output more legible. - * Improved renamed table detection in migrations. - * Added the ``piccolo migrations clean`` command for removing orphaned rows - from the migrations table. - * Fixed a bug where ``get_migration_managers`` wasn't inclusive. - * Raising a ``ValueError`` if ``is_in`` or ``not_in`` query clauses are passed - an empty list. - * Changed the migration commands to be top level async. - * Combined ``print`` and ``sys.exit`` statements. +* Improved the method signature of the ``output`` query clause (explicitly + added args, instead of using ``**kwargs``). +* Fixed a bug where ``output(as_list=True)`` would fail if no rows were found. +* Made ``piccolo migrations forwards`` command output more legible. +* Improved renamed table detection in migrations. +* Added the ``piccolo migrations clean`` command for removing orphaned rows + from the migrations table. +* Fixed a bug where ``get_migration_managers`` wasn't inclusive. +* Raising a ``ValueError`` if ``is_in`` or ``not_in`` query clauses are passed + an empty list. +* Changed the migration commands to be top level async. +* Combined ``print`` and ``sys.exit`` statements. ------------------------------------------------------------------------------- 0.14.7 ------ - * Added missing type annotation for ``run_sync``. - * Updating type annotations for column default values - allowing callables. - * Replaced instances of ``asyncio.run`` with ``run_sync``. - * Tidied up aiosqlite imports. +* Added missing type annotation for ``run_sync``. +* Updating type annotations for column default values - allowing callables. +* Replaced instances of ``asyncio.run`` with ``run_sync``. +* Tidied up aiosqlite imports. ------------------------------------------------------------------------------- 0.14.6 ------ - * Added JSON and JSONB column types, and the arrow function for JSONB. - * Fixed a bug with the distinct clause. - * Added ``as_alias``, so select queries can override column names in the - response (i.e. SELECT foo AS bar from baz). - * Refactored JSON encoding into a separate utils file. +* Added JSON and JSONB column types, and the arrow function for JSONB. +* Fixed a bug with the distinct clause. +* Added ``as_alias``, so select queries can override column names in the + response (i.e. SELECT foo AS bar from baz). +* Refactored JSON encoding into a separate utils file. ------------------------------------------------------------------------------- 0.14.5 ------ - * Removed old iPython version recommendation in the ``piccolo shell run`` and - ``piccolo playground run``, and enabled top level await. - * Fixing outstanding mypy warnings. - * Added optional requirements for the playground to setup.py +* Removed old iPython version recommendation in the ``piccolo shell run`` and + ``piccolo playground run``, and enabled top level await. +* Fixing outstanding mypy warnings. +* Added optional requirements for the playground to setup.py ------------------------------------------------------------------------------- 0.14.4 ------ - * Added ``piccolo sql_shell run`` command, which launches the psql or sqlite3 - shell, using the connection parameters defined in ``piccolo_conf.py``. - This is convenient when you want to run raw SQL on your database. - * ``run_sync`` now handles more edge cases, for example if there's already - an event loop in the current thread. - * Removed asgiref dependency. +* Added ``piccolo sql_shell run`` command, which launches the psql or sqlite3 + shell, using the connection parameters defined in ``piccolo_conf.py``. + This is convenient when you want to run raw SQL on your database. +* ``run_sync`` now handles more edge cases, for example if there's already + an event loop in the current thread. +* Removed asgiref dependency. ------------------------------------------------------------------------------- 0.14.3 ------ - * Queries can be directly awaited - ``await MyTable.select()``, as an - alternative to using the run method ``await MyTable.select().run()``. - * The ``piccolo asgi new`` command now accepts a ``name`` argument, which is - used to populate the default database name within the template. +* Queries can be directly awaited - ``await MyTable.select()``, as an + alternative to using the run method ``await MyTable.select().run()``. +* The ``piccolo asgi new`` command now accepts a ``name`` argument, which is + used to populate the default database name within the template. ------------------------------------------------------------------------------- 0.14.2 ------ - * Centralised code for importing Piccolo apps and tables - laying the - foundation for fixtures. - * Made orjson an optional dependency, installable using - ``pip install piccolo[orjson]``. - * Improved version number parsing in Postgres. +* Centralised code for importing Piccolo apps and tables - laying the + foundation for fixtures. +* Made orjson an optional dependency, installable using + ``pip install piccolo[orjson]``. +* Improved version number parsing in Postgres. ------------------------------------------------------------------------------- @@ -1710,15 +1710,15 @@ Added ``Interval`` column type. 0.13.5 ------ - * Added ``allowed_hosts`` to ``create_admin`` in ASGI template. - * Fixing bug with default ``root`` argument in some piccolo commands. +* Added ``allowed_hosts`` to ``create_admin`` in ASGI template. +* Fixing bug with default ``root`` argument in some piccolo commands. ------------------------------------------------------------------------------- 0.13.4 ------ - * Fixed bug with ``SchemaSnapshot`` when dropping columns. - * Added custom ``__repr__`` method to ``Table``. +* Fixed bug with ``SchemaSnapshot`` when dropping columns. +* Added custom ``__repr__`` method to ``Table``. ------------------------------------------------------------------------------- @@ -1730,9 +1730,9 @@ Added ``piccolo shell run`` command for running adhoc queries using Piccolo. 0.13.2 ------ - * Fixing bug with auto migrations when dropping columns. - * Added a ``root`` argument to ``piccolo asgi new``, ``piccolo app new`` and - ``piccolo project new`` commands, to override where the files are placed. +* Fixing bug with auto migrations when dropping columns. +* Added a ``root`` argument to ``piccolo asgi new``, ``piccolo app new`` and + ``piccolo project new`` commands, to override where the files are placed. ------------------------------------------------------------------------------- @@ -1752,19 +1752,19 @@ generating forms and serialisers. 0.12.6 ------ - * Fixing a typo in ``TimestampCustom`` arguments. - * Fixing bug in ``TimestampCustom`` SQL representation. - * Added more extensive deserialisation for migrations. +* Fixing a typo in ``TimestampCustom`` arguments. +* Fixing bug in ``TimestampCustom`` SQL representation. +* Added more extensive deserialisation for migrations. ------------------------------------------------------------------------------- 0.12.5 ------ - * Improved ``PostgresEngine`` docstring. - * Resolving rename migrations before adding columns. - * Fixed bug serialising ``TimestampCustom``. - * Fixed bug with altering column defaults to be non-static values. - * Removed ``response_handler`` from ``Alter`` query. +* Improved ``PostgresEngine`` docstring. +* Resolving rename migrations before adding columns. +* Fixed bug serialising ``TimestampCustom``. +* Fixed bug with altering column defaults to be non-static values. +* Removed ``response_handler`` from ``Alter`` query. ------------------------------------------------------------------------------- @@ -1790,19 +1790,19 @@ Fixing bug when sorting ``extra_definitions`` in auto migrations. 0.12.1 ------ - * Fixed typos. - * Bumped requirements. +* Fixed typos. +* Bumped requirements. ------------------------------------------------------------------------------- 0.12.0 ------ - * Added ``Date`` and ``Time`` columns. - * Improved support for column default values. - * Auto migrations can now serialise more Python types. - * Added ``Table.indexes`` method for listing table indexes. - * Auto migrations can handle adding / removing indexes. - * Improved ASGI template for FastAPI. +* Added ``Date`` and ``Time`` columns. +* Improved support for column default values. +* Auto migrations can now serialise more Python types. +* Added ``Table.indexes`` method for listing table indexes. +* Auto migrations can handle adding / removing indexes. +* Improved ASGI template for FastAPI. ------------------------------------------------------------------------------- @@ -1814,10 +1814,10 @@ ASGI template fix. 0.11.7 ------ - * Improved ``UUID`` columns in SQLite - prepending 'uuid:' to the stored value - to make the type more explicit for the engine. - * Removed SQLite as an option for ``piccolo asgi new`` until auto migrations - are supported. +* Improved ``UUID`` columns in SQLite - prepending 'uuid:' to the stored value + to make the type more explicit for the engine. +* Removed SQLite as an option for ``piccolo asgi new`` until auto migrations + are supported. ------------------------------------------------------------------------------- @@ -1836,9 +1836,9 @@ excluding non-Python files well enough. 0.11.4 ------ - * Stopped ``piccolo migrations new`` from creating a config.py file - was - legacy. - * Added a README file to the `piccolo_migrations` folder in the ASGI template. +* Stopped ``piccolo migrations new`` from creating a config.py file - was + legacy. +* Added a README file to the `piccolo_migrations` folder in the ASGI template. ------------------------------------------------------------------------------- @@ -1850,21 +1850,21 @@ Fixed `__pycache__` bug when using ``piccolo asgi new``. 0.11.2 ------ - * Showing a warning if trying auto migrations with SQLite. - * Added a command for creating a new ASGI app - ``piccolo asgi new``. - * Added a meta app for printing out the Piccolo version - - ``piccolo meta version``. - * Added example queries to the playground. +* Showing a warning if trying auto migrations with SQLite. +* Added a command for creating a new ASGI app - ``piccolo asgi new``. +* Added a meta app for printing out the Piccolo version - + ``piccolo meta version``. +* Added example queries to the playground. ------------------------------------------------------------------------------- 0.11.1 ------ - * Added ``table_finder``, for use in ``AppConfig``. - * Added support for concatenating strings using an update query. - * Added more tables to the playground, with more column types. - * Improved consistency between SQLite and Postgres with ``UUID`` columns, - ``Integer`` columns, and ``exists`` queries. +* Added ``table_finder``, for use in ``AppConfig``. +* Added support for concatenating strings using an update query. +* Added more tables to the playground, with more column types. +* Improved consistency between SQLite and Postgres with ``UUID`` columns, + ``Integer`` columns, and ``exists`` queries. ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/authentication/baseuser.rst b/docs/src/piccolo/authentication/baseuser.rst index 31fa90e68..d498d2984 100644 --- a/docs/src/piccolo/authentication/baseuser.rst +++ b/docs/src/piccolo/authentication/baseuser.rst @@ -68,8 +68,8 @@ For example: The Piccolo Admin (see :ref:`Ecosystem`) uses these attributes to control who can login and what they can do. - * **active** and **admin** - must be true for a user to be able to login. - * **superuser** - must be true for a user to be able to change other user's +* **active** and **admin** - must be true for a user to be able to login. +* **superuser** - must be true for a user to be able to change other user's passwords. ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/projects_and_apps/included_apps.rst b/docs/src/piccolo/projects_and_apps/included_apps.rst index e965a9ea6..1dfd48bec 100644 --- a/docs/src/piccolo/projects_and_apps/included_apps.rst +++ b/docs/src/piccolo/projects_and_apps/included_apps.rst @@ -52,8 +52,8 @@ Databases such as Postgres have inbuilt ways of dumping and restoring data (via ``pg_dump`` and ``pg_restore``). Some reasons to use the fixtures app instead: - * When you want the data to be loadable in a range of database versions. - * Fixtures are stored in JSON, which are a bit friendlier for source control. +* When you want the data to be loadable in a range of database versions. +* Fixtures are stored in JSON, which are a bit friendlier for source control. To dump the data into a new fixture file: diff --git a/piccolo/conf/apps.py b/piccolo/conf/apps.py index c3921f640..a202328bf 100644 --- a/piccolo/conf/apps.py +++ b/piccolo/conf/apps.py @@ -336,8 +336,8 @@ def get_piccolo_conf_module( Searches the path for a 'piccolo_conf.py' module to import. The location searched can be overriden by: - * Explicitly passing a module name into this method. - * Setting the PICCOLO_CONF environment variable. + * Explicitly passing a module name into this method. + * Setting the PICCOLO_CONF environment variable. An example override is 'my_folder.piccolo_conf'. From 503bf245961f5d9e7b9612e2f5170246447cec21 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 18 Feb 2022 23:17:25 +0000 Subject: [PATCH 264/727] 438 Piccolo docs theme (#439) * minor doc tweaks * use Piccolo theme, and sphinx-autobuild --- docs/serve_docs.py | 14 ------------ docs/src/conf.py | 6 ++--- docs/src/piccolo/contributing/index.rst | 2 +- .../getting_started/installing_piccolo.rst | 2 +- docs/src/piccolo/query_clauses/index.rst | 2 +- piccolo/columns/base.py | 3 ++- piccolo/columns/column_types.py | 22 +++++++++---------- requirements/doc-requirements.txt | 4 ++-- scripts/run-docs.sh | 2 ++ 9 files changed, 23 insertions(+), 34 deletions(-) delete mode 100755 docs/serve_docs.py create mode 100755 scripts/run-docs.sh diff --git a/docs/serve_docs.py b/docs/serve_docs.py deleted file mode 100755 index 50ed6a05f..000000000 --- a/docs/serve_docs.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python -from livereload import Server, shell - - -server = Server() -server.watch( - 'src/', - shell('make html') -) -server.watch( - '../piccolo', - shell('make html') -) -server.serve(root='build/html') diff --git a/docs/src/conf.py b/docs/src/conf.py index 1b2352aa6..1d2cdbe56 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -87,9 +87,9 @@ # -- Options for HTML output ------------------------------------------------- -html_theme = "sphinx_rtd_theme" -html_logo = "logo.png" -html_theme_options = {"logo_only": True} +html_theme = "piccolo_theme" +html_show_sphinx = False +globaltoc_maxdepth = 3 # -- Options for HTMLHelp output --------------------------------------------- diff --git a/docs/src/piccolo/contributing/index.rst b/docs/src/piccolo/contributing/index.rst index 1f4565de0..853ceabc2 100644 --- a/docs/src/piccolo/contributing/index.rst +++ b/docs/src/piccolo/contributing/index.rst @@ -32,7 +32,7 @@ The docs are written using Sphinx. To get them running locally: * Install the requirements: ``pip install -r requirements/doc-requirements.txt`` * ``cd docs`` * Do an initial build of the docs: ``make html`` -* Serve the docs: ``python serve_docs.py`` +* Serve the docs: ``./scripts/run-docs.sh`` * The docs will auto rebuild as you make changes. ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/getting_started/installing_piccolo.rst b/docs/src/piccolo/getting_started/installing_piccolo.rst index c4460bc7a..cdc695771 100644 --- a/docs/src/piccolo/getting_started/installing_piccolo.rst +++ b/docs/src/piccolo/getting_started/installing_piccolo.rst @@ -11,7 +11,7 @@ You need `Python 3.7 `_ or above installed on Pip --- -Now install piccolo, ideally inside a `virtualenv `_: +Now install Piccolo, ideally inside a `virtualenv `_: .. code-block:: python diff --git a/docs/src/piccolo/query_clauses/index.rst b/docs/src/piccolo/query_clauses/index.rst index 14f6ea96f..e4129c83a 100644 --- a/docs/src/piccolo/query_clauses/index.rst +++ b/docs/src/piccolo/query_clauses/index.rst @@ -7,7 +7,7 @@ Query clauses are used to modify a query by making it more specific, or by modifying the return values. .. toctree:: - :maxdepth: 0 + :maxdepth: 1 ./first ./distinct diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index c7d31e5c5..1b35d1cd4 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -366,7 +366,8 @@ class Column(Selectable): the speed of selects, but can slow down inserts. :param index_method: - If index is set to True, this specifies what type of index is created. + If index is set to ``True``, this specifies what type of index is + created. :param required: This isn't used by the database - it's to indicate to other tools that diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index fe3f8a6ca..159d8b5e2 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -1506,11 +1506,11 @@ class Band(Table): Options: - * ``OnDelete.cascade`` (default) - * ``OnDelete.restrict`` - * ``OnDelete.no_action`` - * ``OnDelete.set_null`` - * ``OnDelete.set_default`` + * ``OnDelete.cascade`` (default) + * ``OnDelete.restrict`` + * ``OnDelete.no_action`` + * ``OnDelete.set_null`` + * ``OnDelete.set_default`` To learn more about the different options, see the `Postgres docs `_. @@ -1532,11 +1532,11 @@ class Band(Table): Options: - * ``OnUpdate.cascade`` (default) - * ``OnUpdate.restrict`` - * ``OnUpdate.no_action`` - * ``OnUpdate.set_null`` - * ``OnUpdate.set_default`` + * ``OnUpdate.cascade`` (default) + * ``OnUpdate.restrict`` + * ``OnUpdate.no_action`` + * ``OnUpdate.set_null`` + * ``OnUpdate.set_default`` To learn more about the different options, see the `Postgres docs `_. @@ -1815,7 +1815,7 @@ def set_proxy_columns(self): setattr(self, _column._meta.name, _column) _fk_meta.proxy_columns.append(_column) - def __getattribute__(self, name: str): + def __getattribute__(self, name: str) -> t.Union[Column, t.Any]: """ Returns attributes unmodified unless they're Column instances, in which case a copy is returned with an updated call_chain (which records the diff --git a/requirements/doc-requirements.txt b/requirements/doc-requirements.txt index cfc1d6a37..4d195bdea 100644 --- a/requirements/doc-requirements.txt +++ b/requirements/doc-requirements.txt @@ -1,3 +1,3 @@ Sphinx==4.4.0 -sphinx-rtd-theme==1.0.0 -livereload==2.6.3 +piccolo-theme>=0.2.3 +sphinx-autobuild==2021.3.14 diff --git a/scripts/run-docs.sh b/scripts/run-docs.sh new file mode 100755 index 000000000..bac9a6ebe --- /dev/null +++ b/scripts/run-docs.sh @@ -0,0 +1,2 @@ +#!/bin/bash +sphinx-autobuild -a docs/src docs/build/html --watch piccolo From 6c9be3ff1bc93e87acd5af671502a11059673620 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 24 Feb 2022 13:08:09 +0000 Subject: [PATCH 265/727] tidy up conf.py There was lots of noise in there. --- docs/src/conf.py | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/docs/src/conf.py b/docs/src/conf.py index 1d2cdbe56..e0e8f0940 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -15,7 +15,6 @@ import datetime import os import sys -import typing as t sys.path.insert(0, os.path.abspath("../..")) @@ -34,21 +33,10 @@ # -- General configuration --------------------------------------------------- -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. extensions = [ - "sphinx.ext.todo", "sphinx.ext.coverage", ] -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # @@ -58,21 +46,6 @@ # The master toctree document. master_doc = "index" -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns: t.List[str] = [] - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = None - # -- Intersphinx ------------------------------------------------------------- intersphinx_mapping = {"python": ("https://docs.python.org/3", None)} @@ -96,14 +69,8 @@ # Output file base name for HTML help builder. htmlhelp_basename = "Piccolodoc" - # -- Options for manual page output ------------------------------------------ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [(master_doc, "piccolo", "Piccolo Documentation", [author], 1)] - -# -- Options for todo extension ---------------------------------------------- - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = True From 5652127aa2e56209242cd08e90e6c4ec66f44cbf Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 24 Feb 2022 20:10:06 +0000 Subject: [PATCH 266/727] minor doc changes --- docs/src/index.rst | 8 ++++---- docs/src/piccolo/features/tab_completion.rst | 3 ++- docs/src/piccolo/getting_started/what_is_piccolo.rst | 12 ++++++------ docs/src/piccolo/playground/advanced.rst | 2 +- docs/src/piccolo/query_types/django_comparison.rst | 11 ++++++++++- 5 files changed, 23 insertions(+), 13 deletions(-) diff --git a/docs/src/index.rst b/docs/src/index.rst index 6af89899b..dba01f7b3 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -1,5 +1,5 @@ -Welcome to Piccolo's documentation! -=================================== +Piccolo +======= Piccolo is a modern, async query builder and ORM for Python, with lots of batteries included. @@ -53,8 +53,8 @@ Give me an ASGI web app! piccolo asgi new -FastAPI, Starlette, and BlackSheep are currently supported, with more coming -soon. +FastAPI, Starlette, BlackSheep, and Xpresso are currently supported, with more +coming soon. ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/features/tab_completion.rst b/docs/src/piccolo/features/tab_completion.rst index 674f05f1a..c7cff94c6 100644 --- a/docs/src/piccolo/features/tab_completion.rst +++ b/docs/src/piccolo/features/tab_completion.rst @@ -6,4 +6,5 @@ Tab Completion Piccolo does everything possible to support tab completion. It has been tested with iPython and VSCode. -To find out more about how it was done, read `this article `_. +To find out more about how it was done, read `this article about type annotations `_, +and `this article about descriptors `_. diff --git a/docs/src/piccolo/getting_started/what_is_piccolo.rst b/docs/src/piccolo/getting_started/what_is_piccolo.rst index a7ab67cda..6a20875d3 100644 --- a/docs/src/piccolo/getting_started/what_is_piccolo.rst +++ b/docs/src/piccolo/getting_started/what_is_piccolo.rst @@ -5,10 +5,10 @@ Piccolo is a fast, easy to learn ORM and query builder. Some of it's stand out features are: -* Support for sync and async - see :ref:`SyncAndAsync`. -* A builtin playground, which makes learning a breeze - see :ref:`Playground`. -* Works great with `iPython `_ and - `VSCode `_ - see :ref:`tab_completion`. -* Batteries included - a :ref:`User model and authentication `, :ref:`migrations `, an :ref:`admin `, - and more. +* Support for :ref:`sync and async `. +* A builtin :ref:`playground `, which makes learning a breeze. +* Great :ref:`tab completion ` - it works great with + `iPython `_ and `VSCode `_. +* Batteries included - a :ref:`User model and authentication `, + :ref:`migrations `, an :ref:`admin `, and more. * Templates for creating your own :ref:`ASGI web app `. diff --git a/docs/src/piccolo/playground/advanced.rst b/docs/src/piccolo/playground/advanced.rst index 8c9320c4e..24fb585ab 100644 --- a/docs/src/piccolo/playground/advanced.rst +++ b/docs/src/piccolo/playground/advanced.rst @@ -13,7 +13,7 @@ first. Install Postgres ~~~~~~~~~~~~~~~~ -See :ref:`setting_up_postgres`. +See :ref:`the docs on settings up Postgres `. Create database ~~~~~~~~~~~~~~~ diff --git a/docs/src/piccolo/query_types/django_comparison.rst b/docs/src/piccolo/query_types/django_comparison.rst index bf707c14a..76734a3a5 100644 --- a/docs/src/piccolo/query_types/django_comparison.rst +++ b/docs/src/piccolo/query_types/django_comparison.rst @@ -180,8 +180,17 @@ Piccolo has something similar: ------------------------------------------------------------------------------- -Database Settings +Database settings ----------------- In Django you configure your database in ``settings.py``. With Piccolo, you define an ``Engine`` in ``piccolo_conf.py``. See :ref:`Engines`. + +------------------------------------------------------------------------------- + +Creating a new project +---------------------- + +With Django you use ``django-admin startproject mysite``. + +In Piccolo you use ``piccolo asgi new`` (see :ref:`ASGICommand`). From 7b5948d2de1fe178edd40d1d4cc0af5f989fba25 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 26 Feb 2022 16:35:17 +0000 Subject: [PATCH 267/727] bump docs theme for dark mode --- requirements/doc-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/doc-requirements.txt b/requirements/doc-requirements.txt index 4d195bdea..b646370b0 100644 --- a/requirements/doc-requirements.txt +++ b/requirements/doc-requirements.txt @@ -1,3 +1,3 @@ Sphinx==4.4.0 -piccolo-theme>=0.2.3 +piccolo-theme>=0.3.0 sphinx-autobuild==2021.3.14 From e37d8621fa52b3a2ca8adfadbb212c60ccaa9bf2 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 26 Feb 2022 18:27:59 +0000 Subject: [PATCH 268/727] more migration examples (#443) --- docs/src/piccolo/migrations/create.rst | 134 +++++++++++++++++++++---- 1 file changed, 115 insertions(+), 19 deletions(-) diff --git a/docs/src/piccolo/migrations/create.rst b/docs/src/piccolo/migrations/create.rst index b836bf86e..5e7494f97 100644 --- a/docs/src/piccolo/migrations/create.rst +++ b/docs/src/piccolo/migrations/create.rst @@ -5,7 +5,16 @@ Migrations are Python files which are used to modify the database schema in a controlled way. Each migration belongs to a :ref:`Piccolo app `. You can either manually populate migrations, or allow Piccolo to do it for you -automatically. To create an empty migration: +automatically. + +We recommend using automatic migrations where possible, as it saves you time. + +------------------------------------------------------------------------------- + +Manual migrations +----------------- + +First, let's create an empty migration: .. code-block:: bash @@ -28,15 +37,20 @@ The contents of an empty migration file looks like this: from piccolo.apps.migrations.auto.migration_manager import MigrationManager - ID = '2021-08-06T16:22:51:415781' - VERSION = "0.29.0" # The version of Piccolo used to create it + ID = "2022-02-26T17:38:44:758593" + VERSION = "0.69.2" # The version of Piccolo used to create it DESCRIPTION = "Optional description" async def forwards(): - manager = MigrationManager(migration_id=ID, app_name="my_app", description=DESCRIPTION) + manager = MigrationManager( + migration_id=ID, + app_name="my_app", + description=DESCRIPTION + ) def run(): + # Replace this with something useful: print(f"running {ID}") manager.add_raw(run) @@ -48,37 +62,119 @@ shouldn't be changed. Replace the ``run`` function with whatever you want the migration to do - typically running some SQL. It can be a function or a coroutine. -------------------------------------------------------------------------------- +Running raw SQL +~~~~~~~~~~~~~~~ -The golden rule ---------------- +If you want to run raw SQL within your migration, you can do so as follows: + +.. code-block:: python + + from piccolo.apps.migrations.auto.migration_manager import MigrationManager + from piccolo.table import Table + + + ID = "2022-02-26T17:38:44:758593" + VERSION = "0.69.2" + DESCRIPTION = "Updating each band's popularity" + + + # This is just a dummy table we use to execute raw SQL with: + class RawTable(Table): + pass + + + async def forwards(): + manager = MigrationManager( + migration_id=ID, + app_name="my_app", + description=DESCRIPTION + ) + + async def run(): + await RawTable.raw('UPDATE band SET popularity={}', 1000) + + manager.add_raw(run) + return manager + +.. hint:: You can learn more about :ref:`raw queries here `. -Never import your tables directly into a migration, and run methods on them. +Using your ``Table`` classes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This is a **bad example**: +In the above example, we executed raw SQL, but what if we wanted to use the +``Table`` classes from our project instead? + +We have to be quite careful with this. Here's an example: .. code-block:: python - from ..tables import Band + from piccolo.apps.migrations.auto.migration_manager import MigrationManager - ID = '2021-08-06T16:22:51:415781' - VERSION = "0.29.0" # The version of Piccolo used to create it - DESCRIPTION = "Optional description" + # We're importing a table from our project: + from music.tables import Band + + + ID = "2022-02-26T17:38:44:758593" + VERSION = "0.69.2" + DESCRIPTION = "Updating each band's popularity" async def forwards(): - manager = MigrationManager(migration_id=ID) + manager = MigrationManager( + migration_id=ID, + app_name="my_app", + description=DESCRIPTION + ) async def run(): - await Band.create_table().run() + await Band.update({Band.popularity: 1000}) manager.add_raw(run) return manager -The reason you don't want to do this, is your tables will change over time. If -someone runs your migrations in the future, they will get different results. -Make your migrations completely independent of other code, so they're -self contained and repeatable. +We want our migrations to be repeatable - so if someone runs them a year from +now, they will get the same results. + +By directly importing our tables, we have the following risks: + +* If the ``Band`` class is deleted from the codebase, it could break old + migrations. +* If we modify the ``Band`` class, perhaps by removing columns, this could also + break old migrations. + +Try and make your migration files independent of other application code, so +they're self contained and repeatable. Even though it goes against `DRY `_, +it's better to copy the relevant tables into your migration file: + +.. code-block:: python + + from piccolo.apps.migrations.auto.migration_manager import MigrationManager + from piccolo.columns.column_types import Integer + from piccolo.table import Table + + + ID = "2022-02-26T17:38:44:758593" + VERSION = "0.69.2" + DESCRIPTION = "Updating each band's popularity" + + + # We defined the table within the file, rather than importing it. + class Band(Table): + popularity = Integer() + + + async def forwards(): + manager = MigrationManager( + migration_id=ID, + app_name="my_app", + description=DESCRIPTION + ) + + async def run(): + await Band.update({Band.popularity: 1000}) + + manager.add_raw(run) + return manager ------------------------------------------------------------------------------- From 1b09607e7e0e6fdadfdb5d9c8ada8e1a7f0d8b1d Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 26 Feb 2022 18:34:15 +0000 Subject: [PATCH 269/727] fix typo in migration file name --- docs/src/piccolo/migrations/create.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/piccolo/migrations/create.rst b/docs/src/piccolo/migrations/create.rst index 5e7494f97..28c078214 100644 --- a/docs/src/piccolo/migrations/create.rst +++ b/docs/src/piccolo/migrations/create.rst @@ -26,7 +26,7 @@ migration filename is a timestamp: .. code-block:: bash piccolo_migrations/ - 2021-08-06T16-22-51-415781.py + 2022-02-26T17-38-44-758593.py .. hint:: You can rename this file if you like to make it more memorable. From 38255971654939d2041efa8bdf2b0a78739b4bdc Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 1 Mar 2022 05:32:09 -0500 Subject: [PATCH 270/727] fix example in docs - now showing async version --- docs/src/piccolo/query_types/objects.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/piccolo/query_types/objects.rst b/docs/src/piccolo/query_types/objects.rst index 7183e7d38..31aceffc0 100644 --- a/docs/src/piccolo/query_types/objects.rst +++ b/docs/src/piccolo/query_types/objects.rst @@ -164,7 +164,7 @@ database, just as you would expect: .. code-block:: python ticket.concert.band_1.name = 'Pythonistas 2' - ticket.concert.band_1.save().run_sync() + await ticket.concert.band_1.save() Instead of passing the ``ForeignKey`` columns into the ``objects`` method, you can use the ``prefetch`` clause if you prefer. From 3734e182f6f3f544ead4ebb89874b1522b8c9037 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 1 Mar 2022 06:09:00 -0500 Subject: [PATCH 271/727] fix typo in `is_null` docstrings --- piccolo/columns/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 1b35d1cd4..43fb4f1ee 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -678,14 +678,14 @@ def __hash__(self): def is_null(self) -> Where: """ - Can be used instead of ``MyTable.column != None``, because some linters + Can be used instead of ``MyTable.column == None``, because some linters don't like a comparison to ``None``. """ return Where(column=self, operator=IsNull) def is_not_null(self) -> Where: """ - Can be used instead of ``MyTable.column == None``, because some linters + Can be used instead of ``MyTable.column != None``, because some linters don't like a comparison to ``None``. """ return Where(column=self, operator=IsNotNull) From 7a6746d8f09ea0f7d9bc799d1e9c8955ab595f0b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 1 Mar 2022 12:11:47 +0000 Subject: [PATCH 272/727] Warn if a `bool` is passed into a `where` clause (#445) * fix timezone error in test * raise error if bool is passed into a where clause --- piccolo/query/mixins.py | 8 ++++++++ tests/apps/fixtures/commands/test_dump_load.py | 6 +++++- tests/table/test_select.py | 9 +++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index d7d235d88..499bf6cf2 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -116,6 +116,14 @@ def _extract_columns(self, combinable: Combinable): def where(self, *where: Combinable): for arg in where: + if isinstance(arg, bool): + raise ValueError( + "A boolean value has been passed in to a where clause. " + "This is probably a mistake. For example " + "`.where(MyTable.some_column is None)` instead of " + "`.where(MyTable.some_column.is_null())`." + ) + self._where = And(self._where, arg) if self._where else arg diff --git a/tests/apps/fixtures/commands/test_dump_load.py b/tests/apps/fixtures/commands/test_dump_load.py index 98996ca4b..0c27bdcda 100644 --- a/tests/apps/fixtures/commands/test_dump_load.py +++ b/tests/apps/fixtures/commands/test_dump_load.py @@ -18,6 +18,8 @@ class TestDumpLoad(TestCase): together. """ + maxDiff = None + def setUp(self): for table_class in (SmallTable, MegaTable): table_class.create_table().run_sync() @@ -46,7 +48,9 @@ def insert_row(self): smallint_col=1, text_col="hello", timestamp_col=datetime.datetime(year=2021, month=1, day=1), - timestamptz_col=datetime.datetime(year=2021, month=1, day=1), + timestamptz_col=datetime.datetime( + year=2021, month=1, day=1, tzinfo=datetime.timezone.utc + ), uuid_col=uuid.UUID("12783854-c012-4c15-8183-8eecb46f2c4e"), varchar_col="hello", unique_col="hello", diff --git a/tests/table/test_select.py b/tests/table/test_select.py index bb8308600..18c000e6f 100644 --- a/tests/table/test_select.py +++ b/tests/table/test_select.py @@ -239,6 +239,15 @@ def test_where_is_null(self): response = query.run_sync() self.assertEqual(response, [{"name": "Managerless"}]) + def test_where_bool(self): + """ + If passing a boolean into a where clause, an exception should be + raised. It's possible for a user to do this by accident, for example + ``where(Band.has_drummer is None)``, which evaluates to a boolean. + """ + with self.assertRaises(ValueError): + Band.select().where(False) + def test_where_is_not_null(self): self.insert_rows() From 5aea0fdc4f40a99615f534ec49a38b242a5f5cb0 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 1 Mar 2022 07:46:40 -0500 Subject: [PATCH 273/727] bumped version --- CHANGES.rst | 19 +++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 69cce5797..89398ca88 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,25 @@ Changes ======= +0.69.3 +------ + +The ``where`` clause now raises a ``ValueError`` if a boolean value is +passed in by accident. This was possible in the following situation: + +.. code-block:: python + + await Band.select().where(Band.has_drummer is None) + +Piccolo can't override the ``is`` operator because Python doesn't allow it, +so ``Band.has_drummer is None`` will always equal ``False``. Thanks to +@trondhindenes for reporting this issue. + +We've also put a lot of effort into improving documentation throughout the +project. + +------------------------------------------------------------------------------- + 0.69.2 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index d5fc96a55..7784b8bc9 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.69.2" +__VERSION__ = "0.69.3" From ed90e4b3bc612d766e86f9f9fb417fadaa8c2a6a Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 3 Mar 2022 13:36:08 +0000 Subject: [PATCH 274/727] mypy ignore override (#448) --- piccolo/columns/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 43fb4f1ee..390c8e873 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -661,13 +661,13 @@ def _equals(self, column: Column, including_joins: bool = False) -> bool: return False - def __eq__(self, value) -> Where: # type: ignore + def __eq__(self, value) -> Where: # type: ignore[override] if value is None: return Where(column=self, operator=IsNull) else: return Where(column=self, value=value, operator=Equal) - def __ne__(self, value) -> Where: # type: ignore + def __ne__(self, value) -> Where: # type: ignore[override] if value is None: return Where(column=self, operator=IsNotNull) else: From 2d61fc8a26ccbf15bf688d770e4cdb57a013e3b4 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 4 Mar 2022 18:36:26 +0000 Subject: [PATCH 275/727] remove null check (#450) --- piccolo/columns/base.py | 6 ------ tests/columns/test_defaults.py | 20 -------------------- 2 files changed, 26 deletions(-) diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 390c8e873..805dd8c97 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -458,12 +458,6 @@ def __init__( } ) - if kwargs.get("default", ...) is None and not null: - raise ValueError( - "A default value of None isn't allowed if the column is " - "not nullable." - ) - if choices is not None: self._validate_choices(choices, allowed_type=self.value_type) diff --git a/tests/columns/test_defaults.py b/tests/columns/test_defaults.py index 1e2bb48bc..db95b165c 100644 --- a/tests/columns/test_defaults.py +++ b/tests/columns/test_defaults.py @@ -36,8 +36,6 @@ def test_int(self): _type(default=None, null=True) with self.assertRaises(ValueError): _type(default="hello world") - with self.assertRaises(ValueError): - _type(default=None, null=False) def test_text(self): for _type in (Text, Varchar): @@ -45,32 +43,24 @@ def test_text(self): _type(default=None, null=True) with self.assertRaises(ValueError): _type(default=123) - with self.assertRaises(ValueError): - _type(default=None, null=False) def test_real(self): Real(default=0.0) Real(default=None, null=True) with self.assertRaises(ValueError): Real(default="hello world") - with self.assertRaises(ValueError): - Real(default=None, null=False) def test_double_precision(self): DoublePrecision(default=0.0) DoublePrecision(default=None, null=True) with self.assertRaises(ValueError): DoublePrecision(default="hello world") - with self.assertRaises(ValueError): - DoublePrecision(default=None, null=False) def test_numeric(self): Numeric(default=decimal.Decimal(1.0)) Numeric(default=None, null=True) with self.assertRaises(ValueError): Numeric(default="hello world") - with self.assertRaises(ValueError): - Numeric(default=None, null=False) def test_uuid(self): UUID(default=None, null=True) @@ -78,8 +68,6 @@ def test_uuid(self): UUID(default=uuid.uuid4()) with self.assertRaises(ValueError): UUID(default="hello world") - with self.assertRaises(ValueError): - UUID(default=None, null=False) def test_time(self): Time(default=None, null=True) @@ -87,8 +75,6 @@ def test_time(self): Time(default=datetime.datetime.now().time()) with self.assertRaises(ValueError): Time(default="hello world") - with self.assertRaises(ValueError): - Time(default=None, null=False) def test_date(self): Date(default=None, null=True) @@ -96,8 +82,6 @@ def test_date(self): Date(default=datetime.datetime.now().date()) with self.assertRaises(ValueError): Date(default="hello world") - with self.assertRaises(ValueError): - Date(default=None, null=False) def test_timestamp(self): Timestamp(default=None, null=True) @@ -105,8 +89,6 @@ def test_timestamp(self): Timestamp(default=datetime.datetime.now()) with self.assertRaises(ValueError): Timestamp(default="hello world") - with self.assertRaises(ValueError): - Timestamp(default=None, null=False) def test_foreignkey(self): class MyTable(Table): @@ -116,5 +98,3 @@ class MyTable(Table): ForeignKey(references=MyTable, default=1) with self.assertRaises(ValueError): ForeignKey(references=MyTable, default="hello world") - with self.assertRaises(ValueError): - ForeignKey(references=MyTable, default=None, null=False) From b384f718e4c3a1d7ce9291076419109ef8b73365 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 5 Mar 2022 10:57:06 +0000 Subject: [PATCH 276/727] bumped version --- CHANGES.rst | 10 ++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 89398ca88..22c14906d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,16 @@ Changes ======= +0.69.4 +------ + +We used to raise a ``ValueError`` if a column was both ``null=False`` and +``default=None``. This has now been removed, as there are situations where +it's valid for columns to be configured that way. Thanks to @gmos for +suggesting this change. + +------------------------------------------------------------------------------- + 0.69.3 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 7784b8bc9..cb998c559 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.69.3" +__VERSION__ = "0.69.4" From 527f1637b4a2aae22a36e8c6fe2783837b579220 Mon Sep 17 00:00:00 2001 From: ali sayyah Date: Mon, 7 Mar 2022 02:41:17 +0330 Subject: [PATCH 277/727] get_output_schema exception with table name (#449) * get_output_schema will raise an exception with the name of the corresponding table if something goes wrong while generating the tables. * better testing * lint * handle AsyncMock in python 3.7 * lint * A workaround for lack of AsyncMock in python 3.7 * better testing * lint * using `AsyncMock` everywhere instead of `set_mock_return_value` * use `asyncio.run` * printing out index errors Co-authored-by: Daniel Townsend --- piccolo/apps/schema/commands/exceptions.py | 14 ++++ piccolo/apps/schema/commands/generate.py | 73 ++++++++++++++++--- .../migrations/auto/test_migration_manager.py | 14 ++-- tests/apps/schema/commands/test_generate.py | 51 ++++++++++++- tests/base.py | 21 ++---- 5 files changed, 144 insertions(+), 29 deletions(-) create mode 100644 piccolo/apps/schema/commands/exceptions.py diff --git a/piccolo/apps/schema/commands/exceptions.py b/piccolo/apps/schema/commands/exceptions.py new file mode 100644 index 000000000..58a8f5537 --- /dev/null +++ b/piccolo/apps/schema/commands/exceptions.py @@ -0,0 +1,14 @@ +class SchemaCommandError(Exception): + """ + Base class for all schema command errors. + """ + + pass + + +class GenerateError(SchemaCommandError): + """ + Raised when an error occurs during schema generation. + """ + + pass diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index 035a95e61..ca4891950 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -2,6 +2,7 @@ import asyncio import dataclasses +import itertools import json import re import typing as t @@ -12,6 +13,7 @@ from typing_extensions import Literal from piccolo.apps.migrations.auto.serialisation import serialise_params +from piccolo.apps.schema.commands.exceptions import GenerateError from piccolo.columns import defaults from piccolo.columns.base import Column, OnDelete, OnUpdate from piccolo.columns.column_types import ( @@ -197,11 +199,19 @@ def __post_init__(self): \(\"?(?P\w+?\"?)\)""", re.VERBOSE, ) - groups = re.match(pat, self.indexdef).groupdict() + match = re.match(pat, self.indexdef) + if match is None: + self.column_name = None + self.unique = None + self.method = None + self.warnings = [f"{self.indexdef};"] + else: + groups = match.groupdict() - self.column_name = groups["column_name"].lstrip('"').rstrip('"') - self.unique = True if "unique" in groups else False - self.method = INDEX_METHOD_MAP[groups["method"]] + self.column_name = groups["column_name"].lstrip('"').rstrip('"') + self.unique = True if "unique" in groups else False + self.method = INDEX_METHOD_MAP[groups["method"]] + self.warnings = [] @dataclasses.dataclass @@ -220,6 +230,14 @@ def get_column_index(self, column_name: str) -> t.Optional[Index]: return None + def get_warnings(self) -> t.List[str]: + return [ + i + for i in itertools.chain( + *[index.warnings for index in self.indexes] + ) + ] + @dataclasses.dataclass class OutputSchema: @@ -229,12 +247,15 @@ class OutputSchema: e.g. ["from piccolo.table import Table"] :param warnings: e.g. ["some_table.some_column unrecognised_type"] + :param index_warnings: + Warnings if column indexes can't be parsed. :param tables: e.g. ["class MyTable(Table): ..."] """ imports: t.List[str] = dataclasses.field(default_factory=list) warnings: t.List[str] = dataclasses.field(default_factory=list) + index_warnings: t.List[str] = dataclasses.field(default_factory=list) tables: t.List[t.Type[Table]] = dataclasses.field(default_factory=list) def get_table_with_name(self, tablename: str) -> t.Optional[t.Type[Table]]: @@ -254,12 +275,14 @@ def __radd__(self, value: OutputSchema) -> OutputSchema: return self value.imports.extend(self.imports) value.warnings.extend(self.warnings) + value.index_warnings.extend(self.index_warnings) value.tables.extend(self.tables) return value def __add__(self, value: OutputSchema) -> OutputSchema: self.imports.extend(value.imports) self.warnings.extend(value.warnings) + self.index_warnings.extend(value.index_warnings) self.tables.extend(value.tables) return self @@ -526,7 +549,6 @@ async def get_constraints( :param schema_name: Name of the schema. - """ constraints = await table_class.raw( ( @@ -636,9 +658,13 @@ def get_table_name(name: str, schema: str) -> str: async def create_table_class_from_db( table_class: t.Type[Table], tablename: str, schema_name: str ) -> OutputSchema: + output_schema = OutputSchema() + indexes = await get_indexes( table_class=table_class, tablename=tablename, schema_name=schema_name ) + output_schema.index_warnings.extend(indexes.get_warnings()) + constraints = await get_constraints( table_class=table_class, tablename=tablename, schema_name=schema_name ) @@ -648,7 +674,7 @@ async def create_table_class_from_db( table_schema = await get_table_schema( table_class=table_class, tablename=tablename, schema_name=schema_name ) - output_schema = OutputSchema() + columns: t.Dict[str, Column] = {} for pg_row_meta in table_schema: @@ -806,14 +832,35 @@ class Schema(Table, db=engine): if not include: include = await get_tablenames(Schema, schema_name=schema_name) + tablenames = [ + tablename for tablename in include if tablename not in exclude + ] table_coroutines = ( create_table_class_from_db( table_class=Schema, tablename=tablename, schema_name=schema_name ) - for tablename in include - if tablename not in exclude + for tablename in tablenames + ) + output_schemas = await asyncio.gather( + *table_coroutines, return_exceptions=True ) - output_schemas = await asyncio.gather(*table_coroutines) + + # handle exceptions + exceptions = [] + for obj, tablename in zip(output_schemas, tablenames): + if isinstance(obj, Exception): + exceptions.append((obj, tablename)) + + if exceptions: + raise GenerateError( + [ + type(e)( + f"Exception occurred while generating" + f" `{tablename}` table: {e}" + ) + for e, tablename in exceptions + ] + ) # Merge all the output schemas to a single OutputSchema object output_schema: OutputSchema = sum(output_schemas) # type: ignore @@ -853,6 +900,14 @@ async def generate(schema_name: str = "public"): output.append(warning_str) output.append('"""') + if output_schema.index_warnings: + warning_str = "\n".join(i for i in set(output_schema.index_warnings)) + + output.append('"""') + output.append("WARNING: Unable to parse the following indexes:") + output.append(warning_str) + output.append('"""') + nicely_formatted = black.format_str( "\n".join(output), mode=black.FileMode(line_length=79) ) diff --git a/tests/apps/migrations/auto/test_migration_manager.py b/tests/apps/migrations/auto/test_migration_manager.py index f3fd1f47a..2f1055271 100644 --- a/tests/apps/migrations/auto/test_migration_manager.py +++ b/tests/apps/migrations/auto/test_migration_manager.py @@ -10,7 +10,7 @@ from piccolo.conf.apps import AppConfig from piccolo.table import Table, sort_table_classes from piccolo.utils.lazy_loader import LazyLoader -from tests.base import DBTestCase, postgres_only, set_mock_return_value +from tests.base import AsyncMock, DBTestCase, postgres_only from tests.example_apps.music.tables import Band, Concert, Manager, Venue asyncpg = LazyLoader("asyncpg", globals(), "asyncpg") @@ -321,7 +321,9 @@ def test_add_non_nullable_column(self): asyncio.run(manager.run()) @postgres_only - @patch.object(BaseMigrationManager, "get_migration_managers") + @patch.object( + BaseMigrationManager, "get_migration_managers", new_callable=AsyncMock + ) @patch.object(BaseMigrationManager, "get_app_config") def test_drop_column( self, get_app_config: MagicMock, get_migration_managers: MagicMock @@ -355,7 +357,7 @@ def test_drop_column( self.assertEqual(response, [{"id": 1}]) # Reverse - set_mock_return_value(get_migration_managers, [manager_1]) + get_migration_managers.return_value = [manager_1] app_config = AppConfig(app_name="music", migrations_folder_path="") get_app_config.return_value = app_config asyncio.run(manager_2.run_backwards()) @@ -672,7 +674,9 @@ def test_alter_column_set_length(self): ) @postgres_only - @patch.object(BaseMigrationManager, "get_migration_managers") + @patch.object( + BaseMigrationManager, "get_migration_managers", new_callable=AsyncMock + ) @patch.object(BaseMigrationManager, "get_app_config") def test_drop_table( self, get_app_config: MagicMock, get_migration_managers: MagicMock @@ -695,7 +699,7 @@ def test_drop_table( self.assertTrue(not self.table_exists("musician")) # Reverse - set_mock_return_value(get_migration_managers, [manager_1]) + get_migration_managers.return_value = [manager_1] app_config = AppConfig(app_name="music", migrations_folder_path="") get_app_config.return_value = app_config asyncio.run(manager_2.run_backwards()) diff --git a/tests/apps/schema/commands/test_generate.py b/tests/apps/schema/commands/test_generate.py index a46dea39e..920436ce9 100644 --- a/tests/apps/schema/commands/test_generate.py +++ b/tests/apps/schema/commands/test_generate.py @@ -1,10 +1,12 @@ from __future__ import annotations import ast +import asyncio import typing as t from unittest import TestCase from unittest.mock import MagicMock, patch +from piccolo.apps.schema.commands.exceptions import GenerateError from piccolo.apps.schema.commands.generate import ( OutputSchema, generate, @@ -21,7 +23,7 @@ from piccolo.engine import Engine, engine_finder from piccolo.table import Table from piccolo.utils.sync import run_sync -from tests.base import postgres_only +from tests.base import AsyncMock, postgres_only from tests.example_apps.mega.tables import MegaTable, SmallTable @@ -260,3 +262,50 @@ def test_reference_to_another_schema(self): # Make sure foreign key values are correct. self.assertEqual(writer.publication, publication) self.assertEqual(book.writer, writer) + + +@postgres_only +class TestGenerateWithException(TestCase): + def setUp(self): + for table_class in (SmallTable, MegaTable): + table_class.create_table().run_sync() + + def tearDown(self): + for table_class in (MegaTable, SmallTable): + table_class.alter().drop_table(if_exists=True).run_sync() + + @patch( + "piccolo.apps.schema.commands.generate.create_table_class_from_db", + new_callable=AsyncMock, + ) + def test_exception(self, create_table_class_from_db_mock: AsyncMock): + """ + Make sure that a GenerateError exception + is raised with all the exceptions gathered. + """ + create_table_class_from_db_mock.side_effect = [ + ValueError("Test"), + TypeError("Test"), + ] + + # Make sure the exception is raised. + with self.assertRaises(GenerateError) as e: + asyncio.run(get_output_schema()) + + # Make sure the exception contains the correct number of errors. + self.assertEqual(len(e.exception.args[0]), 2) + # assert that the two exceptions are ValueError and TypeError + exception_types = [type(e) for e in e.exception.args[0]] + self.assertIn(ValueError, exception_types) + self.assertIn(TypeError, exception_types) + + # Make sure the exception contains the correct error messages. + exception_messages = [str(e) for e in e.exception.args[0]] + self.assertIn( + "Exception occurred while generating `small_table` table: Test", + exception_messages, + ) + self.assertIn( + "Exception occurred while generating `mega_table` table: Test", + exception_messages, + ) diff --git a/tests/base.py b/tests/base.py index 3d9c03851..0025c0551 100644 --- a/tests/base.py +++ b/tests/base.py @@ -15,36 +15,29 @@ ENGINE = engine_finder() - postgres_only = pytest.mark.skipif( not isinstance(ENGINE, PostgresEngine), reason="Only running for Postgres" ) - sqlite_only = pytest.mark.skipif( not isinstance(ENGINE, SQLiteEngine), reason="Only running for SQLite" ) - unix_only = pytest.mark.skipif( sys.platform.startswith("win"), reason="Only running on a Unix system" ) -def set_mock_return_value(magic_mock: MagicMock, return_value: t.Any): +class AsyncMock(MagicMock): """ - Python 3.8 has good support for mocking coroutines. For older versions, - we must set the return value to be an awaitable explicitly. - """ - if magic_mock.__class__.__name__ == "AsyncMock": - # Python 3.8 and above - magic_mock.return_value = return_value - else: + Async MagicMock for python 3.7+. - async def coroutine(*args, **kwargs): - return return_value + This is a workaround for the fact that MagicMock is not async compatible in + Python 3.7. + """ - magic_mock.return_value = coroutine() + async def __call__(self, *args, **kwargs): + return super(AsyncMock, self).__call__(*args, **kwargs) class DBTestCase(TestCase): From bf14671a9580249cc5ac4bc6be3d76f6c6505894 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 6 Mar 2022 23:26:20 +0000 Subject: [PATCH 278/727] bumped version --- CHANGES.rst | 16 ++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 22c14906d..df53298ba 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,22 @@ Changes ======= +0.69.5 +------ + +Made improvements to ``piccolo schema generate``, which automatically generates +Piccolo ``Table`` classes from an existing database. + +There were situations where it would fail ungracefully when it couldn't parse +an index definition. It no longer crashes, and we print out the problematic +index definitions. See `PR 449 `_. +Thanks to @gmos for originally reporting this issue. + +We also improved the error messages if schema generation fails for some reason +by letting the user know which table caused the error. Courtesy @AliSayyah. + +------------------------------------------------------------------------------- + 0.69.4 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index cb998c559..2b637993d 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.69.4" +__VERSION__ = "0.69.5" From 0edbdb4b983dfec56a8d187526d8a90d64319f4c Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 7 Mar 2022 23:24:46 +0000 Subject: [PATCH 279/727] profiling proof of concept (#454) * profiling proof of concept * add viztracer as an option * improve performance converting raw data into dicts * fix broken sqlite tests * remove unused variable * stop redundant querystring lookup Running `self.querystring` is actually quite time consuming, so store a local copy rather than looking it up twice. * remove redundant self.querystring lookup * tidied up profiling code --- .gitignore | 1 + docs/src/piccolo/contributing/index.rst | 12 ++++++++ piccolo/query/base.py | 20 +++++++++---- profiling/README.md | 5 ++++ profiling/__init__.py | 0 profiling/run_profile.py | 40 +++++++++++++++++++++++++ requirements/profile-requirements.txt | 1 + scripts/README.md | 1 + scripts/profile.sh | 2 ++ 9 files changed, 77 insertions(+), 5 deletions(-) create mode 100644 profiling/README.md create mode 100644 profiling/__init__.py create mode 100644 profiling/run_profile.py create mode 100644 requirements/profile-requirements.txt create mode 100755 scripts/profile.sh diff --git a/.gitignore b/.gitignore index 2734abca6..c14d1f2f0 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ htmlcov/ prof/ .env/ .venv/ +result.json diff --git a/docs/src/piccolo/contributing/index.rst b/docs/src/piccolo/contributing/index.rst index 853ceabc2..9e22fdd68 100644 --- a/docs/src/piccolo/contributing/index.rst +++ b/docs/src/piccolo/contributing/index.rst @@ -61,3 +61,15 @@ You can configure `VSCode `_ by modifying } Type hints are used throughout the project. + +------------------------------------------------------------------------------- + +Profiling +--------- + +This isn't required to contribute to Piccolo, but is useful when investigating +performance problems. + + * Install the dependencies: ``pip install requirements/profile-requirements.txt`` + * Make sure a Postgres database called ``piccolo_profile`` exists. + * Run ``./scripts/profile.sh`` to get performance data. diff --git a/piccolo/query/base.py b/piccolo/query/base.py index 1573b3405..509bc8bd8 100644 --- a/piccolo/query/base.py +++ b/piccolo/query/base.py @@ -49,7 +49,15 @@ async def _process_results(self, results): # noqa: C901 if results: keys = results[0].keys() keys = [i.replace("$", ".") for i in keys] - raw = [dict(zip(keys, i.values())) for i in results] + if self.engine_type == "postgres": + # asyncpg returns a special Record object. We can pass it + # directly into zip without calling `values` on it. This can + # save us hundreds of microseconds, depending on the number of + # results. + raw = [dict(zip(keys, i)) for i in results] + else: + # SQLite returns a list of dictionaries. + raw = [dict(zip(keys, i.values())) for i in results] else: raw = [] @@ -174,15 +182,17 @@ async def run(self, in_pool=True): "_meta" ) - if len(self.querystrings) == 1: + querystrings = self.querystrings + + if len(querystrings) == 1: results = await engine.run_querystring( - self.querystrings[0], in_pool=in_pool + querystrings[0], in_pool=in_pool ) return await self._process_results(results) else: responses = [] # TODO - run in a transaction - for querystring in self.querystrings: + for querystring in querystrings: results = await engine.run_querystring( querystring, in_pool=in_pool ) @@ -291,7 +301,7 @@ async def top_bands(self, request): # Copy the query, so we don't store any references to the original. query = self.__class__( - table=self.table, frozen_querystrings=self.querystrings + table=self.table, frozen_querystrings=querystrings ) if hasattr(self, "limit_delegate"): diff --git a/profiling/README.md b/profiling/README.md new file mode 100644 index 000000000..22e6eb852 --- /dev/null +++ b/profiling/README.md @@ -0,0 +1,5 @@ +# Profiling + +Tests we run to evaluate Piccolo performance. + +You need to setup a local Postgres database called 'piccolo_profile'. diff --git a/profiling/__init__.py b/profiling/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/profiling/run_profile.py b/profiling/run_profile.py new file mode 100644 index 000000000..9dc53e081 --- /dev/null +++ b/profiling/run_profile.py @@ -0,0 +1,40 @@ +import asyncio + +from viztracer import VizTracer + +from piccolo.columns.column_types import Varchar +from piccolo.engine.postgres import PostgresEngine +from piccolo.table import Table + +DB = PostgresEngine(config={"database": "piccolo_profile"}) + + +class Band(Table, db=DB): + name = Varchar() + + +async def setup(): + await Band.alter().drop_table(if_exists=True) + await Band.create_table(if_not_exists=True) + await Band.insert(*[Band(name="test") for _ in range(1000)]) + + +class Trace: + def __enter__(self): + self.tracer = VizTracer(log_async=True) + self.tracer.start() + + def __exit__(self, *args): + self.tracer.stop() + self.tracer.save() + + +async def run_queries(): + await setup() + + with Trace(): + await Band.select() + + +if __name__ == "__main__": + asyncio.run(run_queries()) diff --git a/requirements/profile-requirements.txt b/requirements/profile-requirements.txt new file mode 100644 index 000000000..7c0fca7d8 --- /dev/null +++ b/requirements/profile-requirements.txt @@ -0,0 +1 @@ +viztracer==0.15.0 diff --git a/scripts/README.md b/scripts/README.md index 2140f82ad..6ffbafa09 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -7,6 +7,7 @@ Call them from the root of the project, e.g. `./scripts/lint.sh`. - `scripts/format.sh` - Format the code to the required standards. - `scripts/lint.sh` - Run the automated code linting/formatting tools. - `scripts/piccolo.sh` - Run the Piccolo CLI on the example project in the `tests` folder. +- `scripts/profile.sh` - Run a profiler to test performance. - `scripts/release.sh` - Publish package to PyPI. - `scripts/test-postgres.sh` - Run the test suite with Postgres. - `scripts/test-sqlite.sh` - Run the test suite with SQLite. diff --git a/scripts/profile.sh b/scripts/profile.sh new file mode 100755 index 000000000..5396f2bcf --- /dev/null +++ b/scripts/profile.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python -m profiling.run_profile && vizviewer result.json From 753ae548f6bab982134e64be0e2b91370cec7e3b Mon Sep 17 00:00:00 2001 From: ali sayyah Date: Tue, 8 Mar 2022 03:03:15 +0330 Subject: [PATCH 280/727] comprehensions and literals (#455) * get_output_schema will raise an exception with the name of the corresponding table if something goes wrong while generating the tables. * better testing * lint * handle AsyncMock in python 3.7 * lint * A workaround for lack of AsyncMock in python 3.7 * better testing * lint * removed unnecessary comprehensions and used literals where ever is possible * lint * Unnecessary list comprehension --- piccolo/apps/fixtures/commands/load.py | 4 +--- piccolo/apps/migrations/auto/migration_manager.py | 8 ++++---- piccolo/apps/migrations/auto/schema_differ.py | 2 +- piccolo/apps/migrations/auto/serialisation.py | 10 +++++----- piccolo/apps/migrations/commands/base.py | 2 +- piccolo/apps/migrations/commands/new.py | 4 ++-- piccolo/apps/schema/commands/generate.py | 11 ++++------- piccolo/columns/column_types.py | 4 +--- piccolo/conf/apps.py | 4 ++-- piccolo/query/methods/alter.py | 5 +---- piccolo/query/methods/select.py | 3 +-- piccolo/query/methods/table_exists.py | 2 +- piccolo/table.py | 2 +- piccolo/utils/dictionary.py | 2 +- piccolo/utils/pydantic.py | 7 +++---- 15 files changed, 29 insertions(+), 41 deletions(-) diff --git a/piccolo/apps/fixtures/commands/load.py b/piccolo/apps/fixtures/commands/load.py index b9a21a858..8d1d2cc60 100644 --- a/piccolo/apps/fixtures/commands/load.py +++ b/piccolo/apps/fixtures/commands/load.py @@ -24,9 +24,7 @@ async def load_json_string(json_string: str): fixture_configs = [ FixtureConfig( app_name=app_name, - table_class_names=[ - i for i in deserialised_contents[app_name].keys() - ], + table_class_names=list(deserialised_contents[app_name].keys()), ) for app_name in app_names ] diff --git a/piccolo/apps/migrations/auto/migration_manager.py b/piccolo/apps/migrations/auto/migration_manager.py index 59a64da0a..405a9ef3b 100644 --- a/piccolo/apps/migrations/auto/migration_manager.py +++ b/piccolo/apps/migrations/auto/migration_manager.py @@ -53,7 +53,7 @@ def columns_for_table_class_name( @property def table_class_names(self) -> t.List[str]: - return list(set([i.table_class_name for i in self.add_columns])) + return list({i.table_class_name for i in self.add_columns}) @dataclass @@ -74,7 +74,7 @@ def for_table_class_name( @property def table_class_names(self) -> t.List[str]: - return list(set([i.table_class_name for i in self.drop_columns])) + return list({i.table_class_name for i in self.drop_columns}) @dataclass @@ -95,7 +95,7 @@ def for_table_class_name( @property def table_class_names(self) -> t.List[str]: - return list(set([i.table_class_name for i in self.rename_columns])) + return list({i.table_class_name for i in self.rename_columns}) @dataclass @@ -116,7 +116,7 @@ def for_table_class_name( @property def table_class_names(self) -> t.List[str]: - return list(set([i.table_class_name for i in self.alter_columns])) + return list({i.table_class_name for i in self.alter_columns}) @dataclass diff --git a/piccolo/apps/migrations/auto/schema_differ.py b/piccolo/apps/migrations/auto/schema_differ.py index 859bf662b..54fd1015b 100644 --- a/piccolo/apps/migrations/auto/schema_differ.py +++ b/piccolo/apps/migrations/auto/schema_differ.py @@ -519,4 +519,4 @@ def get_alter_statements(self) -> t.List[AlterStatements]: count = len(statements.statements) print(f"{_message} {count}") - return [i for i in alter_statements.values()] + return list(alter_statements.values()) diff --git a/piccolo/apps/migrations/auto/serialisation.py b/piccolo/apps/migrations/auto/serialisation.py index 44d99e9d3..e1de712be 100644 --- a/piccolo/apps/migrations/auto/serialisation.py +++ b/piccolo/apps/migrations/auto/serialisation.py @@ -52,11 +52,11 @@ def __new__(mcs, name, bases, class_attributes): bases, mcs.merge_class_attributes( class_attributes_with_columns, - dict( - unique_names=mcs.get_unique_class_attribute_values( + { + "unique_names": mcs.get_unique_class_attribute_values( class_attributes_with_columns ) - ), + }, ), ) @@ -706,10 +706,10 @@ def serialise_params(params: t.Dict[str, t.Any]) -> SerialisedParams: # All other types can remain as is. - unique_extra_imports = [i for i in set(extra_imports)] + unique_extra_imports = list(set(extra_imports)) UniqueGlobalNames.warn_if_are_conflicting_objects(unique_extra_imports) - unique_extra_definitions = [i for i in set(extra_definitions)] + unique_extra_definitions = list(set(extra_definitions)) UniqueGlobalNames.warn_if_are_conflicting_objects(unique_extra_definitions) return SerialisedParams( diff --git a/piccolo/apps/migrations/commands/base.py b/piccolo/apps/migrations/commands/base.py index dcf6734e4..fce1b37c1 100644 --- a/piccolo/apps/migrations/commands/base.py +++ b/piccolo/apps/migrations/commands/base.py @@ -67,7 +67,7 @@ def get_migration_ids( """ Returns a list of migration IDs, from the Python migration files. """ - return sorted(list(migration_module_dict.keys())) + return sorted(migration_module_dict.keys()) async def get_migration_managers( self, diff --git a/piccolo/apps/migrations/commands/new.py b/piccolo/apps/migrations/commands/new.py index 49174e61a..937cb8627 100644 --- a/piccolo/apps/migrations/commands/new.py +++ b/piccolo/apps/migrations/commands/new.py @@ -107,11 +107,11 @@ async def _create_new_migration( chain(*[i.statements for i in alter_statements]) ) extra_imports = sorted( - list(set(chain(*[i.extra_imports for i in alter_statements]))), + set(chain(*[i.extra_imports for i in alter_statements])), key=lambda x: x.__repr__(), ) extra_definitions = sorted( - list(set(chain(*[i.extra_definitions for i in alter_statements]))), + set(chain(*[i.extra_definitions for i in alter_statements])), ) if sum(len(i.statements) for i in alter_statements) == 0: diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index ca4891950..e0505cf81 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -231,12 +231,9 @@ def get_column_index(self, column_name: str) -> t.Optional[Index]: return None def get_warnings(self) -> t.List[str]: - return [ - i - for i in itertools.chain( - *[index.warnings for index in self.indexes] - ) - ] + return list( + itertools.chain(*[index.warnings for index in self.indexes]) + ) @dataclasses.dataclass @@ -869,7 +866,7 @@ class Schema(Table, db=engine): output_schema.tables = sort_table_classes( sorted(output_schema.tables, key=lambda x: x._meta.tablename) ) - output_schema.imports = sorted(list(set(output_schema.imports))) + output_schema.imports = sorted(set(output_schema.imports)) return output_schema diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 159d8b5e2..0c513d9f5 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -1873,9 +1873,7 @@ def __getattribute__(self, name: str) -> t.Union[Column, t.Any]: column ) in value._foreign_key_meta.resolved_references._meta.columns: _column: Column = column.copy() - _column._meta.call_chain = [ - i for i in new_column._meta.call_chain - ] + _column._meta.call_chain = list(new_column._meta.call_chain) setattr(new_column, _column._meta.name, _column) foreign_key_meta.proxy_columns.append(_column) diff --git a/piccolo/conf/apps.py b/piccolo/conf/apps.py index a202328bf..346de7678 100644 --- a/piccolo/conf/apps.py +++ b/piccolo/conf/apps.py @@ -229,7 +229,7 @@ def _validate_app_names(app_names: t.List[str]): app_names.sort() grouped = itertools.groupby(app_names) for key, value in grouped: - count = len([i for i in value]) + count = len(list(value)) if count > 1: raise ValueError( f"There are {count} apps with the name `{key}`. This can " @@ -303,7 +303,7 @@ def _deduplicate( Remove all duplicates - just leaving the first instance. """ # Deduplicate, but preserve order - which is why set() isn't used. - return list(dict([(c, None) for c in config_modules]).keys()) + return list({c: None for c in config_modules}.keys()) def _import_app_modules( self, config_module_paths: t.List[str] diff --git a/piccolo/query/methods/alter.py b/piccolo/query/methods/alter.py index 518eb3b81..47c01f8e7 100644 --- a/piccolo/query/methods/alter.py +++ b/piccolo/query/methods/alter.py @@ -15,7 +15,7 @@ class AlterStatement: - __slots__ = tuple() # type: ignore + __slots__ = () # type: ignore @property def ddl(self) -> str: @@ -168,7 +168,6 @@ def ddl(self) -> str: @dataclass class SetLength(AlterColumnStatement): - __slots__ = ("length",) length: int @@ -222,7 +221,6 @@ def ddl(self) -> str: @dataclass class SetDigits(AlterColumnStatement): - __slots__ = ("digits", "column_type") digits: t.Optional[t.Tuple[int, int]] @@ -263,7 +261,6 @@ def ddl(self) -> str: class Alter(DDL): - __slots__ = ( "_add_foreign_key_constraint", "_add", diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 12bf4d2db..b9024f48a 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -211,7 +211,6 @@ def get_select_string(self, engine_type: str, just_alias=False) -> str: class Select(Query): - __slots__ = ( "columns_list", "exclude_secrets", @@ -286,7 +285,7 @@ async def _splice_m2m_rows( as_list: bool = False, ): row_ids = list( - {i for i in itertools.chain(*[row[m2m_name] for row in response])} + set(itertools.chain(*[row[m2m_name] for row in response])) ) extra_rows = ( ( diff --git a/piccolo/query/methods/table_exists.py b/piccolo/query/methods/table_exists.py index 72a399ff5..836b6e54a 100644 --- a/piccolo/query/methods/table_exists.py +++ b/piccolo/query/methods/table_exists.py @@ -8,7 +8,7 @@ class TableExists(Query): - __slots__: t.Tuple = tuple() + __slots__: t.Tuple = () async def response_handler(self, response): return bool(response[0]["exists"]) diff --git a/piccolo/table.py b/piccolo/table.py index 8bff8e6b9..53d6c62c9 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -310,7 +310,7 @@ def __init__( unrecognized = kwargs.keys() if unrecognized: - unrecognised_list = [i for i in unrecognized] + unrecognised_list = list(unrecognized) raise ValueError(f"Unrecognized columns - {unrecognised_list}") @classmethod diff --git a/piccolo/utils/dictionary.py b/piccolo/utils/dictionary.py index e15e477c3..82fa2368d 100644 --- a/piccolo/utils/dictionary.py +++ b/piccolo/utils/dictionary.py @@ -22,7 +22,7 @@ def make_nested(dictionary: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]: """ output: t.Dict[str, t.Any] = {} - items = [i for i in dictionary.items()] + items = list(dictionary.items()) items.sort(key=lambda x: x[0]) for key, value in items: diff --git a/piccolo/utils/pydantic.py b/piccolo/utils/pydantic.py index 5f270faf9..47ea1caa2 100644 --- a/piccolo/utils/pydantic.py +++ b/piccolo/utils/pydantic.py @@ -174,12 +174,11 @@ def create_pydantic_model( ) if include_columns: - include_columns_plus_ancestors = [ - i - for i in itertools.chain( + include_columns_plus_ancestors = list( + itertools.chain( include_columns, *[i._meta.call_chain for i in include_columns] ) - ] + ) piccolo_columns = tuple( i for i in piccolo_columns From ddfc34488e7efc0ae0829207ee239fd242aca8ec Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 8 Mar 2022 00:23:11 +0000 Subject: [PATCH 281/727] bumped version --- CHANGES.rst | 26 ++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index df53298ba..34c9870a3 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,32 @@ Changes ======= +0.70.0 +------ + +We ran a profiler on the Piccolo codebase and identified some optimisations. +For example, we were calling ``self.querystring`` multiple times in a method, +rather than assigning it to a local variable. + +We also ran a linter which identified when list / set / dict comprehensions +could be more efficient. + +The performance is now slightly improved (especially when fetching large +numbers of rows from the database). + +Example query times on a MacBook, when fetching 1000 rows from a local Postgres +database (using ``await SomeTable.select()``): + +* 8 ms without a connection pool +* 2 ms with a connection pool + +As you can see, having a connection pool is the main thing you can do to +improve performance. + +Thanks to @AliSayyah for all his work on this. + +------------------------------------------------------------------------------- + 0.69.5 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 2b637993d..bd812e826 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.69.5" +__VERSION__ = "0.70.0" From 3219b72541e346d15e6fd37bab95b859b87fe2e4 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 9 Mar 2022 09:55:39 +0000 Subject: [PATCH 282/727] Rename multiple columns in an auto migration (#457) * add extra tests for detecting renamed columns * make order deterministic * improve rename column checks If we already know that column A was renamed to column B, don't then ask if column C was also renamed to column B. * one more unit test --- .../apps/migrations/auto/diffable_table.py | 20 +- piccolo/apps/migrations/auto/schema_differ.py | 15 +- .../migrations/auto/test_schema_differ.py | 192 +++++++++++++++++- 3 files changed, 212 insertions(+), 15 deletions(-) diff --git a/piccolo/apps/migrations/auto/diffable_table.py b/piccolo/apps/migrations/auto/diffable_table.py index 318edfd29..e73065ede 100644 --- a/piccolo/apps/migrations/auto/diffable_table.py +++ b/piccolo/apps/migrations/auto/diffable_table.py @@ -109,6 +109,12 @@ def __sub__(self, value: DiffableTable) -> TableDelta: "The two tables don't appear to have the same name." ) + ####################################################################### + + # Because we're using sets here, the order is indeterminate. We sort + # them, otherwise it's difficult to write good unit tests if the order + # constantly changes. + add_columns = [ AddColumn( table_class_name=self.class_name, @@ -118,9 +124,12 @@ def __sub__(self, value: DiffableTable) -> TableDelta: column_class=i.column.__class__, params=i.column._meta.params, ) - for i in ( + for i in sorted( {ColumnComparison(column=column) for column in self.columns} - - {ColumnComparison(column=column) for column in value.columns} + - { + ColumnComparison(column=column) for column in value.columns + }, + key=lambda x: x.column._meta.name, ) ] @@ -131,12 +140,15 @@ def __sub__(self, value: DiffableTable) -> TableDelta: db_column_name=i.column._meta.db_column_name, tablename=value.tablename, ) - for i in ( + for i in sorted( {ColumnComparison(column=column) for column in value.columns} - - {ColumnComparison(column=column) for column in self.columns} + - {ColumnComparison(column=column) for column in self.columns}, + key=lambda x: x.column._meta.name, ) ] + ####################################################################### + alter_columns: t.List[AlterColumn] = [] for existing_column in value.columns: diff --git a/piccolo/apps/migrations/auto/schema_differ.py b/piccolo/apps/migrations/auto/schema_differ.py index 54fd1015b..71c4cc508 100644 --- a/piccolo/apps/migrations/auto/schema_differ.py +++ b/piccolo/apps/migrations/auto/schema_differ.py @@ -200,13 +200,17 @@ def check_renamed_columns(self) -> RenameColumnCollection: # type. For now, each time a column is added and removed from a # table, ask if it's a rename. - renamed_column_names: t.List[str] = [] + # We track which dropped columns have already been identified by + # the user as renames, so we don't ask them if another column + # was also renamed from it. + used_drop_column_names: t.List[str] = [] for add_column in delta.add_columns: - if add_column.table_class_name in renamed_column_names: - continue for drop_column in delta.drop_columns: + if drop_column.column_name in used_drop_column_names: + continue + user_response = ( self.auto_input if self.auto_input @@ -217,9 +221,7 @@ def check_renamed_columns(self) -> RenameColumnCollection: ) ) if user_response.lower() == "y": - renamed_column_names.append( - add_column.table_class_name - ) + used_drop_column_names.append(drop_column.column_name) collection.append( RenameColumn( table_class_name=add_column.table_class_name, @@ -230,6 +232,7 @@ def check_renamed_columns(self) -> RenameColumnCollection: new_db_column_name=add_column.db_column_name, ) ) + break return collection diff --git a/tests/apps/migrations/auto/test_schema_differ.py b/tests/apps/migrations/auto/test_schema_differ.py index 35e4a6a1f..6fccff2e5 100644 --- a/tests/apps/migrations/auto/test_schema_differ.py +++ b/tests/apps/migrations/auto/test_schema_differ.py @@ -2,6 +2,7 @@ import typing as t from unittest import TestCase +from unittest.mock import MagicMock, call, patch from piccolo.apps.migrations.auto import DiffableTable, SchemaDiffer from piccolo.columns.column_types import Numeric, Varchar @@ -164,12 +165,20 @@ def test_rename_column(self): """ Test renaming a column in an existing table. """ + # We're going to rename the 'name' column to 'title' name_column = Varchar() name_column._meta.name = "name" title_column = Varchar() title_column._meta.name = "title" + schema_snapshot: t.List[DiffableTable] = [ + DiffableTable( + class_name="Band", + tablename="band", + columns=[title_column], + ) + ] schema: t.List[DiffableTable] = [ DiffableTable( class_name="Band", @@ -177,22 +186,195 @@ def test_rename_column(self): columns=[name_column], ) ] + + # Test 1 - Tell Piccolo the column was renamed + schema_differ = SchemaDiffer( + schema=schema, schema_snapshot=schema_snapshot, auto_input="y" + ) + self.assertEqual(schema_differ.add_columns.statements, []) + self.assertEqual(schema_differ.drop_columns.statements, []) + self.assertEqual( + schema_differ.rename_columns.statements, + [ + "manager.rename_column(table_class_name='Band', tablename='band', old_column_name='title', new_column_name='name', old_db_column_name='title', new_db_column_name='name')" # noqa + ], + ) + + # Test 2 - Tell Piccolo the column wasn't renamed + schema_differ = SchemaDiffer( + schema=schema, schema_snapshot=schema_snapshot, auto_input="n" + ) + self.assertEqual( + schema_differ.add_columns.statements, + [ + "manager.add_column(table_class_name='Band', tablename='band', column_name='name', db_column_name='name', column_class_name='Varchar', column_class=Varchar, params={'length': 255, 'default': '', 'null': False, 'primary_key': False, 'unique': False, 'index': False, 'index_method': IndexMethod.btree, 'choices': None, 'db_column_name': None, 'secret': False})" # noqa: E501 + ], + ) + self.assertEqual( + schema_differ.drop_columns.statements, + [ + "manager.drop_column(table_class_name='Band', tablename='band', column_name='title', db_column_name='title')" # noqa: E501 + ], + ) + self.assertTrue(schema_differ.rename_columns.statements == []) + + @patch("piccolo.apps.migrations.auto.schema_differ.input") + def test_rename_multiple_columns(self, input: MagicMock): + """ + Make sure renaming columns works when several columns have been + renamed. + """ + # We're going to rename a1 to a2, and b1 to b2. + a1 = Varchar() + a1._meta.name = "a1" + + a2 = Varchar() + a2._meta.name = "a2" + + b1 = Varchar() + b1._meta.name = "b1" + + b2 = Varchar() + b2._meta.name = "b2" + schema_snapshot: t.List[DiffableTable] = [ DiffableTable( class_name="Band", tablename="band", - columns=[title_column], + columns=[a1, b1], + ) + ] + schema: t.List[DiffableTable] = [ + DiffableTable( + class_name="Band", + tablename="band", + columns=[a2, b2], + ) + ] + + def mock_input(value: str): + """ + We need to dynamically set the return value based on what's passed + in. + """ + return ( + "y" + if value + in ( + "Did you rename the `a1` column to `a2` on the `Band` table? (y/N)", # noqa: E501 + "Did you rename the `b1` column to `b2` on the `Band` table? (y/N)", # noqa: E501 + ) + else "n" + ) + + input.side_effect = mock_input + + schema_differ = SchemaDiffer( + schema=schema, schema_snapshot=schema_snapshot + ) + self.assertEqual(schema_differ.add_columns.statements, []) + self.assertEqual(schema_differ.drop_columns.statements, []) + self.assertEqual( + schema_differ.rename_columns.statements, + [ + "manager.rename_column(table_class_name='Band', tablename='band', old_column_name='a1', new_column_name='a2', old_db_column_name='a1', new_db_column_name='a2')", # noqa: E501 + "manager.rename_column(table_class_name='Band', tablename='band', old_column_name='b1', new_column_name='b2', old_db_column_name='b1', new_db_column_name='b2')", # noqa: E501 + ], + ) + + self.assertEqual( + input.call_args_list, + [ + call( + "Did you rename the `a1` column to `a2` on the `Band` table? (y/N)" # noqa: E501 + ), + call( + "Did you rename the `b1` column to `b2` on the `Band` table? (y/N)" # noqa: E501 + ), + ], + ) + + @patch("piccolo.apps.migrations.auto.schema_differ.input") + def test_rename_some_columns(self, input: MagicMock): + """ + Make sure that some columns can be marked as renamed, and others are + dropped / created. + """ + # We're going to rename a1 to a2, but want b1 to be dropped, and b2 to + # be created. + a1 = Varchar() + a1._meta.name = "a1" + + a2 = Varchar() + a2._meta.name = "a2" + + b1 = Varchar() + b1._meta.name = "b1" + + b2 = Varchar() + b2._meta.name = "b2" + + schema_snapshot: t.List[DiffableTable] = [ + DiffableTable( + class_name="Band", + tablename="band", + columns=[a1, b1], ) ] + schema: t.List[DiffableTable] = [ + DiffableTable( + class_name="Band", + tablename="band", + columns=[a2, b2], + ) + ] + + def mock_input(value: str): + """ + We need to dynamically set the return value based on what's passed + in. + """ + return ( + "y" + if value + == "Did you rename the `a1` column to `a2` on the `Band` table? (y/N)" # noqa: E501 + else "n" + ) + + input.side_effect = mock_input schema_differ = SchemaDiffer( - schema=schema, schema_snapshot=schema_snapshot, auto_input="y" + schema=schema, schema_snapshot=schema_snapshot + ) + self.assertEqual( + schema_differ.add_columns.statements, + [ + "manager.add_column(table_class_name='Band', tablename='band', column_name='b2', db_column_name='b2', column_class_name='Varchar', column_class=Varchar, params={'length': 255, 'default': '', 'null': False, 'primary_key': False, 'unique': False, 'index': False, 'index_method': IndexMethod.btree, 'choices': None, 'db_column_name': None, 'secret': False})" # noqa: E501 + ], + ) + self.assertEqual( + schema_differ.drop_columns.statements, + [ + "manager.drop_column(table_class_name='Band', tablename='band', column_name='b1', db_column_name='b1')" # noqa: E501 + ], + ) + self.assertEqual( + schema_differ.rename_columns.statements, + [ + "manager.rename_column(table_class_name='Band', tablename='band', old_column_name='a1', new_column_name='a2', old_db_column_name='a1', new_db_column_name='a2')", # noqa: E501 + ], ) - self.assertTrue(len(schema_differ.rename_columns.statements) == 1) self.assertEqual( - schema_differ.rename_columns.statements[0], - "manager.rename_column(table_class_name='Band', tablename='band', old_column_name='title', new_column_name='name', old_db_column_name='title', new_db_column_name='name')", # noqa + input.call_args_list, + [ + call( + "Did you rename the `a1` column to `a2` on the `Band` table? (y/N)" # noqa: E501 + ), + call( + "Did you rename the `b1` column to `b2` on the `Band` table? (y/N)" # noqa: E501 + ), + ], ) def test_alter_column_precision(self): From 2314736ee4657fbd5ad8d3c462bb749f30d800f4 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 9 Mar 2022 10:00:32 +0000 Subject: [PATCH 283/727] bumped version --- CHANGES.rst | 9 +++++++++ piccolo/__init__.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 34c9870a3..21f05d429 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,15 @@ Changes ======= +0.70.1 +------ + +Fixed a bug with auto migrations. If renaming multiple columns at once, it +could get confused. Thanks to @theelderbeever for reporting this issue, and +@sinisaos for helping to replicate it. See `PR 457 `_. + +------------------------------------------------------------------------------- + 0.70.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index bd812e826..9ec35a636 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.70.0" +__VERSION__ = "0.70.1" From fb503eb941cd48de96613ff947ba507539ec0aad Mon Sep 17 00:00:00 2001 From: Yasser Tahiri Date: Wed, 9 Mar 2022 19:23:20 +0100 Subject: [PATCH 284/727] =?UTF-8?q?=F0=9F=AA=84=20=20Chore(refactor):=20im?= =?UTF-8?q?prove=20and=20Refactor=20Piccolo=20Functionality=20(#456)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐍: Refactor app Folder and Fix Code Issue * ⚡️: Fix Engine Code * 🌪: Fix Issue relate to Return & Refactor the Code * ❄️: Refactor piccolo configuration * ☀️: Refactor Utils & Fix Code Issues * 🪶: Fix Queries Code * 🐍: Refactor Multipart of Code * 🐛: Extract Some Methods and Variables * 🪄: Revert `test_apps` & `test_insert` * 📦: Use `any()` instead of for loop * Combines multiple `isinstance` functions * Convert `for` loop into dictionary comprehension * chore(ref): Improve and Refactor code functionality * fix: delete the ref named expression * chore(fix): format the files --- piccolo/apps/app/commands/new.py | 2 +- piccolo/apps/asgi/commands/new.py | 2 +- piccolo/apps/fixtures/commands/shared.py | 6 +-- .../apps/migrations/auto/diffable_table.py | 4 +- .../apps/migrations/auto/migration_manager.py | 19 +++++--- piccolo/apps/migrations/auto/schema_differ.py | 17 ++----- piccolo/apps/migrations/auto/serialisation.py | 6 +-- .../migrations/auto/serialisation_legacy.py | 29 +++++------- piccolo/apps/migrations/commands/backwards.py | 23 +++++----- piccolo/apps/migrations/commands/base.py | 2 +- piccolo/apps/schema/commands/generate.py | 19 +++----- piccolo/apps/user/tables.py | 2 +- piccolo/columns/base.py | 5 +- piccolo/columns/column_types.py | 46 +++++++++---------- piccolo/columns/combination.py | 6 +-- piccolo/columns/m2m.py | 2 +- piccolo/columns/readable.py | 4 +- piccolo/conf/apps.py | 24 ++++++---- piccolo/engine/postgres.py | 12 +++-- piccolo/engine/sqlite.py | 10 ++-- piccolo/query/base.py | 44 ++++++++---------- piccolo/query/methods/objects.py | 4 +- piccolo/query/methods/select.py | 4 +- piccolo/query/methods/update.py | 6 ++- piccolo/query/mixins.py | 11 +++-- piccolo/table.py | 33 ++++++++----- piccolo/table_reflection.py | 5 +- piccolo/utils/encoding.py | 5 +- piccolo/utils/lazy_loader.py | 6 +-- piccolo/utils/printing.py | 4 +- piccolo/utils/pydantic.py | 6 +-- piccolo/utils/sql_values.py | 5 +- setup.py | 15 +++--- 33 files changed, 187 insertions(+), 201 deletions(-) diff --git a/piccolo/apps/app/commands/new.py b/piccolo/apps/app/commands/new.py index 775da5240..4badf00e2 100644 --- a/piccolo/apps/app/commands/new.py +++ b/piccolo/apps/app/commands/new.py @@ -56,7 +56,7 @@ def new_app(app_name: str, root: str = "."): for filename, context in templates.items(): with open(os.path.join(app_root, filename), "w") as f: - template = JINJA_ENV.get_template(filename + ".jinja") + template = JINJA_ENV.get_template(f"{filename}.jinja") file_contents = template.render(**context) file_contents = black.format_str( file_contents, mode=black.FileMode(line_length=80) diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index 8e66e0410..390b3f5b3 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -96,7 +96,7 @@ def new(root: str = ".", name: str = "piccolo_project"): ) except Exception as exception: print(f"Problem processing {output_file_name}") - raise exception + raise exception from exception with open( os.path.join(output_dir_path, output_file_name), "w" diff --git a/piccolo/apps/fixtures/commands/shared.py b/piccolo/apps/fixtures/commands/shared.py index c2e2c39f8..9290605a7 100644 --- a/piccolo/apps/fixtures/commands/shared.py +++ b/piccolo/apps/fixtures/commands/shared.py @@ -50,8 +50,4 @@ def create_pydantic_fixture_model(fixture_configs: t.List[FixtureConfig]): columns[fixture_config.app_name] = (app_model, ...) - model: t.Type[pydantic.BaseModel] = pydantic.create_model( - "FixtureModel", **columns - ) - - return model + return pydantic.create_model("FixtureModel", **columns) diff --git a/piccolo/apps/migrations/auto/diffable_table.py b/piccolo/apps/migrations/auto/diffable_table.py index e73065ede..18a7c2809 100644 --- a/piccolo/apps/migrations/auto/diffable_table.py +++ b/piccolo/apps/migrations/auto/diffable_table.py @@ -210,12 +210,10 @@ def to_table_class(self) -> t.Type[Table]: """ Converts the DiffableTable into a Table subclass. """ - _Table: t.Type[Table] = create_table_class( + return create_table_class( class_name=self.class_name, class_kwargs={"tablename": self.tablename}, class_members={ column._meta.name: column for column in self.columns }, ) - - return _Table diff --git a/piccolo/apps/migrations/auto/migration_manager.py b/piccolo/apps/migrations/auto/migration_manager.py index 405a9ef3b..3f1a591f1 100644 --- a/piccolo/apps/migrations/auto/migration_manager.py +++ b/piccolo/apps/migrations/auto/migration_manager.py @@ -193,7 +193,7 @@ def add_column( db_column_name: t.Optional[str] = None, column_class_name: str = "", column_class: t.Optional[t.Type[Column]] = None, - params: t.Dict[str, t.Any] = {}, + params: t.Dict[str, t.Any] = None, ): """ Add a new column to the table. @@ -207,6 +207,8 @@ def add_column( A direct reference to a ``Column`` subclass. """ + if params is None: + params = {} column_class = column_class or getattr(column_types, column_class_name) if column_class is None: @@ -267,14 +269,18 @@ def alter_column( tablename: str, column_name: str, db_column_name: t.Optional[str] = None, - params: t.Dict[str, t.Any] = {}, - old_params: t.Dict[str, t.Any] = {}, + params: t.Dict[str, t.Any] = None, + old_params: t.Dict[str, t.Any] = None, column_class: t.Optional[t.Type[Column]] = None, old_column_class: t.Optional[t.Type[Column]] = None, ): """ All possible alterations aren't currently supported. """ + if params is None: + params = {} + if old_params is None: + old_params = {} self.alter_columns.append( AlterColumn( table_class_name=table_class_name, @@ -521,16 +527,15 @@ async def _run_alter_columns(self, backwards=False): ).run() async def _run_drop_tables(self, backwards=False): - if backwards: - for diffable_table in self.drop_tables: + for diffable_table in self.drop_tables: + if backwards: _Table = await self.get_table_from_snaphot( table_class_name=diffable_table.class_name, app_name=self.app_name, offset=-1, ) await _Table.create_table().run() - else: - for diffable_table in self.drop_tables: + else: await ( diffable_table.to_table_class().alter().drop_table().run() ) diff --git a/piccolo/apps/migrations/auto/schema_differ.py b/piccolo/apps/migrations/auto/schema_differ.py index 71c4cc508..97dfd7d5c 100644 --- a/piccolo/apps/migrations/auto/schema_differ.py +++ b/piccolo/apps/migrations/auto/schema_differ.py @@ -40,10 +40,7 @@ def renamed_from(self, new_class_name: str) -> t.Optional[str]: rename = [ i for i in self.rename_tables if i.new_class_name == new_class_name ] - if len(rename) > 0: - return rename[0].old_class_name - else: - return None + return rename[0].old_class_name if rename else None @dataclass @@ -211,14 +208,10 @@ def check_renamed_columns(self) -> RenameColumnCollection: if drop_column.column_name in used_drop_column_names: continue - user_response = ( - self.auto_input - if self.auto_input - else input( - f"Did you rename the `{drop_column.db_column_name}` " # noqa: E501 - f"column to `{add_column.db_column_name}` on the " - f"`{ add_column.table_class_name }` table? (y/N)" - ) + user_response = self.auto_input or input( + f"Did you rename the `{drop_column.db_column_name}` " # noqa: E501 + f"column to `{add_column.db_column_name}` on the " + f"`{ add_column.table_class_name }` table? (y/N)" ) if user_response.lower() == "y": used_drop_column_names.append(drop_column.column_name) diff --git a/piccolo/apps/migrations/auto/serialisation.py b/piccolo/apps/migrations/auto/serialisation.py index e1de712be..ea2f06248 100644 --- a/piccolo/apps/migrations/auto/serialisation.py +++ b/piccolo/apps/migrations/auto/serialisation.py @@ -225,11 +225,7 @@ def __lt__(self, other): return repr(self) < repr(other) def warn_if_is_conflicting_with_global_name(self): - if self.target is None: - name = self.module - else: - name = self.target - + name = self.module if self.target is None else self.target if name == self.expect_conflict_with_global_name: return diff --git a/piccolo/apps/migrations/auto/serialisation_legacy.py b/piccolo/apps/migrations/auto/serialisation_legacy.py index b1e776054..b212943bc 100644 --- a/piccolo/apps/migrations/auto/serialisation_legacy.py +++ b/piccolo/apps/migrations/auto/serialisation_legacy.py @@ -5,7 +5,7 @@ from piccolo.columns.column_types import OnDelete, OnUpdate from piccolo.columns.defaults.timestamp import TimestampNow -from piccolo.table import Table, create_table_class +from piccolo.table import create_table_class def deserialise_legacy_params(name: str, value: str) -> t.Any: @@ -26,25 +26,10 @@ def deserialise_legacy_params(name: str, value: str) -> t.Any: "`SomeClassName` or `SomeClassName|some_table_name`." ) - _Table: t.Type[Table] = create_table_class( + return create_table_class( class_name=class_name, class_kwargs={"tablename": tablename} if tablename else {}, ) - return _Table - - ########################################################################### - - if name == "on_delete": - enum_name, item_name = value.split(".") - if enum_name == "OnDelete": - return getattr(OnDelete, item_name) - - ########################################################################### - - elif name == "on_update": - enum_name, item_name = value.split(".") - if enum_name == "OnUpdate": - return getattr(OnUpdate, item_name) ########################################################################### @@ -58,4 +43,14 @@ def deserialise_legacy_params(name: str, value: str) -> t.Any: else: return _value + elif name == "on_delete": + enum_name, item_name = value.split(".") + if enum_name == "OnDelete": + return getattr(OnDelete, item_name) + + elif name == "on_update": + enum_name, item_name = value.split(".") + if enum_name == "OnUpdate": + return getattr(OnUpdate, item_name) + return value diff --git a/piccolo/apps/migrations/commands/backwards.py b/piccolo/apps/migrations/commands/backwards.py index b7501a0e0..107cb9635 100644 --- a/piccolo/apps/migrations/commands/backwards.py +++ b/piccolo/apps/migrations/commands/backwards.py @@ -131,19 +131,18 @@ async def run_backwards( ) ) - if _continue in "yY": - for _app_name in sorted_app_names: - print(f"\n{_app_name.upper():^64}") - print("-" * 64) - manager = BackwardsMigrationManager( - app_name=_app_name, - migration_id="all", - auto_agree=auto_agree, - ) - await manager.run() - return MigrationResult(success=True) - else: + if _continue not in "yY": return MigrationResult(success=False, message="user cancelled") + for _app_name in sorted_app_names: + print(f"\n{_app_name.upper():^64}") + print("-" * 64) + manager = BackwardsMigrationManager( + app_name=_app_name, + migration_id="all", + auto_agree=auto_agree, + ) + await manager.run() + return MigrationResult(success=True) else: manager = BackwardsMigrationManager( app_name=app_name, diff --git a/piccolo/apps/migrations/commands/base.py b/piccolo/apps/migrations/commands/base.py index fce1b37c1..2a715070b 100644 --- a/piccolo/apps/migrations/commands/base.py +++ b/piccolo/apps/migrations/commands/base.py @@ -110,7 +110,7 @@ async def get_migration_managers( "Positive offset values aren't currently supported" ) elif offset < 0: - return migration_managers[0:offset] + return migration_managers[:offset] else: return migration_managers diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index e0505cf81..ed7fbf9c0 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -209,7 +209,7 @@ def __post_init__(self): groups = match.groupdict() self.column_name = groups["column_name"].lstrip('"').rstrip('"') - self.unique = True if "unique" in groups else False + self.unique = "unique" in groups self.method = INDEX_METHOD_MAP[groups["method"]] self.warnings = [] @@ -224,11 +224,9 @@ class TableIndexes: indexes: t.List[Index] def get_column_index(self, column_name: str) -> t.Optional[Index]: - for i in self.indexes: - if i.column_name == column_name: - return i - - return None + return next( + (i for i in self.indexes if i.column_name == column_name), None + ) def get_warnings(self) -> t.List[str]: return list( @@ -579,7 +577,7 @@ async def get_tablenames( A list of tablenames for the given schema. """ - tablenames: t.List[str] = [ + return [ i["tablename"] for i in await table_class.raw( ( @@ -589,7 +587,6 @@ async def get_tablenames( schema_name, ).run() ] - return tablenames async def get_table_schema( @@ -647,9 +644,7 @@ async def get_foreign_key_reference( def get_table_name(name: str, schema: str) -> str: - if schema == "public": - return name - return f"{schema}.{name}" + return name if schema == "public" else f"{schema}.{name}" async def create_table_class_from_db( @@ -898,7 +893,7 @@ async def generate(schema_name: str = "public"): output.append('"""') if output_schema.index_warnings: - warning_str = "\n".join(i for i in set(output_schema.index_warnings)) + warning_str = "\n".join(set(output_schema.index_warnings)) output.append('"""') output.append("WARNING: Unable to parse the following indexes:") diff --git a/piccolo/apps/user/tables.py b/piccolo/apps/user/tables.py index 10fb4b1e4..7df949cad 100644 --- a/piccolo/apps/user/tables.py +++ b/piccolo/apps/user/tables.py @@ -114,7 +114,7 @@ def hash_password( logger.warning("Excessively long password provided.") raise ValueError("The password is too long.") - if salt == "": + if not salt: salt = cls.get_salt() hashed = hashlib.pbkdf2_hmac( "sha256", diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 805dd8c97..3aa13fa4e 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -287,10 +287,7 @@ def get_full_name( else: alias = f"{self.call_chain[-1]._meta.table_alias}.{self.name}" - if just_alias: - return alias - else: - return f'{alias} AS "{column_name}"' + return alias if just_alias else f'{alias} AS "{column_name}"' ########################################################################### diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 0c513d9f5..c8f7eda04 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -141,10 +141,7 @@ def get_querystring( "Adding values across joins isn't currently supported." ) column_name = column._meta.db_column_name - if reverse: - return QueryString(f"{column_name} {operator} {column_name}") - else: - return QueryString(f"{column_name} {operator} {column_name}") + return QueryString(f"{column_name} {operator} {column_name}") elif isinstance(value, (int, float)): if reverse: return QueryString(f"{{}} {operator} {column_name}", value) @@ -202,10 +199,7 @@ def __init__( @property def column_type(self): - if self.length: - return f"VARCHAR({self.length})" - else: - return "VARCHAR" + return f"VARCHAR({self.length})" if self.length else "VARCHAR" def __add__(self, value: t.Union[str, Varchar, Text]) -> QueryString: engine_type = self._meta.table._meta.db.engine_type @@ -377,10 +371,10 @@ def __init__(self, default: UUIDArg = UUID4(), **kwargs) -> None: if isinstance(default, str): try: default = uuid.UUID(default) - except ValueError: + except ValueError as e: raise ValueError( "The default is a string, but not a valid uuid." - ) + ) from e self.default = default kwargs.update({"default": default}) @@ -1218,9 +1212,8 @@ def __init__( "with the first value being the precision, and the second " "value being the scale." ) - else: - if digits is not None: - raise ValueError("The digits argument should be a tuple.") + elif digits is not None: + raise ValueError("The digits argument should be a tuple.") self._validate_default(default, (decimal.Decimal, None)) @@ -1700,7 +1693,7 @@ def copy(self) -> ForeignKey: return column def all_columns( - self, exclude: t.List[t.Union[Column, str]] = [] + self, exclude: t.List[t.Union[Column, str]] = None ) -> t.List[Column]: """ Allow a user to access all of the columns on the related table. This is @@ -1736,6 +1729,8 @@ def all_columns( instance. For example ``['id']`` or ``[Band.manager.id]``. """ + if exclude is None: + exclude = [] _fk_meta = object.__getattribute__(self, "_foreign_key_meta") excluded_column_names = [ @@ -1749,7 +1744,7 @@ def all_columns( ] def all_related( - self, exclude: t.List[t.Union[ForeignKey, str]] = [] + self, exclude: t.List[t.Union[ForeignKey, str]] = None ) -> t.List[ForeignKey]: """ Returns each ``ForeignKey`` column on the related table. This is @@ -1787,6 +1782,8 @@ class Tour(Table): ``[Tour.concert.band_1]``. """ + if exclude is None: + exclude = [] _fk_meta: ForeignKeyMeta = object.__getattribute__( self, "_foreign_key_meta" ) @@ -1994,16 +1991,17 @@ def get_select_string(self, engine_type: str, just_alias=False) -> str: select_string = self._meta.get_full_name( just_alias=just_alias, include_quotes=True ) - if self.json_operator is None: - if self.alias is None: - return select_string - else: - return f"{select_string} AS {self.alias}" + if self.json_operator is not None: + return ( + f"{select_string} {self.json_operator}" + if self.alias is None + else f"{select_string} {self.json_operator} AS {self.alias}" + ) + + if self.alias is None: + return select_string else: - if self.alias is None: - return f"{select_string} {self.json_operator}" - else: - return f"{select_string} {self.json_operator} AS {self.alias}" + return f"{select_string} AS {self.alias}" def eq(self, value) -> Where: """ diff --git a/piccolo/columns/combination.py b/piccolo/columns/combination.py index 82668c3d8..6162aac72 100644 --- a/piccolo/columns/combination.py +++ b/piccolo/columns/combination.py @@ -128,11 +128,7 @@ def __init__( """ self.column = column - if value == UNDEFINED: - self.value = value - else: - self.value = self.clean_value(value) - + self.value = value if value == UNDEFINED else self.clean_value(value) if values == UNDEFINED: self.values = values else: diff --git a/piccolo/columns/m2m.py b/piccolo/columns/m2m.py index 7d7fe25b4..57f403073 100644 --- a/piccolo/columns/m2m.py +++ b/piccolo/columns/m2m.py @@ -409,7 +409,7 @@ def __call__( :param load_json: If ``True``, any JSON strings are loaded as Python objects. """ - if len(columns) == 0: + if not columns: columns = tuple(self._meta.secondary_table._meta.columns) if as_list and len(columns) != 1: diff --git a/piccolo/columns/readable.py b/piccolo/columns/readable.py index 7780ac05b..651f5b1a5 100644 --- a/piccolo/columns/readable.py +++ b/piccolo/columns/readable.py @@ -44,7 +44,7 @@ def postgres_string(self) -> str: def get_select_string(self, engine_type: str, just_alias=False) -> str: try: return getattr(self, f"{engine_type}_string") - except AttributeError: + except AttributeError as e: raise ValueError( f"Unrecognised engine_type - received {engine_type}" - ) + ) from e diff --git a/piccolo/conf/apps.py b/piccolo/conf/apps.py index 346de7678..81ef43c71 100644 --- a/piccolo/conf/apps.py +++ b/piccolo/conf/apps.py @@ -31,8 +31,8 @@ class PiccoloAppModule(ModuleType): def table_finder( modules: t.Sequence[str], - include_tags: t.Sequence[str] = ["__all__"], - exclude_tags: t.Sequence[str] = [], + include_tags: t.Sequence[str] = None, + exclude_tags: t.Sequence[str] = None, exclude_imported: bool = False, ) -> t.List[t.Type[Table]]: """ @@ -68,6 +68,10 @@ class Task(Table): # included creator = ForeignKey(BaseUser) """ # noqa: E501 + if include_tags is None: + include_tags = ["__all__"] + if exclude_tags is None: + exclude_tags = [] if isinstance(modules, str): # Guard against the user just entering a string, for example # 'blog.tables', instead of ['blog.tables']. @@ -80,7 +84,7 @@ class Task(Table): # included module = import_module(module_path) except ImportError as exception: print(f"Unable to import {module_path}") - raise exception + raise exception from exception object_names = [i for i in dir(module) if not i.startswith("_")] @@ -207,7 +211,7 @@ def __post_init__(self): app_config: AppConfig = getattr(app_conf_module, "APP_CONFIG") except (ImportError, AttributeError) as e: if app.endswith(".piccolo_app"): - raise e + raise e from e app += ".piccolo_app" app_conf_module = import_module(app) app_config: AppConfig = getattr(app_conf_module, "APP_CONFIG") @@ -319,8 +323,10 @@ def _import_app_modules( config_module = t.cast( PiccoloAppModule, import_module(config_module_path) ) - except ImportError: - raise Exception(f"Unable to import {config_module_path}") + except ImportError as e: + raise Exception( + f"Unable to import {config_module_path}" + ) from e app_config: AppConfig = getattr(config_module, "APP_CONFIG") dependency_config_modules = self._import_app_modules( app_config.migration_dependencies @@ -367,14 +373,14 @@ def get_piccolo_conf_module( raise ModuleNotFoundError( "PostgreSQL driver not found. " "Try running `pip install 'piccolo[postgres]'`" - ) + ) from exc elif str(exc) == "No module named 'aiosqlite'": raise ModuleNotFoundError( "SQLite driver not found. " "Try running `pip install 'piccolo[sqlite]'`" - ) + ) from exc else: - raise exc + raise exc from exc else: return module diff --git a/piccolo/engine/postgres.py b/piccolo/engine/postgres.py index c9bcb95e6..56aad2b23 100644 --- a/piccolo/engine/postgres.py +++ b/piccolo/engine/postgres.py @@ -242,9 +242,11 @@ class PostgresEngine(Engine): def __init__( self, config: t.Dict[str, t.Any], - extensions: t.Sequence[str] = ["uuid-ossp"], + extensions: t.Sequence[str] = None, log_queries: bool = False, ) -> None: + if extensions is None: + extensions = ["uuid-ossp"] self.config = config self.extensions = extensions self.log_queries = log_queries @@ -361,7 +363,9 @@ async def batch(self, query: Query, batch_size: int = 100) -> AsyncBatch: ########################################################################### - async def _run_in_pool(self, query: str, args: t.Sequence[t.Any] = []): + async def _run_in_pool(self, query: str, args: t.Sequence[t.Any] = None): + if args is None: + args = [] if not self.pool: raise ValueError("A pool isn't currently running.") @@ -371,8 +375,10 @@ async def _run_in_pool(self, query: str, args: t.Sequence[t.Any] = []): return response async def _run_in_new_connection( - self, query: str, args: t.Sequence[t.Any] = [] + self, query: str, args: t.Sequence[t.Any] = None ): + if args is None: + args = [] connection = await self.get_new_connection() results = await connection.fetch(query, *args) await connection.close() diff --git a/piccolo/engine/sqlite.py b/piccolo/engine/sqlite.py index d363f6725..0cad6e574 100644 --- a/piccolo/engine/sqlite.py +++ b/piccolo/engine/sqlite.py @@ -280,7 +280,7 @@ async def run(self): await connection.execute("ROLLBACK") await connection.close() self.queries = [] - raise exception + raise exception from exception else: await connection.execute("COMMIT") await connection.close() @@ -454,10 +454,12 @@ async def _get_inserted_pk(self, cursor, table: t.Type[Table]) -> t.Any: async def _run_in_new_connection( self, query: str, - args: t.List[t.Any] = [], + args: t.List[t.Any] = None, query_type: str = "generic", table: t.Optional[t.Type[Table]] = None, ): + if args is None: + args = [] async with aiosqlite.connect(**self.connection_kwargs) as connection: await connection.execute("PRAGMA foreign_keys = 1") @@ -476,13 +478,15 @@ async def _run_in_existing_connection( self, connection, query: str, - args: t.List[t.Any] = [], + args: t.List[t.Any] = None, query_type: str = "generic", table: t.Optional[t.Type[Table]] = None, ): """ This is used when a transaction is currently active. """ + if args is None: + args = [] await connection.execute("PRAGMA foreign_keys = 1") connection.row_factory = dict_factory diff --git a/piccolo/query/base.py b/piccolo/query/base.py index 509bc8bd8..57f592ea0 100644 --- a/piccolo/query/base.py +++ b/piccolo/query/base.py @@ -132,9 +132,7 @@ async def _process_results(self, results): # noqa: C901 self.table(**columns, exists_in_db=True) for columns in raw ] - elif raw is None: - pass - else: + elif raw is not None: if output._output.nested: raw = make_nested_object(raw, self.table) else: @@ -143,15 +141,12 @@ async def _process_results(self, results): # noqa: C901 if output._output.as_list: if len(raw) == 0: return [] + if len(raw[0].keys()) != 1: + raise ValueError( + "Each row returned more than one value" + ) else: - if len(raw[0].keys()) != 1: - raise ValueError( - "Each row returned more than one value" - ) - else: - raw = list( - itertools.chain(*[j.values() for j in raw]) - ) + raw = list(itertools.chain(*[j.values() for j in raw])) if output._output.as_json: raw = dump_json(raw) @@ -205,10 +200,9 @@ def run_sync(self, timed=False, *args, **kwargs): """ coroutine = self.run(*args, **kwargs, in_pool=False) - if timed: - with Timer(): - return run_sync(coroutine) - else: + if not timed: + return run_sync(coroutine) + with Timer(): return run_sync(coroutine) async def response_handler(self, response): @@ -408,13 +402,12 @@ async def run(self, in_pool=True): if len(self.ddl) == 1: return await engine.run_ddl(self.ddl[0], in_pool=in_pool) - else: - responses = [] - # TODO - run in a transaction - for ddl in self.ddl: - response = await engine.run_ddl(ddl, in_pool=in_pool) - responses.append(response) - return responses + responses = [] + # TODO - run in a transaction + for ddl in self.ddl: + response = await engine.run_ddl(ddl, in_pool=in_pool) + responses.append(response) + return responses def run_sync(self, timed=False, *args, **kwargs): """ @@ -422,10 +415,9 @@ def run_sync(self, timed=False, *args, **kwargs): """ coroutine = self.run(*args, **kwargs, in_pool=False) - if timed: - with Timer(): - return run_sync(coroutine) - else: + if not timed: + return run_sync(coroutine) + with Timer(): return run_sync(coroutine) def __str__(self) -> str: diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index e814da129..050edd169 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -176,8 +176,10 @@ def offset(self, number: int) -> Objects: def get_or_create( self, where: Combinable, - defaults: t.Dict[t.Union[Column, str], t.Any] = {}, + defaults: t.Dict[t.Union[Column, str], t.Any] = None, ): + if defaults is None: + defaults = {} return GetOrCreate(query=self, where=where, defaults=defaults) def create(self, **columns: t.Any): diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index b9024f48a..9b05f85b8 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -227,10 +227,12 @@ class Select(Query): def __init__( self, table: t.Type[Table], - columns_list: t.Sequence[t.Union[Selectable, str]] = [], + columns_list: t.Sequence[t.Union[Selectable, str]] = None, exclude_secrets: bool = False, **kwargs, ): + if columns_list is None: + columns_list = [] super().__init__(table, **kwargs) self.exclude_secrets = exclude_secrets diff --git a/piccolo/query/methods/update.py b/piccolo/query/methods/update.py index a6b0fbce0..e9c9dbadb 100644 --- a/piccolo/query/methods/update.py +++ b/piccolo/query/methods/update.py @@ -27,8 +27,10 @@ def __init__(self, table: t.Type[Table], force: bool = False, **kwargs): self.where_delegate = WhereDelegate() def values( - self, values: t.Dict[t.Union[Column, str], t.Any] = {}, **kwargs + self, values: t.Dict[t.Union[Column, str], t.Any] = None, **kwargs ) -> Update: + if values is None: + values = {} values = dict(values, **kwargs) self.values_delegate.values(values) return self @@ -62,7 +64,7 @@ def default_querystrings(self) -> t.Sequence[QueryString]: for col, _ in self.values_delegate._values.items() ) - query = f"UPDATE {self.table._meta.tablename} SET " + columns_str + query = f"UPDATE {self.table._meta.tablename} SET {columns_str}" querystring = QueryString( query, *self.values_delegate.get_sql_values() diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index 499bf6cf2..af8197ced 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -295,11 +295,12 @@ def columns(self, *columns: t.Union[Selectable, t.List[Selectable]]): self.selected_columns = combined def remove_secret_columns(self): - non_secret = [] - for i in self.selected_columns: - if isinstance(i, Column) and i._meta.secret: - continue - non_secret.append(i) + non_secret = [ + i + for i in self.selected_columns + if not isinstance(i, Column) or not i._meta.secret + ] + self.selected_columns = non_secret diff --git a/piccolo/table.py b/piccolo/table.py index 53d6c62c9..b0e732271 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -81,10 +81,7 @@ class TableMeta: @property def foreign_key_references(self) -> t.List[ForeignKey]: - foreign_keys: t.List[ForeignKey] = [] - for reference in self._foreign_key_references: - foreign_keys.append(reference) - + foreign_keys: t.List[ForeignKey] = list(self._foreign_key_references) lazy_column_references = LAZY_COLUMN_REFERENCES.for_tablename( tablename=self.tablename ) @@ -126,10 +123,10 @@ def get_column_by_name(self, name: str) -> Column: for reference_name in components[1:]: try: column_object = getattr(column_object, reference_name) - except AttributeError: + except AttributeError as e: raise ValueError( f"Unable to find column - {reference_name}" - ) + ) from e return column_object @@ -152,9 +149,9 @@ def __init_subclass__( cls, tablename: t.Optional[str] = None, db: t.Optional[Engine] = None, - tags: t.List[str] = [], + tags: t.List[str] = None, help_text: t.Optional[str] = None, - ): + ): # sourcery no-metrics """ Automatically populate the _meta, which includes the tablename, and columns. @@ -174,6 +171,8 @@ def __init_subclass__( Admin for tooltips. """ + if tags is None: + tags = [] tablename = tablename or _camel_to_snake(cls.__name__) if tablename in PROTECTED_TABLENAMES: @@ -665,7 +664,7 @@ def __repr__(self) -> str: @classmethod def all_related( - cls, exclude: t.List[t.Union[str, ForeignKey]] = [] + cls, exclude: t.List[t.Union[str, ForeignKey]] = None ) -> t.List[Column]: """ Used in conjunction with ``objects`` queries. Just as we can use @@ -701,6 +700,8 @@ def all_related( You can request all columns, except these. """ + if exclude is None: + exclude = [] excluded_column_names = [ i._meta.name if isinstance(i, ForeignKey) else i for i in exclude ] @@ -713,7 +714,7 @@ def all_related( @classmethod def all_columns( - cls, exclude: t.List[t.Union[str, Column]] = [] + cls, exclude: t.List[t.Union[str, Column]] = None ) -> t.List[Column]: """ Used in conjunction with ``select`` queries. Just as we can use @@ -735,6 +736,8 @@ def all_columns( You can request all columns, except these. """ + if exclude is None: + exclude = [] excluded_column_names = [ i._meta.name if isinstance(i, Column) else i for i in exclude ] @@ -979,7 +982,7 @@ def table_exists(cls) -> TableExists: @classmethod def update( cls, - values: t.Dict[t.Union[Column, str], t.Any] = {}, + values: t.Dict[t.Union[Column, str], t.Any] = None, force: bool = False, **kwargs, ) -> Update: @@ -1013,6 +1016,8 @@ def update( ``where`` clause, to prevent accidental mass overriding of data. """ + if values is None: + values = {} values = dict(values, **kwargs) return Update(table=cls, force=force).values(values) @@ -1078,7 +1083,9 @@ def _get_index_name(cls, column_names: t.List[str]) -> str: ########################################################################### @classmethod - def _table_str(cls, abbreviated=False, excluded_params: t.List[str] = []): + def _table_str( + cls, abbreviated=False, excluded_params: t.List[str] = None + ): """ Returns a basic string representation of the table and its columns. Used by the playground. @@ -1091,6 +1098,8 @@ def _table_str(cls, abbreviated=False, excluded_params: t.List[str] = []): `['index_method']`, if we want to show all kwargs but index_method. """ + if excluded_params is None: + excluded_params = [] spacer = "\n " columns = [] for col in cls._meta.columns: diff --git a/piccolo/table_reflection.py b/piccolo/table_reflection.py index 13f4d5c0f..69c0bb7e2 100644 --- a/piccolo/table_reflection.py +++ b/piccolo/table_reflection.py @@ -189,10 +189,7 @@ def _add_to_schema_tables(self, schema_name: str, table_name: str) -> None: @staticmethod def _get_table_name(name: str, schema: str): - if schema == "public": - return name - else: - return schema + "." + name + return name if schema == "public" else f"{schema}.{name}" def __repr__(self): return f"{[tablename for tablename, _ in self.tables.items()]}" diff --git a/piccolo/utils/encoding.py b/piccolo/utils/encoding.py index 3feda117f..48a131dc5 100644 --- a/piccolo/utils/encoding.py +++ b/piccolo/utils/encoding.py @@ -28,7 +28,4 @@ def dump_json(data: t.Any, pretty: bool = False) -> str: def load_json(data: str) -> t.Any: - if ORJSON: - return orjson.loads(data) - else: - return json.loads(data) + return orjson.loads(data) if ORJSON else json.loads(data) diff --git a/piccolo/utils/lazy_loader.py b/piccolo/utils/lazy_loader.py index 51469b06e..5b7211ce3 100644 --- a/piccolo/utils/lazy_loader.py +++ b/piccolo/utils/lazy_loader.py @@ -39,14 +39,14 @@ def _load(self) -> types.ModuleType: raise ModuleNotFoundError( "PostgreSQL driver not found. " "Try running `pip install 'piccolo[postgres]'`" - ) + ) from exc elif str(exc) == "No module named 'aiosqlite'": raise ModuleNotFoundError( "SQLite driver not found. " "Try running `pip install 'piccolo[sqlite]'`" - ) + ) from exc else: - raise exc + raise exc from exc def __getattr__(self, item) -> t.Any: module = self._load() diff --git a/piccolo/utils/printing.py b/piccolo/utils/printing.py index 84c8926d0..258594d71 100644 --- a/piccolo/utils/printing.py +++ b/piccolo/utils/printing.py @@ -3,6 +3,6 @@ def get_fixed_length_string(string: str, length=20) -> str: Add spacing to the end of the string so it's a fixed length. """ if len(string) > length: - return string[: length - 3] + "..." - spacing = "".join(" " for i in range(length - len(string))) + return f"{string[: length - 3]}..." + spacing = "".join(" " for _ in range(length - len(string))) return f"{string}{spacing}" diff --git a/piccolo/utils/pydantic.py b/piccolo/utils/pydantic.py index 47ea1caa2..216807539 100644 --- a/piccolo/utils/pydantic.py +++ b/piccolo/utils/pydantic.py @@ -39,8 +39,8 @@ class Config(pydantic.BaseConfig): def pydantic_json_validator(cls, value): try: load_json(value) - except json.JSONDecodeError: - raise ValueError("Unable to parse the JSON.") + except json.JSONDecodeError as e: + raise ValueError("Unable to parse the JSON.") from e else: return value @@ -302,7 +302,7 @@ class CustomConfig(Config): **schema_extra_kwargs, } - model = pydantic.create_model( + model = pydantic.create_model( # type: ignore model_name, __config__=CustomConfig, __validators__=validators, diff --git a/piccolo/utils/sql_values.py b/piccolo/utils/sql_values.py index b61be6fd1..6c13da99c 100644 --- a/piccolo/utils/sql_values.py +++ b/piccolo/utils/sql_values.py @@ -34,9 +34,6 @@ def convert_to_sql_value(value: t.Any, column: Column) -> t.Any: elif isinstance(value, Enum): return value.value elif isinstance(column, (JSON, JSONB)) and not isinstance(value, str): - if value is None: - return None - else: - return dump_json(value) + return None if value is None else dump_json(value) else: return value diff --git a/setup.py b/setup.py index 295b97ef8..46fef6b3e 100644 --- a/setup.py +++ b/setup.py @@ -36,12 +36,15 @@ def extras_require() -> t.Dict[str, t.List[str]]: """ Parse requirements in requirements/extras directory """ - extra_requirements = {extra: parse_requirement( - os.path.join("extras", extra + ".txt") - ) for extra in extras} - extra_requirements["all"] = [ - i for i in itertools.chain.from_iterable(extra_requirements.values()) - ] + extra_requirements = { + extra: parse_requirement(os.path.join("extras", f'{extra}.txt')) + for extra in extras + } + + extra_requirements["all"] = list( + itertools.chain.from_iterable(extra_requirements.values()) + ) + return extra_requirements From d04325aeeea87f5079545923eb11b68667f32c95 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 9 Mar 2022 18:32:38 +0000 Subject: [PATCH 285/727] add docs for troubleshooting type warnings (#458) --- docs/src/piccolo/features/index.rst | 2 +- docs/src/piccolo/features/tab_completion.rst | 10 ---- .../features/types_and_tab_completion.rst | 49 +++++++++++++++++++ .../getting_started/what_is_piccolo.rst | 2 +- 4 files changed, 51 insertions(+), 12 deletions(-) delete mode 100644 docs/src/piccolo/features/tab_completion.rst create mode 100644 docs/src/piccolo/features/types_and_tab_completion.rst diff --git a/docs/src/piccolo/features/index.rst b/docs/src/piccolo/features/index.rst index d1d4b015e..c6ff9ebcc 100644 --- a/docs/src/piccolo/features/index.rst +++ b/docs/src/piccolo/features/index.rst @@ -4,7 +4,7 @@ Features .. toctree:: :maxdepth: 1 - ./tab_completion + ./types_and_tab_completion ./supported_databases ./security ./syntax diff --git a/docs/src/piccolo/features/tab_completion.rst b/docs/src/piccolo/features/tab_completion.rst deleted file mode 100644 index c7cff94c6..000000000 --- a/docs/src/piccolo/features/tab_completion.rst +++ /dev/null @@ -1,10 +0,0 @@ -.. _tab_completion: - -Tab Completion -============== - -Piccolo does everything possible to support tab completion. It has been tested -with iPython and VSCode. - -To find out more about how it was done, read `this article about type annotations `_, -and `this article about descriptors `_. diff --git a/docs/src/piccolo/features/types_and_tab_completion.rst b/docs/src/piccolo/features/types_and_tab_completion.rst new file mode 100644 index 000000000..ed7d1158f --- /dev/null +++ b/docs/src/piccolo/features/types_and_tab_completion.rst @@ -0,0 +1,49 @@ +.. _tab_completion: + +Types and Tab Completion +======================== + +Type annotations +---------------- + +The Piccolo codebase uses type annotations extensively. This means it has great +tab completion support in tools like iPython and VSCode. + +It also means it works well with type checkers like Mypy. + +To learn more about how Piccolo achieves this, read this `article about type annotations `_, +and this `article about descriptors `_. + +------------------------------------------------------------------------------- + +Troubleshooting +--------------- + +Here are some issues you may encounter when using Mypy, or another type +checker. + +``id`` column doesn't exist +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you don't explicitly declare a column on your table with ``primary_key=True``, +Piccolo creates a ``Serial`` column for you called ``id``. + +In the following situation, the type checker might complains that ``id`` +doesn't exist: + +.. code-block:: python + + await Band.select(Band.id) + +You can fix this as follows: + +.. code-block:: python + + # tables.py + from piccolo.table import Table + from piccolo.columns.column_types import Serial, Varchar + + + class Band(Table): + id: Serial # Add an annotation + name = Varchar() diff --git a/docs/src/piccolo/getting_started/what_is_piccolo.rst b/docs/src/piccolo/getting_started/what_is_piccolo.rst index 6a20875d3..e187c9946 100644 --- a/docs/src/piccolo/getting_started/what_is_piccolo.rst +++ b/docs/src/piccolo/getting_started/what_is_piccolo.rst @@ -7,7 +7,7 @@ Some of it's stand out features are: * Support for :ref:`sync and async `. * A builtin :ref:`playground `, which makes learning a breeze. -* Great :ref:`tab completion ` - it works great with +* Fully type annotated, with great :ref:`tab completion support ` - it works great with `iPython `_ and `VSCode `_. * Batteries included - a :ref:`User model and authentication `, :ref:`migrations `, an :ref:`admin `, and more. From 930da74c0948b992bf7c6ab097aecd93936e97f4 Mon Sep 17 00:00:00 2001 From: backwardspy Date: Fri, 11 Mar 2022 19:43:23 +0000 Subject: [PATCH 286/727] add array column support to modelbuilder (#459) * add array column support to modelbuilder * switch boolean array to real array for sqlite compat --- piccolo/testing/model_builder.py | 9 ++++++++- tests/testing/test_model_builder.py | 12 ++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/piccolo/testing/model_builder.py b/piccolo/testing/model_builder.py index 72fcc26a2..87ff042f6 100644 --- a/piccolo/testing/model_builder.py +++ b/piccolo/testing/model_builder.py @@ -4,7 +4,7 @@ from decimal import Decimal from uuid import UUID -from piccolo.columns.base import Column +from piccolo.columns import Array, Column from piccolo.table import Table from piccolo.testing.random_builder import RandomBuilder from piccolo.utils.sync import run_sync @@ -139,11 +139,18 @@ def _randomize_attribute(cls, column: Column) -> t.Any: Column class to randomize. """ + random_value: t.Any if column.value_type == Decimal: precision, scale = column._meta.params["digits"] random_value = RandomBuilder.next_float( maximum=10 ** (precision - scale), scale=scale ) + elif column.value_type == list: + length = RandomBuilder.next_int(maximum=10) + base_type = t.cast(Array, column).base_column.value_type + random_value = [ + cls.__DEFAULT_MAPPER[base_type]() for _ in range(length) + ] elif column._meta.choices: random_value = RandomBuilder.next_enum(column._meta.choices) else: diff --git a/tests/testing/test_model_builder.py b/tests/testing/test_model_builder.py index 7806bb07a..9918399b0 100644 --- a/tests/testing/test_model_builder.py +++ b/tests/testing/test_model_builder.py @@ -1,6 +1,8 @@ import asyncio import unittest +from piccolo.columns import Array, Integer, Real, Varchar +from piccolo.table import Table from piccolo.testing.model_builder import ModelBuilder from tests.example_apps.music.tables import ( Band, @@ -14,6 +16,12 @@ ) +class TableWithArrayField(Table): + strings = Array(Varchar(30)) + integers = Array(Integer()) + floats = Array(Real()) + + class TestModelBuilder(unittest.TestCase): @classmethod def setUpClass(cls): @@ -27,12 +35,14 @@ def setUpClass(cls): Venue, Concert, Ticket, + TableWithArrayField, ): table_class.create_table().run_sync() @classmethod def tearDownClass(cls) -> None: for table_class in ( + TableWithArrayField, Ticket, Concert, Venue, @@ -52,12 +62,14 @@ async def build_model(model): asyncio.run(build_model(Ticket)) asyncio.run(build_model(Poster)) asyncio.run(build_model(RecordingStudio)) + asyncio.run(build_model(TableWithArrayField)) def test_model_builder_sync(self): ModelBuilder.build_sync(Manager) ModelBuilder.build_sync(Ticket) ModelBuilder.build_sync(Poster) ModelBuilder.build_sync(RecordingStudio) + ModelBuilder.build_sync(TableWithArrayField) def test_model_builder_with_choices(self): shirt = ModelBuilder.build_sync(Shirt) From a529cdcf3303b1f16c9c6da5a64e4ea790de5255 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 11 Mar 2022 20:12:44 +0000 Subject: [PATCH 287/727] bumped version --- CHANGES.rst | 14 ++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 21f05d429..a23fcd63c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,20 @@ Changes ======= +0.71.0 +------ + +The ``ModelBuilder`` class, which is used to generate mock data in tests, now +supports ``Array`` columns. Courtesy @backwardspy. + +Lots of internal code optimisations and clean up. Courtesy @yezz123. + +Added docs for troubleshooting common MyPy errors. + +Also thanks to @adriangb for helping us with our dependency issues. + +------------------------------------------------------------------------------- + 0.70.1 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 9ec35a636..24cbbe8a7 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.70.1" +__VERSION__ = "0.71.0" From 0f1205f57390b939a45851228afc094699771935 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 12 Mar 2022 18:23:20 +0000 Subject: [PATCH 288/727] limit GitHub Action workflow duration (#460) --- .github/workflows/tests.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index c85c11f51..f92e72fc1 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -9,6 +9,7 @@ on: jobs: linters: runs-on: ubuntu-latest + timeout-minutes: 30 strategy: matrix: python-version: ["3.7", "3.8", "3.9", "3.10"] @@ -30,6 +31,7 @@ jobs: integration: runs-on: ubuntu-latest + timeout-minutes: 30 strategy: matrix: # These tests are slow, so we only run on the latest Python @@ -76,6 +78,7 @@ jobs: postgres: runs-on: ubuntu-latest + timeout-minutes: 30 strategy: matrix: python-version: ["3.7", "3.8", "3.9", "3.10"] @@ -131,6 +134,7 @@ jobs: sqlite: runs-on: ubuntu-latest + timeout-minutes: 30 strategy: matrix: python-version: ["3.7", "3.8", "3.9", "3.10"] From 4ada3ae5eea99c4f9be770096c2fa2bdf81863d0 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 13 Mar 2022 08:30:46 +0000 Subject: [PATCH 289/727] fix `ModelBuilder` bug with `ignore_missing` (#462) --- piccolo/testing/model_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piccolo/testing/model_builder.py b/piccolo/testing/model_builder.py index 87ff042f6..8d3fabb6b 100644 --- a/piccolo/testing/model_builder.py +++ b/piccolo/testing/model_builder.py @@ -94,7 +94,7 @@ async def _build( minimal: bool = False, persist: bool = True, ) -> Table: - model = table_class() + model = table_class(ignore_missing=True) defaults = {} if not defaults else defaults for column, value in defaults.items(): From aca1500b829ecf8e9fcd9cac00494256c720be05 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 13 Mar 2022 08:33:55 +0000 Subject: [PATCH 290/727] bumped version --- CHANGES.rst | 8 ++++++++ piccolo/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index a23fcd63c..2ba51f89c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,14 @@ Changes ======= +0.71.1 +------ + +Fixed a bug with ``ModelBuilder`` and nullable columns (see `PR 462 `_). +Thanks to @fiolet069 for reporting this issue. + +------------------------------------------------------------------------------- + 0.71.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 24cbbe8a7..c35bd42b4 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.71.0" +__VERSION__ = "0.71.1" From 52f01aebbea0e1e1d22d4ba8decae6749ccd7dee Mon Sep 17 00:00:00 2001 From: Patrick Smyth Date: Thu, 24 Mar 2022 04:57:29 -0400 Subject: [PATCH 291/727] Changed occurances of contraint in the repository to constraint, including three occurances in piccolo/query/methods/alter.py (#468) --- piccolo/apps/migrations/auto/migration_manager.py | 2 +- piccolo/columns/base.py | 2 +- piccolo/query/methods/alter.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/piccolo/apps/migrations/auto/migration_manager.py b/piccolo/apps/migrations/auto/migration_manager.py index 3f1a591f1..a03407efb 100644 --- a/piccolo/apps/migrations/auto/migration_manager.py +++ b/piccolo/apps/migrations/auto/migration_manager.py @@ -457,7 +457,7 @@ async def _run_alter_columns(self, backwards=False): unique = params.get("unique") if unique is not None: - # When modifying unique contraints, we need to pass in + # When modifying unique constraints, we need to pass in # a column type, and not just the column name. column = Column() column._meta._table = _Table diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 3aa13fa4e..5a92322fb 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -356,7 +356,7 @@ class Column(Selectable): The column value to use if not specified by the user. :param unique: - If set, a unique contraint will be added to the column. + If set, a unique constraint will be added to the column. :param index: Whether an index is created for the column, which can improve diff --git a/piccolo/query/methods/alter.py b/piccolo/query/methods/alter.py index 47c01f8e7..115a35d0d 100644 --- a/piccolo/query/methods/alter.py +++ b/piccolo/query/methods/alter.py @@ -264,7 +264,7 @@ class Alter(DDL): __slots__ = ( "_add_foreign_key_constraint", "_add", - "_drop_contraint", + "_drop_constraint", "_drop_default", "_drop_table", "_drop", @@ -282,7 +282,7 @@ def __init__(self, table: t.Type[Table], **kwargs): super().__init__(table, **kwargs) self._add_foreign_key_constraint: t.List[AddForeignKeyConstraint] = [] self._add: t.List[AddColumn] = [] - self._drop_contraint: t.List[DropConstraint] = [] + self._drop_constraint: t.List[DropConstraint] = [] self._drop_default: t.List[DropDefault] = [] self._drop_table: t.Optional[DropTable] = None self._drop: t.List[DropColumn] = [] @@ -434,7 +434,7 @@ def _get_constraint_name(self, column: t.Union[str, ForeignKey]) -> str: return f"{tablename}_{column_name}_fk" def drop_constraint(self, constraint_name: str) -> Alter: - self._drop_contraint.append( + self._drop_constraint.append( DropConstraint(constraint_name=constraint_name) ) return self From 2a6f2e51f3ad4411e70f502aa8bfbd756d46da2f Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 24 Mar 2022 11:28:52 +0000 Subject: [PATCH 292/727] add `html_short_title` to Sphinx `conf.py` (#470) --- docs/src/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/src/conf.py b/docs/src/conf.py index e0e8f0940..ca8a64159 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -61,6 +61,7 @@ # -- Options for HTML output ------------------------------------------------- html_theme = "piccolo_theme" +html_short_title = "Piccolo" html_show_sphinx = False globaltoc_maxdepth = 3 From c773da7fabfa2b24bb6e3467a62a0ef2d8d0ec33 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 25 Mar 2022 23:39:16 +0000 Subject: [PATCH 293/727] Doc improvements (#473) * replace raw links They look a bit ugly in the docs * `AppRegistry` is no longer a dataclass There was no point really - the class only had one argument, and dataclasses look bad when using autodoc. * fix list indentation * show `None` in a code block * cleaner link to Piccolo Admin * cleaner `Engine` link * break code examples over more lines You had to scroll to see all of the code snippets * use `:class` to link JSON and JSONB to definitions * remove redundant docs from `BaseUser` * link to Python's `Enum` docs * replace raw link with something prettier * fix issue with autodoc and `Array` columns It's a bit of a hack, but it's better than autodoc looking broken. * use `:class` to link to column definitions --- docs/src/piccolo/authentication/baseuser.rst | 5 ++-- .../projects_and_apps/piccolo_projects.rst | 3 +- docs/src/piccolo/query_clauses/output.rst | 4 +-- docs/src/piccolo/query_clauses/where.rst | 13 ++++---- docs/src/piccolo/query_types/objects.rst | 14 +++++---- docs/src/piccolo/query_types/select.rst | 9 ++++-- docs/src/piccolo/schema/advanced.rst | 2 +- piccolo/columns/column_types.py | 30 +++++++++++++++---- piccolo/conf/apps.py | 8 ++--- piccolo/engine/postgres.py | 5 ++-- piccolo/engine/sqlite.py | 6 ++-- 11 files changed, 62 insertions(+), 37 deletions(-) diff --git a/docs/src/piccolo/authentication/baseuser.rst b/docs/src/piccolo/authentication/baseuser.rst index d498d2984..ae687ad5b 100644 --- a/docs/src/piccolo/authentication/baseuser.rst +++ b/docs/src/piccolo/authentication/baseuser.rst @@ -65,12 +65,12 @@ For example: piccolo user change_permissions some_user --active=true -The Piccolo Admin (see :ref:`Ecosystem`) uses these attributes to control who +The :ref:`Piccolo Admin` uses these attributes to control who can login and what they can do. * **active** and **admin** - must be true for a user to be able to login. * **superuser** - must be true for a user to be able to change other user's - passwords. + passwords. ------------------------------------------------------------------------------- @@ -172,3 +172,4 @@ Source .. autoclass:: BaseUser :members: create_user, create_user_sync, login, login_sync, update_password, update_password_sync + :class-doc-from: class diff --git a/docs/src/piccolo/projects_and_apps/piccolo_projects.rst b/docs/src/piccolo/projects_and_apps/piccolo_projects.rst index 31e27bd4a..945a0da4d 100644 --- a/docs/src/piccolo/projects_and_apps/piccolo_projects.rst +++ b/docs/src/piccolo/projects_and_apps/piccolo_projects.rst @@ -87,8 +87,7 @@ Here's an example: DB -- -The DB setting is an ``Engine`` instance. To learn more Engines, see -:ref:`Engines`. +The DB setting is an ``Engine`` instance (see the :ref:`Engine docs `). ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/query_clauses/output.rst b/docs/src/piccolo/query_clauses/output.rst index e4eacefb6..f9161b1cf 100644 --- a/docs/src/piccolo/query_clauses/output.rst +++ b/docs/src/piccolo/query_clauses/output.rst @@ -59,8 +59,8 @@ Select and Objects queries load_json ~~~~~~~~~ -If querying JSON or JSONB columns, you can tell Piccolo to deserialise the JSON -values automatically. +If querying :class:`JSON ` or :class:`JSONB ` +columns, you can tell Piccolo to deserialise the JSON values automatically. .. code-block:: python diff --git a/docs/src/piccolo/query_clauses/where.rst b/docs/src/piccolo/query_clauses/where.rst index cc2785729..b80c955b0 100644 --- a/docs/src/piccolo/query_clauses/where.rst +++ b/docs/src/piccolo/query_clauses/where.rst @@ -30,11 +30,12 @@ Equal / Not Equal Band.name != 'Rustaceans' ) -.. hint:: With ``Boolean`` columns, some linters will complain if you write - ``SomeTable.some_column == True`` (because it's more Pythonic to do - ``is True``). To work around this, you can do - ``SomeTable.some_column.eq(True)``. Likewise, with ``!=`` you can use - ``SomeTable.some_column.ne(True)`` +.. hint:: With :class:`Boolean ` columns, + some linters will complain if you write + ``SomeTable.some_column == True`` (because it's more Pythonic to do + ``is True``). To work around this, you can do + ``SomeTable.some_column.eq(True)``. Likewise, with ``!=`` you can use + ``SomeTable.some_column.ne(True)`` ------------------------------------------------------------------------------- @@ -116,7 +117,7 @@ is_null / is_not_null --------------------- These queries work, but some linters will complain about doing a comparison -with None: +with ``None``: .. code-block:: python diff --git a/docs/src/piccolo/query_types/objects.rst b/docs/src/piccolo/query_types/objects.rst index 31aceffc0..f33f03922 100644 --- a/docs/src/piccolo/query_types/objects.rst +++ b/docs/src/piccolo/query_types/objects.rst @@ -107,8 +107,9 @@ Fetching related objects get_related ~~~~~~~~~~~ -If you have an object from a table with a ``ForeignKey`` column, and you want -to fetch the related row as an object, you can do so using ``get_related``. +If you have an object from a table with a :class:`ForeignKey ` +column, and you want to fetch the related row as an object, you can do so +using ``get_related``. .. code-block:: python @@ -126,8 +127,8 @@ Prefetching related objects ~~~~~~~~~~~~~~~~~~~~~~~~~~~ You can also prefetch the rows from related tables, and store them as child -objects. To do this, pass ``ForeignKey`` columns into ``objects``, which -refer to the related rows you want to load. +objects. To do this, pass :class:`ForeignKey ` +columns into ``objects``, which refer to the related rows you want to load. .. code-block:: python @@ -166,8 +167,9 @@ database, just as you would expect: ticket.concert.band_1.name = 'Pythonistas 2' await ticket.concert.band_1.save() -Instead of passing the ``ForeignKey`` columns into the ``objects`` method, you -can use the ``prefetch`` clause if you prefer. +Instead of passing the :class:`ForeignKey ` +columns into the ``objects`` method, you can use the ``prefetch`` clause if you +prefer. .. code-block:: python diff --git a/docs/src/piccolo/query_types/select.rst b/docs/src/piccolo/query_types/select.rst index d1f30d973..0c85500d5 100644 --- a/docs/src/piccolo/query_types/select.rst +++ b/docs/src/piccolo/query_types/select.rst @@ -235,7 +235,10 @@ You also can have multiple different aggregate functions in one query: .. code-block:: python >>> from piccolo.query import Avg, Sum - >>> response = await Band.select(Avg(Band.popularity), Sum(Band.popularity)).first() + >>> response = await Band.select( + ... Avg(Band.popularity), + ... Sum(Band.popularity) + ... ).first() >>> response {"avg": 750.0, "sum": 1500} @@ -244,7 +247,9 @@ And can use aliases for aggregate functions like this: .. code-block:: python # Alternatively, you can use the `as_alias` method. - >>> response = await Band.select(Avg(Band.popularity).as_alias("popularity_avg")).first() + >>> response = await Band.select( + ... Avg(Band.popularity).as_alias("popularity_avg") + ... ).first() >>> response["popularity_avg"] 750.0 diff --git a/docs/src/piccolo/schema/advanced.rst b/docs/src/piccolo/schema/advanced.rst index 0532aebfd..510d4913c 100644 --- a/docs/src/piccolo/schema/advanced.rst +++ b/docs/src/piccolo/schema/advanced.rst @@ -89,7 +89,7 @@ use mixins to reduce the amount of repetition. Choices ------- -You can specify choices for a column, using Python's ``Enum`` support. +You can specify choices for a column, using Python's :class:`Enum ` support. .. code-block:: python diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index c8f7eda04..6fdf878b3 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -2132,6 +2132,28 @@ def __set__(self, obj, value: bytes): ############################################################################### +class List: + """ + This is a hack. Sphinx's autodoc fails if we have this function signature:: + + class Array(Column): + + def __init__(default=list): + ... + + We can't use ``list`` as a default value without breaking autodoc, so + instead we assign an instance of this class, which effectively acts like + the builtin ``list`` function, without breaking autodoc. + + """ + + def __call__(self): + return [] + + def __repr__(self): + return "list" + + class Array(Column): """ Used for storing lists of data. @@ -2157,7 +2179,7 @@ class Ticket(Table): def __init__( self, base_column: Column, - default: t.Union[t.List, Enum, t.Callable[[], t.List], None] = list, + default: t.Union[t.List, Enum, t.Callable[[], t.List], None] = List(), **kwargs, ) -> None: if isinstance(base_column, ForeignKey): @@ -2188,9 +2210,7 @@ def __getitem__(self, value: int) -> Array: """ Allows queries which retrieve an item from the array. The index starts with 0 for the first value. If you were to write the SQL by hand, the - first index would be 1 instead: - - https://www.postgresql.org/docs/current/arrays.html + first index would be 1 instead (see `Postgres docs `_). However, we keep the first index as 0 to fit better with Python. @@ -2202,7 +2222,7 @@ def __getitem__(self, value: int) -> Array: {'seat_numbers': 325} - """ + """ # noqa: E501 engine_type = self._meta.table._meta.db.engine_type if engine_type != "postgres": raise ValueError( diff --git a/piccolo/conf/apps.py b/piccolo/conf/apps.py index 81ef43c71..fd4dccd88 100644 --- a/piccolo/conf/apps.py +++ b/piccolo/conf/apps.py @@ -188,7 +188,6 @@ def get_table_with_name(self, table_class_name: str) -> t.Type[Table]: return filtered[0] -@dataclass class AppRegistry: """ Records all of the Piccolo apps in your project. Kept in @@ -199,9 +198,8 @@ class AppRegistry: """ - apps: t.List[str] = field(default_factory=list) - - def __post_init__(self): + def __init__(self, apps: t.List[str] = None): + self.apps = apps or [] self.app_configs: t.Dict[str, AppConfig] = {} app_names = [] @@ -214,7 +212,7 @@ def __post_init__(self): raise e from e app += ".piccolo_app" app_conf_module = import_module(app) - app_config: AppConfig = getattr(app_conf_module, "APP_CONFIG") + app_config = getattr(app_conf_module, "APP_CONFIG") colored_warning( f"App {app[:-12]} should end with `.piccolo_app`", level=Level.medium, diff --git a/piccolo/engine/postgres.py b/piccolo/engine/postgres.py index 56aad2b23..c59c8614d 100644 --- a/piccolo/engine/postgres.py +++ b/piccolo/engine/postgres.py @@ -214,9 +214,8 @@ class PostgresEngine(Engine): For example, ``{'host': 'localhost', 'port': 5432}``. - To see all available options: - - * https://magicstack.github.io/asyncpg/current/api/index.html#connection + See the `asyncpg docs `_ + for all available options. :param extensions: When the engine starts, it will try and create these extensions diff --git a/piccolo/engine/sqlite.py b/piccolo/engine/sqlite.py index 0cad6e574..317514513 100644 --- a/piccolo/engine/sqlite.py +++ b/piccolo/engine/sqlite.py @@ -344,10 +344,10 @@ class SQLiteEngine(Engine): """ Any connection kwargs are passed into the database adapter. - See here for more info: - https://docs.python.org/3/library/sqlite3.html#sqlite3.connect + See the `SQLite docs `_ + for more info. - """ + """ # noqa: E501 __slots__ = ("connection_kwargs", "transaction_connection") From cfaae7cbf41e4cb59f16a6fb05a43d8b823db019 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 26 Mar 2022 09:38:22 +0000 Subject: [PATCH 294/727] add `project_urls` to setup.py --- setup.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 46fef6b3e..e904e0a41 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ def extras_require() -> t.Dict[str, t.List[str]]: Parse requirements in requirements/extras directory """ extra_requirements = { - extra: parse_requirement(os.path.join("extras", f'{extra}.txt')) + extra: parse_requirement(os.path.join("extras", f"{extra}.txt")) for extra in extras } @@ -45,7 +45,6 @@ def extras_require() -> t.Dict[str, t.List[str]]: itertools.chain.from_iterable(extra_requirements.values()) ) - return extra_requirements @@ -71,6 +70,13 @@ def extras_require() -> t.Dict[str, t.List[str]]: ], "piccolo": ["py.typed"], }, + project_urls={ + "Documentation": ( + "https://piccolo-orm.readthedocs.io/en/latest/index.html" + ), + "Source": "https://github.com/piccolo-orm/piccolo", + "Tracker": "https://github.com/piccolo-orm/piccolo/issues", + }, install_requires=parse_requirement("requirements.txt"), extras_require=extras_require(), license="MIT", From 06ffbc9df26566390df17fc8e6ed079d4b969e4e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 30 Mar 2022 08:32:23 +0100 Subject: [PATCH 295/727] `AppConfig` now accepts `pathlib.Path` arguments (#474) * `AppConfig` now accepts `pathlib.Path` arguments * Update dev-requirements.txt --- piccolo/conf/apps.py | 4 ++++ requirements/dev-requirements.txt | 8 ++++---- tests/conf/test_apps.py | 13 ++++++++++++- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/piccolo/conf/apps.py b/piccolo/conf/apps.py index fd4dccd88..6176224af 100644 --- a/piccolo/conf/apps.py +++ b/piccolo/conf/apps.py @@ -4,6 +4,7 @@ import inspect import itertools import os +import pathlib import traceback import typing as t from dataclasses import dataclass, field @@ -155,6 +156,9 @@ def __post_init__(self): i if isinstance(i, Command) else Command(i) for i in self.commands ] + if isinstance(self.migrations_folder_path, pathlib.Path): + self.migrations_folder_path = str(self.migrations_folder_path) + def register_table(self, table_class: t.Type[Table]): self.table_classes.append(table_class) return table_class diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt index c4e438143..d5a6967cc 100644 --- a/requirements/dev-requirements.txt +++ b/requirements/dev-requirements.txt @@ -1,10 +1,10 @@ -black==22.1.0 +black==22.3.0 ipdb==0.13.9 ipython>=7.31.1 flake8==4.0.1 isort==5.10.1 -slotscheck==0.12.0 -twine==3.7.1 -mypy==0.931 +slotscheck==0.14.0 +twine==3.8.0 +mypy==0.942 pip-upgrader==1.4.15 wheel==0.37.1 diff --git a/tests/conf/test_apps.py b/tests/conf/test_apps.py index 717f9f857..42a10820c 100644 --- a/tests/conf/test_apps.py +++ b/tests/conf/test_apps.py @@ -1,3 +1,4 @@ +import pathlib from unittest import TestCase from piccolo.apps.user.tables import BaseUser @@ -73,11 +74,21 @@ def test_get_table_with_name(self): class TestAppConfig(TestCase): + def test_pathlib(self): + """ + Make sure a ``pathlib.Path`` instance can be passed in as a + ``migrations_folder_path`` argument. + """ + config = AppConfig( + app_name="music", migrations_folder_path=pathlib.Path(__file__) + ) + self.assertEqual(config.migrations_folder_path, __file__) + def test_get_table_with_name(self): """ Register a table, then test retrieving it. """ - config = AppConfig(app_name="Music", migrations_folder_path="") + config = AppConfig(app_name="music", migrations_folder_path="") config.register_table(table_class=Manager) self.assertEqual(config.get_table_with_name("Manager"), Manager) From 84921dd6b61b85517f3d09bd38751f8ebf929b4f Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 30 Mar 2022 10:18:06 +0100 Subject: [PATCH 296/727] `Array` default improvements (#475) * `Array` default - swap to `list` in `__init__` We introduced `List` as a workaround to fix Sphinx's autodoc. But I don't want this hack bleeding out into the wider codebase. So in `Array.__init__` convert `List` to a plain `list`. * renamed `List` to `ListProxy` I could see this getting confused with `typing.List` otherwise. * add test --- piccolo/columns/column_types.py | 23 +++++++++++++++-------- tests/columns/test_array.py | 11 +++++++++++ 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 6fdf878b3..27ed93274 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -2132,19 +2132,19 @@ def __set__(self, obj, value: bytes): ############################################################################### -class List: +class ListProxy: """ - This is a hack. Sphinx's autodoc fails if we have this function signature:: + Sphinx's autodoc fails if we have this function signature:: class Array(Column): def __init__(default=list): ... - We can't use ``list`` as a default value without breaking autodoc, so - instead we assign an instance of this class, which effectively acts like - the builtin ``list`` function, without breaking autodoc. - + We can't use ``list`` as a default value without breaking autodoc (it + doesn't seem to like it when a class type is used as a default), so + instead we assign an instance of this class. It keeps both autodoc and MyPy + happy. In ``Array.__init__`` we then swap it out for ``list``. """ def __call__(self): @@ -2179,12 +2179,19 @@ class Ticket(Table): def __init__( self, base_column: Column, - default: t.Union[t.List, Enum, t.Callable[[], t.List], None] = List(), + default: t.Union[ + t.List, Enum, t.Callable[[], t.List], None + ] = ListProxy(), **kwargs, ) -> None: if isinstance(base_column, ForeignKey): raise ValueError("Arrays of ForeignKeys aren't allowed.") + # This is a workaround because having `list` as a default breaks + # Sphinx's autodoc. + if isinstance(default, ListProxy): + default = list + self._validate_default(default, (list, None)) # Usually columns are given a name by the Table metaclass, but in this @@ -2210,7 +2217,7 @@ def __getitem__(self, value: int) -> Array: """ Allows queries which retrieve an item from the array. The index starts with 0 for the first value. If you were to write the SQL by hand, the - first index would be 1 instead (see `Postgres docs `_). + first index would be 1 instead (see `Postgres array docs `_). However, we keep the first index as 0 to fit better with Python. diff --git a/tests/columns/test_array.py b/tests/columns/test_array.py index 7e0141925..e46ad4c6a 100644 --- a/tests/columns/test_array.py +++ b/tests/columns/test_array.py @@ -9,6 +9,17 @@ class MyTable(Table): value = Array(base_column=Integer()) +class TestArrayDefault(TestCase): + def test_array_default(self): + """ + We use ``ListProxy`` instead of ``list`` as a default, because of + issues with Sphinx's autodoc. Make sure it's correctly converted to a + plain ``list`` in ``Array.__init__``. + """ + column = Array(base_column=Integer()) + self.assertTrue(column.default is list) + + class TestArrayPostgres(TestCase): """ Make sure an Array column can be created. From d01066002e9932ff7b7ea7e2e678b8a54faf666c Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 30 Mar 2022 10:30:18 +0100 Subject: [PATCH 297/727] bumped version --- CHANGES.rst | 25 +++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2ba51f89c..91acb8d9d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,31 @@ Changes ======= +0.72.0 +------ + +Fixed typos with ``drop_constraints``. Courtesy @smythp. + +Lots of documentation improvements, such as fixing Sphinx's autodoc for the +``Array`` column. + +``AppConfig`` now accepts a ``pathlib.Path`` instance. For example: + +.. code-block:: python + + # piccolo_app.py + + import pathlib + + APP_CONFIG = AppConfig( + app_name="blog", + migrations_folder_path=pathlib.Path(__file__) / "piccolo_migrations" + ) + +Thanks to @theelderbeever for recommending this feature. + +------------------------------------------------------------------------------- + 0.71.1 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index c35bd42b4..58cc0d0eb 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.71.1" +__VERSION__ = "0.72.0" From 89fb03142b732e7819b57e84e62c6e9e37cf02c4 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 8 Apr 2022 14:10:10 +0100 Subject: [PATCH 298/727] bump targ version (#480) --- requirements/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 46f200de6..320b5f9c4 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,7 +1,7 @@ black colorama>=0.4.0 Jinja2>=2.11.0 -targ>=0.3.3 +targ>=0.3.7 inflection>=0.5.1 typing-extensions>=3.10.0.0 pydantic>=1.6 From 2ba0891d5ab82a6e59541b680c3c74ba955d5d38 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 8 Apr 2022 17:25:43 +0100 Subject: [PATCH 299/727] prototype for multiple engines (#481) * prototype for multiple engines * fix typos in docstring * add test for `extra_nodes` * import test engine config --- piccolo/engine/postgres.py | 32 ++++++++++++++++++++++++++-- piccolo/query/base.py | 28 ++++++++++++++++++++++--- tests/engine/test_extra_nodes.py | 36 ++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 5 deletions(-) create mode 100644 tests/engine/test_extra_nodes.py diff --git a/piccolo/engine/postgres.py b/piccolo/engine/postgres.py index c59c8614d..b6a4d58a0 100644 --- a/piccolo/engine/postgres.py +++ b/piccolo/engine/postgres.py @@ -222,8 +222,30 @@ class PostgresEngine(Engine): in Postgres. :param log_queries: - If True, all SQL and DDL statements are printed out before being run. - Useful for debugging. + If ``True``, all SQL and DDL statements are printed out before being + run. Useful for debugging. + + :param extra_nodes: + If you have additional database nodes (e.g. read replicas) for the + server, you can specify them here. It's a mapping of a memorable name + to a ``PostgresEngine`` instance. For example:: + + DB = PostgresEngine( + config={'database': 'main_db'}, + extra_nodes={ + 'read_replica_1': PostgresEngine( + config={ + 'database': 'main_db', + host: 'read_replicate.my_db.com' + } + ) + } + ) + + When executing a query, you can specify one of these nodes instead + of the main database. For example:: + + >>> await MyTable.select().run(node="read_replica_1") """ # noqa: E501 @@ -231,6 +253,7 @@ class PostgresEngine(Engine): "config", "extensions", "log_queries", + "extra_nodes", "pool", "transaction_connection", ) @@ -243,12 +266,17 @@ def __init__( config: t.Dict[str, t.Any], extensions: t.Sequence[str] = None, log_queries: bool = False, + extra_nodes: t.Dict[str, PostgresEngine] = None, ) -> None: if extensions is None: extensions = ["uuid-ossp"] + if extra_nodes is None: + extra_nodes = {} + self.config = config self.extensions = extensions self.log_queries = log_queries + self.extra_nodes = extra_nodes self.pool: t.Optional[Pool] = None database_name = config.get("database", "Unknown") self.transaction_connection = contextvars.ContextVar( diff --git a/piccolo/query/base.py b/piccolo/query/base.py index 57f592ea0..ba1b6ff58 100644 --- a/piccolo/query/base.py +++ b/piccolo/query/base.py @@ -167,16 +167,34 @@ def __await__(self): """ return self.run().__await__() - async def run(self, in_pool=True): + async def run(self, node: t.Optional[str] = None, in_pool: bool = True): + """ + Run the query on the database. + + :param node: + If specified, run this query against another database node. Only + available in Postgres. See :class:`PostgresEngine `. + :param in_pool: + Whether to run this in a connection pool if one is running. This is + mostly just for debugging - use a connection pool where possible. + + """ # noqa: E501 self._validate() engine = self.table._meta.db + if not engine: raise ValueError( f"Table {self.table._meta.tablename} has no db defined in " "_meta" ) + if node is not None: + from piccolo.engine.postgres import PostgresEngine + + if isinstance(engine, PostgresEngine): + engine = engine.extra_nodes[node] + querystrings = self.querystrings if len(querystrings) == 1: @@ -186,7 +204,6 @@ async def run(self, in_pool=True): return await self._process_results(results) else: responses = [] - # TODO - run in a transaction for querystring in querystrings: results = await engine.run_querystring( querystring, in_pool=in_pool @@ -197,8 +214,13 @@ async def run(self, in_pool=True): def run_sync(self, timed=False, *args, **kwargs): """ A convenience method for running the coroutine synchronously. + + :param timed: + If ``True``, the time taken to run the query is printed out. Useful + for debugging. + """ - coroutine = self.run(*args, **kwargs, in_pool=False) + coroutine = self.run(*args, **kwargs) if not timed: return run_sync(coroutine) diff --git a/tests/engine/test_extra_nodes.py b/tests/engine/test_extra_nodes.py new file mode 100644 index 000000000..bbe10b940 --- /dev/null +++ b/tests/engine/test_extra_nodes.py @@ -0,0 +1,36 @@ +from unittest import TestCase +from unittest.mock import MagicMock + +from piccolo.columns.column_types import Varchar +from piccolo.engine import engine_finder +from piccolo.engine.postgres import PostgresEngine +from piccolo.table import Table +from tests.base import AsyncMock, postgres_only + + +@postgres_only +class TestExtraNodes(TestCase): + def test_extra_nodes(self): + """ + Make sure that other nodes can be queried. + """ + # Get the test database credentials: + test_engine = engine_finder() + + EXTRA_NODE = MagicMock(spec=PostgresEngine(config=test_engine.config)) + EXTRA_NODE.run_querystring = AsyncMock(return_value=[]) + + DB = PostgresEngine( + config=test_engine.config, extra_nodes={"read_1": EXTRA_NODE} + ) + + class Manager(Table, db=DB): + name = Varchar() + + # Make sure the node is queried + Manager.select().run_sync(node="read_1") + self.assertTrue(EXTRA_NODE.run_querystring.called) + + # Make sure that a non existent node raises an error + with self.assertRaises(KeyError): + Manager.select().run_sync(node="read_2") From ffe173d69156cda0f0bd356175c4f40a9e0c6bdc Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 8 Apr 2022 17:35:39 +0100 Subject: [PATCH 300/727] bumped version --- CHANGES.rst | 35 +++++++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 91acb8d9d..f8a090772 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,41 @@ Changes ======= +0.73.0 +------ + +You can now specify extra nodes for a database. For example, if you have a +read replica. + +.. code-block:: python + + DB = PostgresEngine( + config={'database': 'main_db'}, + extra_nodes={ + 'read_replica_1': PostgresEngine( + config={ + 'database': 'main_db', + 'host': 'read_replica_1.my_db.com' + } + ) + } + ) + +And can then run queries on these other nodes: + +.. code-block:: python + + >>> await MyTable.select().run(node="read_replica_1") + +See `PR 481 `_. Thanks to +@dashsatish for suggesting this feature. + +Also, the ``targ`` library has been updated so it tells users about the +``--trace`` argument which can be used to get a full traceback when a CLI +command fails. + +------------------------------------------------------------------------------- + 0.72.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 58cc0d0eb..91ae994c7 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.72.0" +__VERSION__ = "0.73.0" From aac957351ba13c2d5f2b9bc21493075cfd9b04e5 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 9 Apr 2022 19:46:04 +0100 Subject: [PATCH 301/727] Chores (#484) * include py310 in black target version * use codeblocks in docstings * more docstring codeblocks * remove some `type: ignore` statements We can tell MyPy to ignore errors in third party libraries in pyproject.toml. * remove `run` from docstring examples --- piccolo/apps/app/commands/new.py | 2 +- piccolo/apps/asgi/commands/new.py | 4 ++-- piccolo/apps/migrations/commands/new.py | 2 +- piccolo/apps/playground/commands/run.py | 2 +- piccolo/columns/column_types.py | 10 +++++---- piccolo/engine/postgres.py | 27 +++++++++++++------------ piccolo/main.py | 2 +- pyproject.toml | 10 +++++---- 8 files changed, 32 insertions(+), 27 deletions(-) diff --git a/piccolo/apps/app/commands/new.py b/piccolo/apps/app/commands/new.py index 4badf00e2..973134653 100644 --- a/piccolo/apps/app/commands/new.py +++ b/piccolo/apps/app/commands/new.py @@ -5,7 +5,7 @@ import sys import typing as t -import black # type: ignore +import black import jinja2 TEMPLATE_DIRECTORY = os.path.join( diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index 390b3f5b3..10217793f 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -4,8 +4,8 @@ import shutil import typing as t -import black # type: ignore -import colorama # type: ignore +import black +import colorama from jinja2 import Environment, FileSystemLoader TEMPLATE_DIR = os.path.join(os.path.dirname(__file__), "templates/app/") diff --git a/piccolo/apps/migrations/commands/new.py b/piccolo/apps/migrations/commands/new.py index 937cb8627..3cc04fbb0 100644 --- a/piccolo/apps/migrations/commands/new.py +++ b/piccolo/apps/migrations/commands/new.py @@ -8,7 +8,7 @@ from itertools import chain from types import ModuleType -import black # type: ignore +import black import jinja2 from piccolo import __VERSION__ diff --git a/piccolo/apps/playground/commands/run.py b/piccolo/apps/playground/commands/run.py index 38965da76..598fb7e1d 100644 --- a/piccolo/apps/playground/commands/run.py +++ b/piccolo/apps/playground/commands/run.py @@ -142,7 +142,7 @@ def run( Postgres port """ try: - import IPython # type: ignore + import IPython except ImportError: sys.exit( "Install iPython using `pip install 'piccolo[playground,sqlite]'` " diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 27ed93274..33b2b77b6 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -68,9 +68,10 @@ class Band(Table): class ConcatDelegate: """ - Used in update queries to concatenate two strings - for example: + Used in update queries to concatenate two strings - for example:: + + await Band.update({Band.name: Band.name + 'abc'}) - await Band.update({Band.name: Band.name + 'abc'}).run() """ def get_querystring( @@ -122,9 +123,10 @@ def get_querystring( class MathDelegate: """ - Used in update queries to perform math operations on columns, for example: + Used in update queries to perform math operations on columns, for example:: + + await Band.update({Band.popularity: Band.popularity + 100}) - await Band.update({Band.popularity: Band.popularity + 100}).run() """ def get_querystring( diff --git a/piccolo/engine/postgres.py b/piccolo/engine/postgres.py index b6a4d58a0..2c9330e76 100644 --- a/piccolo/engine/postgres.py +++ b/piccolo/engine/postgres.py @@ -15,9 +15,9 @@ asyncpg = LazyLoader("asyncpg", globals(), "asyncpg") if t.TYPE_CHECKING: # pragma: no cover - from asyncpg.connection import Connection # type: ignore - from asyncpg.cursor import Cursor # type: ignore - from asyncpg.pool import Pool # type: ignore + from asyncpg.connection import Connection + from asyncpg.cursor import Cursor + from asyncpg.pool import Pool @dataclass @@ -78,14 +78,15 @@ class Atomic: This is useful if you want to build up a transaction programatically, by adding queries to it. - Usage: + Usage:: - transaction = engine.atomic() - transaction.add(Foo.create_table()) + transaction = engine.atomic() + transaction.add(Foo.create_table()) + + # Either: + transaction.run_sync() + await transaction.run() - # Either: - transaction.run_sync() - await transaction.run() """ __slots__ = ("engine", "queries") @@ -145,11 +146,11 @@ class Transaction: Used for wrapping queries in a transaction, using a context manager. Currently it's async only. - Usage: + Usage:: - async with engine.transaction(): - # Run some queries: - await Band.select().run() + async with engine.transaction(): + # Run some queries: + await Band.select().run() """ diff --git a/piccolo/main.py b/piccolo/main.py index 33f9a11db..a175ddea4 100644 --- a/piccolo/main.py +++ b/piccolo/main.py @@ -1,7 +1,7 @@ import os import sys -from targ import CLI # type: ignore +from targ import CLI try: import uvloop # type: ignore diff --git a/pyproject.toml b/pyproject.toml index 37430c5a0..6b4f9a201 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.black] line-length = 79 -target-version = ['py37', 'py38', 'py39'] +target-version = ['py37', 'py38', 'py39', 'py310'] [tool.isort] profile = "black" @@ -9,10 +9,12 @@ line_length = 79 [tool.mypy] [[tool.mypy.overrides]] module = [ - "orjson", - "jinja2", + "asyncpg.*", + "colorama", "dateutil", - "asyncpg.pgproto.pgproto" + "IPython", + "jinja2", + "orjson" ] ignore_missing_imports = true From a231da95f01de1f186a572410d9add0d795debb0 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 13 Apr 2022 21:35:26 +0100 Subject: [PATCH 302/727] Add operator support to `Timestamp` updates (#486) * initial prototype * skip SQLite for now * reformat using black * fix `Literal` on Python 3.7 * fix type annotation * added `__radd__` * test all timedelta attributes * allow adding/subtracting timedelta to other column types * standardise test format * added sqlite support * support adding intervals in SQLite * add docstrings * update docs * added tests and docs for updating null values --- docs/src/piccolo/query_types/update.rst | 165 +++++-- piccolo/columns/column_types.py | 300 ++++++++++-- piccolo/columns/operators/__init__.py | 2 +- piccolo/columns/operators/string.py | 6 +- tests/base.py | 13 +- tests/columns/test_varchar.py | 2 +- tests/table/test_update.py | 576 +++++++++++++++++------- 7 files changed, 824 insertions(+), 240 deletions(-) diff --git a/docs/src/piccolo/query_types/update.rst b/docs/src/piccolo/query_types/update.rst index 820a8e1ce..24a6892f3 100644 --- a/docs/src/piccolo/query_types/update.rst +++ b/docs/src/piccolo/query_types/update.rst @@ -1,3 +1,4 @@ + .. _Update: Update @@ -19,9 +20,9 @@ This is used to update any rows in the table which match the criteria. force ----- -Piccolo won't let you run an update query without a where clause, unless you -explicitly tell it to do so. This is to prevent accidentally overwriting -the data in a table. +Piccolo won't let you run an update query without a :ref:`where clause `, +unless you explicitly tell it to do so. This is to prevent accidentally +overwriting the data in a table. .. code-block:: python @@ -31,13 +32,18 @@ the data in a table. # Works fine: >>> await Band.update({Band.popularity: 0}, force=True) + # Or just add a where clause: + >>> await Band.update({Band.popularity: 0}).where(Band.popularity < 50) + ------------------------------------------------------------------------------- Modifying values ---------------- -As well as replacing values with new ones, you can also modify existing values, for -instance by adding to an integer. +As well as replacing values with new ones, you can also modify existing values, +for instance by adding an integer. + +You can currently only combine two values together at a time. Integer columns ~~~~~~~~~~~~~~~ @@ -47,29 +53,44 @@ You can add / subtract / multiply / divide values: .. code-block:: python # Add 100 to the popularity of each band: - await Band.update({ - Band.popularity: Band.popularity + 100 - }) + await Band.update( + { + Band.popularity: Band.popularity + 100 + }, + force=True + ) # Decrease the popularity of each band by 100. - await Band.update({ - Band.popularity: Band.popularity - 100 - }) + await Band.update( + { + Band.popularity: Band.popularity - 100 + }, + force=True + ) # Multiply the popularity of each band by 10. - await Band.update({ - Band.popularity: Band.popularity * 10 - }) + await Band.update( + { + Band.popularity: Band.popularity * 10 + }, + force=True + ) # Divide the popularity of each band by 10. - await Band.update({ - Band.popularity: Band.popularity / 10 - }) + await Band.update( + { + Band.popularity: Band.popularity / 10 + }, + force=True + ) # You can also use the operators in reverse: - await Band.update({ - Band.popularity: 2000 - Band.popularity - }) + await Band.update( + { + Band.popularity: 2000 - Band.popularity + }, + force=True + ) Varchar / Text columns ~~~~~~~~~~~~~~~~~~~~~~ @@ -79,22 +100,96 @@ You can concatenate values: .. code-block:: python # Append "!!!" to each band name. - await Band.update({ - Band.name: Band.name + "!!!" - }) + await Band.update( + { + Band.name: Band.name + "!!!" + }, + force=True + ) # Concatenate the values in each column: - await Band.update({ - Band.name: Band.name + Band.name - }) + await Band.update( + { + Band.name: Band.name + Band.name + }, + force=True + ) # Prepend "!!!" to each band name. - await Band.update({ - Band.popularity: "!!!" + Band.popularity - }) + await Band.update( + { + Band.popularity: "!!!" + Band.popularity + }, + force=True + ) +Date / Timestamp / Timestamptz / Interval columns +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -You can currently only combine two values together at a time. +You can add or substract a :class:`timedelta ` to any of +these columns. + +For example, if we have a ``Concert`` table, and we want each concert to start +one day later, we can simply do this: + +.. code-block:: python + + await Concert.update( + { + Concert.starts: Concert.starts + datetime.timedelta(days=1) + }, + force=True + ) + +Likewise, we can decrease the values by 1 day: + +.. code-block:: python + + await Concert.update( + { + Concert.starts: Concert.starts - datetime.timedelta(days=1) + }, + force=True + ) + +What about null values? +~~~~~~~~~~~~~~~~~~~~~~~ + +If we have a table with a nullable column: + +.. code-block:: python + + class Band(Table): + name = Varchar(null=True) + +Any rows with a value of null aren't modified by an update: + +.. code-block:: python + + >>> await Band.insert(Band(name="Pythonistas"), Band(name=None)) + >>> await Band.update( + ... { + ... Band.name: Band.name + '!!!' + ... }, + ... force=True + ... ) + >>> await Band.select() + # Note how the second row's name value is still `None`: + [{'id': 1, 'name': 'Pythonistas!!!'}, {'id': 2, 'name': None}] + +It's more efficient to exclude any rows with a value of null using a +:ref:`where clause `: + +.. code-block:: python + + await Band.update( + { + Band.name + '!!!' + }, + force=True + ).where( + Band.name.is_not_null() + ) ------------------------------------------------------------------------------- @@ -106,11 +201,11 @@ you prefer: .. code-block:: python - >>> await Band.update( - ... name='Pythonistas 2' - ... ).where( - ... Band.name == 'Pythonistas' - ... ) + await Band.update( + name='Pythonistas 2' + ).where( + Band.name == 'Pythonistas' + ) ------------------------------------------------------------------------------- diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 33b2b77b6..84ee1db8e 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -35,6 +35,8 @@ class Band(Table): from datetime import date, datetime, time, timedelta from enum import Enum +from typing_extensions import Literal + from piccolo.columns.base import Column, ForeignKeyMeta, OnDelete, OnUpdate from piccolo.columns.combination import Where from piccolo.columns.defaults.date import DateArg, DateCustom, DateNow @@ -52,7 +54,7 @@ class Band(Table): ) from piccolo.columns.defaults.uuid import UUID4, UUIDArg from piccolo.columns.operators.comparison import ArrayAll, ArrayAny -from piccolo.columns.operators.string import ConcatPostgres, ConcatSQLite +from piccolo.columns.operators.string import Concat from piccolo.columns.reference import LazyTableReference from piccolo.querystring import QueryString, Unquoted from piccolo.utils.encoding import dump_json @@ -78,11 +80,8 @@ def get_querystring( self, column_name: str, value: t.Union[str, Varchar, Text], - engine_type: str, - reverse=False, + reverse: bool = False, ) -> QueryString: - Concat = ConcatPostgres if engine_type == "postgres" else ConcatSQLite - if isinstance(value, (Varchar, Text)): column: Column = value if len(column._meta.call_chain) > 0: @@ -132,9 +131,9 @@ class MathDelegate: def get_querystring( self, column_name: str, - operator: str, + operator: Literal["+", "-", "/", "*"], value: t.Union[int, float, Integer], - reverse=False, + reverse: bool = False, ) -> QueryString: if isinstance(value, Integer): column: Integer = value @@ -156,6 +155,131 @@ def get_querystring( ) +class TimedeltaDelegate: + """ + Used in update queries to add a timedelta to these columns: + + * ``Timestamp`` + * ``Timestamptz`` + * ``Date`` + * ``Interval`` + + Example:: + + class Concert(Table): + starts = Timestamp() + + # Lets us increase all of the matching values by 1 day: + >>> await Concert.update({ + ... Concert.starts: Concert.starts + datetime.timedelta(days=1) + ... }) + + """ + + # Maps the attribute name in Python's timedelta to what it's called in + # Postgres. + postgres_attr_map: t.Dict[str, str] = { + "days": "DAYS", + "seconds": "SECONDS", + "microseconds": "MICROSECONDS", + } + + def get_postgres_interval_string(self, interval: timedelta) -> str: + """ + :returns: + A string like:: + + "'1 DAYS 5 SECONDS 1000 MICROSECONDS'" + + """ + output = [] + for timedelta_key, postgres_name in self.postgres_attr_map.items(): + timestamp_value = getattr(interval, timedelta_key) + if timestamp_value: + output.append(f"{timestamp_value} {postgres_name}") + + output_string = " ".join(output) + return f"'{output_string}'" + + def get_sqlite_interval_string(self, interval: timedelta) -> str: + """ + :returns: + A string like:: + + "'+1 DAYS', '+5.001 SECONDS'" + + """ + output = [] + + data = { + "DAYS": interval.days, + "SECONDS": interval.seconds + (interval.microseconds / 10**6), + } + + for key, value in data.items(): + if value: + operator = "+" if value >= 0 else "" + output.append(f"'{operator}{value} {key}'") + + output_string = ", ".join(output) + return output_string + + def get_querystring( + self, + column: Column, + operator: Literal["+", "-"], + value: timedelta, + engine_type: str, + ) -> QueryString: + column_name = column._meta.name + + if not isinstance(value, timedelta): + raise ValueError("Only timedelta values can be added.") + + if engine_type == "postgres": + value_string = self.get_postgres_interval_string(interval=value) + return QueryString( + f'"{column_name}" {operator} INTERVAL {value_string}', + ) + elif engine_type == "sqlite": + if isinstance(column, Interval): + # SQLite doesn't have a proper Interval type. Instead we store + # the number of seconds. + return QueryString( + f'CAST("{column_name}" AS REAL) {operator} {value.total_seconds()}' # noqa: E501 + ) + elif isinstance(column, (Timestamp, Timestamptz)): + if ( + round(value.microseconds / 1000) * 1000 + != value.microseconds + ): + raise ValueError( + "timedeltas with such high precision won't save " + "accurately - the max resolution is 1 millisecond." + ) + strftime_format = "%Y-%m-%d %H:%M:%f" + elif isinstance(column, Date): + strftime_format = "%Y-%m-%d" + else: + raise ValueError( + f"{column.__class__.__name__} doesn't support timedelta " + "addition currently." + ) + + if operator == "-": + value = value * -1 + + value_string = self.get_sqlite_interval_string(interval=value) + + # We use `strftime` instead of `datetime`, because `datetime` + # doesn't return microseconds. + return QueryString( + f"strftime('{strftime_format}', \"{column_name}\", {value_string})" # noqa: E501 + ) + else: + raise ValueError("Unrecognised engine") + + ############################################################################### @@ -203,20 +327,18 @@ def __init__( def column_type(self): return f"VARCHAR({self.length})" if self.length else "VARCHAR" + ########################################################################### + # For update queries + def __add__(self, value: t.Union[str, Varchar, Text]) -> QueryString: - engine_type = self._meta.table._meta.db.engine_type return self.concat_delegate.get_querystring( - column_name=self._meta.db_column_name, - value=value, - engine_type=engine_type, + column_name=self._meta.db_column_name, value=value ) def __radd__(self, value: t.Union[str, Varchar, Text]) -> QueryString: - engine_type = self._meta.table._meta.db.engine_type return self.concat_delegate.get_querystring( column_name=self._meta.db_column_name, value=value, - engine_type=engine_type, reverse=True, ) @@ -300,20 +422,18 @@ def __init__( kwargs.update({"default": default}) super().__init__(**kwargs) + ########################################################################### + # For update queries + def __add__(self, value: t.Union[str, Varchar, Text]) -> QueryString: - engine_type = self._meta.table._meta.db.engine_type return self.concat_delegate.get_querystring( - column_name=self._meta.db_column_name, - value=value, - engine_type=engine_type, + column_name=self._meta.db_column_name, value=value ) def __radd__(self, value: t.Union[str, Varchar, Text]) -> QueryString: - engine_type = self._meta.table._meta.db.engine_type return self.concat_delegate.get_querystring( column_name=self._meta.db_column_name, value=value, - engine_type=engine_type, reverse=True, ) @@ -432,6 +552,9 @@ def __init__( kwargs.update({"default": default}) super().__init__(**kwargs) + ########################################################################### + # For update queries + def __add__(self, value: t.Union[int, float, Integer]) -> QueryString: return self.math_delegate.get_querystring( column_name=self._meta.db_column_name, operator="+", value=value @@ -546,7 +669,7 @@ class Band(Table): @property def column_type(self): - engine_type = self._meta.table._meta.db.engine_type + engine_type = self._meta.engine_type if engine_type == "postgres": return "BIGINT" elif engine_type == "sqlite": @@ -594,7 +717,7 @@ class Band(Table): @property def column_type(self): - engine_type = self._meta.table._meta.db.engine_type + engine_type = self._meta.engine_type if engine_type == "postgres": return "SMALLINT" elif engine_type == "sqlite": @@ -633,7 +756,7 @@ class Serial(Column): @property def column_type(self): - engine_type = self._meta.table._meta.db.engine_type + engine_type = self._meta.engine_type if engine_type == "postgres": return "SERIAL" elif engine_type == "sqlite": @@ -641,7 +764,7 @@ def column_type(self): raise Exception("Unrecognized engine type") def default(self): - engine_type = self._meta.table._meta.db.engine_type + engine_type = self._meta.engine_type if engine_type == "postgres": return DEFAULT elif engine_type == "sqlite": @@ -673,7 +796,7 @@ class BigSerial(Serial): @property def column_type(self): - engine_type = self._meta.table._meta.db.engine_type + engine_type = self._meta.engine_type if engine_type == "postgres": return "BIGSERIAL" elif engine_type == "sqlite": @@ -761,6 +884,7 @@ class Concert(Table): """ value_type = datetime + timedelta_delegate = TimedeltaDelegate() def __init__( self, default: TimestampArg = TimestampNow(), **kwargs @@ -782,6 +906,28 @@ def __init__( kwargs.update({"default": default}) super().__init__(**kwargs) + ########################################################################### + # For update queries + + def __add__(self, value: timedelta) -> QueryString: + return self.timedelta_delegate.get_querystring( + column=self, + operator="+", + value=value, + engine_type=self._meta.engine_type, + ) + + def __radd__(self, value: timedelta) -> QueryString: + return self.__add__(value) + + def __sub__(self, value: timedelta) -> QueryString: + return self.timedelta_delegate.get_querystring( + column=self, + operator="-", + value=value, + engine_type=self._meta.engine_type, + ) + ########################################################################### # Descriptors @@ -833,6 +979,7 @@ class Concert(Table): """ value_type = datetime + timedelta_delegate = TimedeltaDelegate() def __init__( self, default: TimestamptzArg = TimestamptzNow(), **kwargs @@ -851,6 +998,28 @@ def __init__( kwargs.update({"default": default}) super().__init__(**kwargs) + ########################################################################### + # For update queries + + def __add__(self, value: timedelta) -> QueryString: + return self.timedelta_delegate.get_querystring( + column=self, + operator="+", + value=value, + engine_type=self._meta.engine_type, + ) + + def __radd__(self, value: timedelta) -> QueryString: + return self.__add__(value) + + def __sub__(self, value: timedelta) -> QueryString: + return self.timedelta_delegate.get_querystring( + column=self, + operator="-", + value=value, + engine_type=self._meta.engine_type, + ) + ########################################################################### # Descriptors @@ -894,6 +1063,7 @@ class Concert(Table): """ value_type = date + timedelta_delegate = TimedeltaDelegate() def __init__(self, default: DateArg = DateNow(), **kwargs) -> None: self._validate_default(default, DateArg.__args__) # type: ignore @@ -908,6 +1078,28 @@ def __init__(self, default: DateArg = DateNow(), **kwargs) -> None: kwargs.update({"default": default}) super().__init__(**kwargs) + ########################################################################### + # For update queries + + def __add__(self, value: timedelta) -> QueryString: + return self.timedelta_delegate.get_querystring( + column=self, + operator="+", + value=value, + engine_type=self._meta.engine_type, + ) + + def __radd__(self, value: timedelta) -> QueryString: + return self.__add__(value) + + def __sub__(self, value: timedelta) -> QueryString: + return self.timedelta_delegate.get_querystring( + column=self, + operator="-", + value=value, + engine_type=self._meta.engine_type, + ) + ########################################################################### # Descriptors @@ -951,6 +1143,7 @@ class Concert(Table): """ value_type = time + timedelta_delegate = TimedeltaDelegate() def __init__(self, default: TimeArg = TimeNow(), **kwargs) -> None: self._validate_default(default, TimeArg.__args__) # type: ignore @@ -962,6 +1155,28 @@ def __init__(self, default: TimeArg = TimeNow(), **kwargs) -> None: kwargs.update({"default": default}) super().__init__(**kwargs) + ########################################################################### + # For update queries + + def __add__(self, value: timedelta) -> QueryString: + return self.timedelta_delegate.get_querystring( + column=self, + operator="+", + value=value, + engine_type=self._meta.engine_type, + ) + + def __radd__(self, value: timedelta) -> QueryString: + return self.__add__(value) + + def __sub__(self, value: timedelta) -> QueryString: + return self.timedelta_delegate.get_querystring( + column=self, + operator="-", + value=value, + engine_type=self._meta.engine_type, + ) + ########################################################################### # Descriptors @@ -1005,6 +1220,7 @@ class Concert(Table): """ value_type = timedelta + timedelta_delegate = TimedeltaDelegate() def __init__( self, default: IntervalArg = IntervalCustom(), **kwargs @@ -1020,16 +1236,38 @@ def __init__( @property def column_type(self): - engine_type = self._meta.table._meta.db.engine_type + engine_type = self._meta.engine_type if engine_type == "postgres": return "INTERVAL" elif engine_type == "sqlite": # We can't use 'INTERVAL' because the type affinity in SQLite would - # make it an integer - but we need a numeric field. + # make it an integer - but we need a text field. # https://sqlite.org/datatype3.html#determination_of_column_affinity return "SECONDS" raise Exception("Unrecognized engine type") + ########################################################################### + # For update queries + + def __add__(self, value: timedelta) -> QueryString: + return self.timedelta_delegate.get_querystring( + column=self, + operator="+", + value=value, + engine_type=self._meta.engine_type, + ) + + def __radd__(self, value: timedelta) -> QueryString: + return self.__add__(value) + + def __sub__(self, value: timedelta) -> QueryString: + return self.timedelta_delegate.get_querystring( + column=self, + operator="-", + value=value, + engine_type=self._meta.engine_type, + ) + ########################################################################### # Descriptors @@ -2062,7 +2300,7 @@ class Token(Table): @property def column_type(self): - engine_type = self._meta.table._meta.db.engine_type + engine_type = self._meta.engine_type if engine_type == "postgres": return "BYTEA" elif engine_type == "sqlite": @@ -2208,7 +2446,7 @@ def __init__( @property def column_type(self): - engine_type = self._meta.table._meta.db.engine_type + engine_type = self._meta.engine_type if engine_type == "postgres": return f"{self.base_column.column_type}[]" elif engine_type == "sqlite": @@ -2232,7 +2470,7 @@ def __getitem__(self, value: int) -> Array: """ # noqa: E501 - engine_type = self._meta.table._meta.db.engine_type + engine_type = self._meta.engine_type if engine_type != "postgres": raise ValueError( "Only Postgres supports array indexing currently." @@ -2270,7 +2508,7 @@ def any(self, value: t.Any) -> Where: >>> await Ticket.select().where(Ticket.seat_numbers.any(510)) """ - engine_type = self._meta.table._meta.db.engine_type + engine_type = self._meta.engine_type if engine_type == "postgres": return Where(column=self, value=value, operator=ArrayAny) @@ -2288,7 +2526,7 @@ def all(self, value: t.Any) -> Where: >>> await Ticket.select().where(Ticket.seat_numbers.all(510)) """ - engine_type = self._meta.table._meta.db.engine_type + engine_type = self._meta.engine_type if engine_type == "postgres": return Where(column=self, value=value, operator=ArrayAll) diff --git a/piccolo/columns/operators/__init__.py b/piccolo/columns/operators/__init__.py index ee0805499..603264170 100644 --- a/piccolo/columns/operators/__init__.py +++ b/piccolo/columns/operators/__init__.py @@ -16,4 +16,4 @@ NotLike, ) from .math import Add, Divide, Multiply, Subtract -from .string import ConcatPostgres, ConcatSQLite +from .string import Concat diff --git a/piccolo/columns/operators/string.py b/piccolo/columns/operators/string.py index 51b0c745f..1c979f94b 100644 --- a/piccolo/columns/operators/string.py +++ b/piccolo/columns/operators/string.py @@ -5,9 +5,5 @@ class StringOperator(Operator): pass -class ConcatPostgres(StringOperator): - template = "CONCAT({value_1}, {value_2})" - - -class ConcatSQLite(StringOperator): +class Concat(StringOperator): template = "{value_1} || {value_2}" diff --git a/tests/base.py b/tests/base.py index 0025c0551..560da9b90 100644 --- a/tests/base.py +++ b/tests/base.py @@ -15,12 +15,21 @@ ENGINE = engine_finder() + +def is_running_postgres(): + return isinstance(ENGINE, PostgresEngine) + + +def is_running_sqlite(): + return isinstance(ENGINE, SQLiteEngine) + + postgres_only = pytest.mark.skipif( - not isinstance(ENGINE, PostgresEngine), reason="Only running for Postgres" + not is_running_postgres(), reason="Only running for Postgres" ) sqlite_only = pytest.mark.skipif( - not isinstance(ENGINE, SQLiteEngine), reason="Only running for SQLite" + not is_running_sqlite(), reason="Only running for SQLite" ) unix_only = pytest.mark.skipif( diff --git a/tests/columns/test_varchar.py b/tests/columns/test_varchar.py index cf258a9d1..bfd24f9e9 100644 --- a/tests/columns/test_varchar.py +++ b/tests/columns/test_varchar.py @@ -17,7 +17,7 @@ class TestVarchar(TestCase): https://www.sqlite.org/faq.html#q9 - Might consider enforncing this at the ORM level instead in the future. + Might consider enforcing this at the ORM level instead in the future. """ def setUp(self): diff --git a/tests/table/test_update.py b/tests/table/test_update.py index a57966357..60cb787af 100644 --- a/tests/table/test_update.py +++ b/tests/table/test_update.py @@ -1,5 +1,22 @@ -from tests.base import DBTestCase -from tests.example_apps.music.tables import Band, Poster +import dataclasses +import datetime +import typing as t +from unittest import TestCase + +from piccolo.columns.base import Column +from piccolo.columns.column_types import ( + Date, + Integer, + Interval, + Text, + Timestamp, + Timestamptz, + Varchar, +) +from piccolo.querystring import QueryString +from piccolo.table import Table +from tests.base import DBTestCase, sqlite_only +from tests.example_apps.music.tables import Band class TestUpdate(DBTestCase): @@ -89,175 +106,404 @@ def test_update_values_with_kwargs(self): self.check_response() -class TestIntUpdateOperators(DBTestCase): - def test_add(self): - self.insert_row() - - Band.update( - {Band.popularity: Band.popularity + 10}, force=True - ).run_sync() - - response = Band.select(Band.popularity).first().run_sync() - - self.assertEqual(response["popularity"], 1010) - - def test_add_column(self): - self.insert_row() - - Band.update( - {Band.popularity: Band.popularity + Band.popularity}, force=True - ).run_sync() - - response = Band.select(Band.popularity).first().run_sync() - - self.assertEqual(response["popularity"], 2000) - - def test_radd(self): - self.insert_row() - - Band.update( - {Band.popularity: 10 + Band.popularity}, force=True - ).run_sync() - - response = Band.select(Band.popularity).first().run_sync() - - self.assertEqual(response["popularity"], 1010) - - def test_sub(self): - self.insert_row() - - Band.update( - {Band.popularity: Band.popularity - 10}, force=True - ).run_sync() - - response = Band.select(Band.popularity).first().run_sync() - - self.assertEqual(response["popularity"], 990) - - def test_rsub(self): - self.insert_row() - - Band.update( - {Band.popularity: 1100 - Band.popularity}, force=True - ).run_sync() - - response = Band.select(Band.popularity).first().run_sync() - - self.assertEqual(response["popularity"], 100) - - def test_mul(self): - self.insert_row() - - Band.update( - {Band.popularity: Band.popularity * 2}, force=True - ).run_sync() - - response = Band.select(Band.popularity).first().run_sync() - - self.assertEqual(response["popularity"], 2000) - - def test_rmul(self): - self.insert_row() - - Band.update( - {Band.popularity: 2 * Band.popularity}, force=True - ).run_sync() - - response = Band.select(Band.popularity).first().run_sync() - - self.assertEqual(response["popularity"], 2000) - - def test_div(self): - self.insert_row() - - Band.update( - {Band.popularity: Band.popularity / 10}, force=True - ).run_sync() - - response = Band.select(Band.popularity).first().run_sync() - - self.assertEqual(response["popularity"], 100) - - def test_rdiv(self): - self.insert_row() - - Band.update( - {Band.popularity: 1000 / Band.popularity}, force=True - ).run_sync() - - response = Band.select(Band.popularity).first().run_sync() - - self.assertEqual(response["popularity"], 1) - - -class TestVarcharUpdateOperators(DBTestCase): - def test_add(self): - self.insert_row() - - Band.update({Band.name: Band.name + "!!!"}, force=True).run_sync() - - response = Band.select(Band.name).first().run_sync() - - self.assertEqual(response["name"], "Pythonistas!!!") - - def test_add_column(self): - self.insert_row() - - Band.update({Band.name: Band.name + Band.name}, force=True).run_sync() - - response = Band.select(Band.name).first().run_sync() - - self.assertEqual(response["name"], "PythonistasPythonistas") - - def test_radd(self): - self.insert_row() - - Band.update({Band.name: "!!!" + Band.name}, force=True).run_sync() - - response = Band.select(Band.name).first().run_sync() - - self.assertEqual(response["name"], "!!!Pythonistas") - - -class TestTextUpdateOperators(DBTestCase): +############################################################################### +# Test operators + + +class MyTable(Table): + integer = Integer(null=True) + timestamp = Timestamp(null=True) + timestamptz = Timestamptz(null=True) + date = Date(null=True) + interval = Interval(null=True) + varchar = Varchar(null=True) + text = Text(null=True) + + +INITIAL_DATETIME = datetime.datetime( + year=2022, month=1, day=1, hour=21, minute=0 +) +INITIAL_INTERVAL = datetime.timedelta(days=1, hours=1, minutes=1) + +DATETIME_DELTA = datetime.timedelta( + days=1, hours=1, minutes=1, seconds=30, microseconds=1000 +) +DATE_DELTA = datetime.timedelta(days=1) + + +@dataclasses.dataclass +class OperatorTestCase: + description: str + column: Column + initial: t.Any + querystring: QueryString + expected: t.Any + + +TEST_CASES = [ + # Text + OperatorTestCase( + description="Add Text", + column=MyTable.text, + initial="Pythonistas", + querystring=MyTable.text + "!!!", + expected="Pythonistas!!!", + ), + OperatorTestCase( + description="Add Text columns", + column=MyTable.text, + initial="Pythonistas", + querystring=MyTable.text + MyTable.text, + expected="PythonistasPythonistas", + ), + OperatorTestCase( + description="Reverse add Text", + column=MyTable.text, + initial="Pythonistas", + querystring="!!!" + MyTable.text, + expected="!!!Pythonistas", + ), + OperatorTestCase( + description="Text is null", + column=MyTable.text, + initial=None, + querystring=MyTable.text + "!!!", + expected=None, + ), + OperatorTestCase( + description="Reverse Text is null", + column=MyTable.text, + initial=None, + querystring="!!!" + MyTable.text, + expected=None, + ), + # Varchar + OperatorTestCase( + description="Add Varchar", + column=MyTable.varchar, + initial="Pythonistas", + querystring=MyTable.varchar + "!!!", + expected="Pythonistas!!!", + ), + OperatorTestCase( + description="Add Varchar columns", + column=MyTable.varchar, + initial="Pythonistas", + querystring=MyTable.varchar + MyTable.varchar, + expected="PythonistasPythonistas", + ), + OperatorTestCase( + description="Reverse add Varchar", + column=MyTable.varchar, + initial="Pythonistas", + querystring="!!!" + MyTable.varchar, + expected="!!!Pythonistas", + ), + OperatorTestCase( + description="Varchar is null", + column=MyTable.varchar, + initial=None, + querystring=MyTable.varchar + "!!!", + expected=None, + ), + OperatorTestCase( + description="Reverse Varchar is null", + column=MyTable.varchar, + initial=None, + querystring="!!!" + MyTable.varchar, + expected=None, + ), + # Integer + OperatorTestCase( + description="Add Integer", + column=MyTable.integer, + initial=1000, + querystring=MyTable.integer + 10, + expected=1010, + ), + OperatorTestCase( + description="Reverse add Integer", + column=MyTable.integer, + initial=1000, + querystring=10 + MyTable.integer, + expected=1010, + ), + OperatorTestCase( + description="Add Integer colums together", + column=MyTable.integer, + initial=1000, + querystring=MyTable.integer + MyTable.integer, + expected=2000, + ), + OperatorTestCase( + description="Subtract Integer", + column=MyTable.integer, + initial=1000, + querystring=MyTable.integer - 10, + expected=990, + ), + OperatorTestCase( + description="Reverse subtract Integer", + column=MyTable.integer, + initial=1000, + querystring=2000 - MyTable.integer, + expected=1000, + ), + OperatorTestCase( + description="Multiply Integer", + column=MyTable.integer, + initial=1000, + querystring=MyTable.integer * 2, + expected=2000, + ), + OperatorTestCase( + description="Reverse multiply Integer", + column=MyTable.integer, + initial=1000, + querystring=2 * MyTable.integer, + expected=2000, + ), + OperatorTestCase( + description="Divide Integer", + column=MyTable.integer, + initial=1000, + querystring=MyTable.integer / 10, + expected=100, + ), + OperatorTestCase( + description="Reverse divide Integer", + column=MyTable.integer, + initial=1000, + querystring=2000 / MyTable.integer, + expected=2, + ), + OperatorTestCase( + description="Integer is null", + column=MyTable.integer, + initial=None, + querystring=MyTable.integer + 1, + expected=None, + ), + OperatorTestCase( + description="Reverse Integer is null", + column=MyTable.integer, + initial=None, + querystring=1 + MyTable.integer, + expected=None, + ), + # Timestamp + OperatorTestCase( + description="Add Timestamp", + column=MyTable.timestamp, + initial=INITIAL_DATETIME, + querystring=MyTable.timestamp + DATETIME_DELTA, + expected=datetime.datetime( + year=2022, + month=1, + day=2, + hour=22, + minute=1, + second=30, + microsecond=1000, + ), + ), + OperatorTestCase( + description="Reverse add Timestamp", + column=MyTable.timestamp, + initial=INITIAL_DATETIME, + querystring=DATETIME_DELTA + MyTable.timestamp, + expected=datetime.datetime( + year=2022, + month=1, + day=2, + hour=22, + minute=1, + second=30, + microsecond=1000, + ), + ), + OperatorTestCase( + description="Subtract Timestamp", + column=MyTable.timestamp, + initial=INITIAL_DATETIME, + querystring=MyTable.timestamp - DATETIME_DELTA, + expected=datetime.datetime( + year=2021, + month=12, + day=31, + hour=19, + minute=58, + second=29, + microsecond=999000, + ), + ), + OperatorTestCase( + description="Timestamp is null", + column=MyTable.timestamp, + initial=None, + querystring=MyTable.timestamp + DATETIME_DELTA, + expected=None, + ), + # Timestamptz + OperatorTestCase( + description="Add Timestamptz", + column=MyTable.timestamptz, + initial=INITIAL_DATETIME, + querystring=MyTable.timestamptz + DATETIME_DELTA, + expected=datetime.datetime( + year=2022, + month=1, + day=2, + hour=22, + minute=1, + second=30, + microsecond=1000, + tzinfo=datetime.timezone.utc, + ), + ), + OperatorTestCase( + description="Reverse add Timestamptz", + column=MyTable.timestamptz, + initial=INITIAL_DATETIME, + querystring=DATETIME_DELTA + MyTable.timestamptz, + expected=datetime.datetime( + year=2022, + month=1, + day=2, + hour=22, + minute=1, + second=30, + microsecond=1000, + tzinfo=datetime.timezone.utc, + ), + ), + OperatorTestCase( + description="Subtract Timestamptz", + column=MyTable.timestamptz, + initial=INITIAL_DATETIME, + querystring=MyTable.timestamptz - DATETIME_DELTA, + expected=datetime.datetime( + year=2021, + month=12, + day=31, + hour=19, + minute=58, + second=29, + microsecond=999000, + tzinfo=datetime.timezone.utc, + ), + ), + OperatorTestCase( + description="Timestamptz is null", + column=MyTable.timestamptz, + initial=None, + querystring=MyTable.timestamptz + DATETIME_DELTA, + expected=None, + ), + # Date + OperatorTestCase( + description="Add Date", + column=MyTable.date, + initial=INITIAL_DATETIME, + querystring=MyTable.date + DATE_DELTA, + expected=datetime.date(year=2022, month=1, day=2), + ), + OperatorTestCase( + description="Reverse add Date", + column=MyTable.date, + initial=INITIAL_DATETIME, + querystring=DATE_DELTA + MyTable.date, + expected=datetime.date(year=2022, month=1, day=2), + ), + OperatorTestCase( + description="Subtract Date", + column=MyTable.date, + initial=INITIAL_DATETIME, + querystring=MyTable.date - DATE_DELTA, + expected=datetime.date(year=2021, month=12, day=31), + ), + OperatorTestCase( + description="Date is null", + column=MyTable.date, + initial=None, + querystring=MyTable.date + DATE_DELTA, + expected=None, + ), + # Interval + OperatorTestCase( + description="Add Interval", + column=MyTable.interval, + initial=INITIAL_INTERVAL, + querystring=MyTable.interval + DATETIME_DELTA, + expected=datetime.timedelta(days=2, seconds=7350, microseconds=1000), + ), + OperatorTestCase( + description="Reverse add Interval", + column=MyTable.interval, + initial=INITIAL_INTERVAL, + querystring=DATETIME_DELTA + MyTable.interval, + expected=datetime.timedelta(days=2, seconds=7350, microseconds=1000), + ), + OperatorTestCase( + description="Subtract Interval", + column=MyTable.interval, + initial=INITIAL_INTERVAL, + querystring=MyTable.interval - DATETIME_DELTA, + expected=datetime.timedelta( + days=-1, seconds=86369, microseconds=999000 + ), + ), + OperatorTestCase( + description="Interval is null", + column=MyTable.interval, + initial=None, + querystring=MyTable.interval + DATETIME_DELTA, + expected=None, + ), +] + + +class TestOperators(TestCase): def setUp(self): - super().setUp() - Poster(content="Join us for this amazing show").save().run_sync() + MyTable.create_table().run_sync() - def test_add(self): - Poster.update( - {Poster.content: Poster.content + "!!!"}, force=True - ).run_sync() + def tearDown(self): + MyTable.alter().drop_table().run_sync() - response = Poster.select(Poster.content).first().run_sync() + def test_operators(self): + for test_case in TEST_CASES: + print(test_case.description) - self.assertEqual( - response["content"], "Join us for this amazing show!!!" - ) + # Create the initial data in the database. + instance = MyTable() + setattr(instance, test_case.column._meta.name, test_case.initial) + instance.save().run_sync() - def test_add_column(self): - self.insert_row() + # Apply the update. + MyTable.update( + {test_case.column: test_case.querystring}, force=True + ).run_sync() - Poster.update( - {Poster.content: Poster.content + Poster.content}, force=True - ).run_sync() + # Make sure the value returned from the database is correct. + new_value = getattr( + MyTable.objects().first().run_sync(), + test_case.column._meta.name, + ) - response = Poster.select(Poster.content).first().run_sync() + self.assertEqual( + new_value, test_case.expected, msg=test_case.description + ) - self.assertEqual( - response["content"], - "Join us for this amazing show" * 2, - ) - - def test_radd(self): - self.insert_row() - - Poster.update( - {Poster.content: "!!!" + Poster.content}, force=True - ).run_sync() + # Clean up + MyTable.delete(force=True).run_sync() - response = Poster.select(Poster.content).first().run_sync() - - self.assertEqual( - response["content"], "!!!Join us for this amazing show" - ) + @sqlite_only + def test_edge_cases(self): + """ + Some usecases aren't supported by SQLite, and should raise a + ``ValueError``. + """ + with self.assertRaises(ValueError): + # An error should be raised because we can't save at this level + # of resolution - 1 millisecond is the minimum. + MyTable.timestamp + datetime.timedelta(microseconds=1) From e0041b9aa4b3a5b142e0daa4693d6219158a9faa Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 13 Apr 2022 21:45:59 +0100 Subject: [PATCH 303/727] bumped version --- CHANGES.rst | 27 +++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index f8a090772..1d0833b01 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,33 @@ Changes ======= +0.74.0 +------ + +We've had the ability to bulk modify rows for a while. Here we append ``'!!!'`` +to each band's name: + +.. code-block:: python + + >>> await Band.update({Band.name: Band.name + '!!!'}, force=True) + +It only worked for some columns - ``Varchar``, ``Text``, ``Integer`` etc. + +We now allow ``Date``, ``Timestamp``, ``Timestamptz`` and ``Interval`` columns +to be bulk modified using a ``timedelta``. Here we modify each concert's start +date, so it's one day later: + +.. code-block:: python + + >>> Concert.update( + ... {Concert.starts: Concert.starts + timedelta(days=1)}, + ... force=True + ... ) + +Thanks to @theelderbeever for suggesting this feature. + +------------------------------------------------------------------------------- + 0.73.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 91ae994c7..bfb3a167a 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.73.0" +__VERSION__ = "0.74.0" From a5e68d9a312c039d8906f9c7cea84a223bf35eef Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 13 Apr 2022 21:48:39 +0100 Subject: [PATCH 304/727] fix typo in changelog --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 1d0833b01..9d8835602 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -19,7 +19,7 @@ date, so it's one day later: .. code-block:: python - >>> Concert.update( + >>> await Concert.update( ... {Concert.starts: Concert.starts + timedelta(days=1)}, ... force=True ... ) From 763530789312eae48946a5c99a7512e15f52dea3 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 16 Apr 2022 21:05:41 +0100 Subject: [PATCH 305/727] fix broken link --- docs/src/piccolo/contributing/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/piccolo/contributing/index.rst b/docs/src/piccolo/contributing/index.rst index 9e22fdd68..a9d20f0de 100644 --- a/docs/src/piccolo/contributing/index.rst +++ b/docs/src/piccolo/contributing/index.rst @@ -42,7 +42,7 @@ Code style Piccolo uses `Black `_ for formatting, preferably with a max line length of 79, to keep it consistent -with `PEP8 `_ . +with `PEP8 `_ . You can configure `VSCode `_ by modifying ``settings.json`` as follows: From 83afa048f256577baf1c34b5d5fca23d0bf4e68d Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 23 Apr 2022 09:31:27 +0100 Subject: [PATCH 306/727] Update uvloop.txt (#492) --- requirements/extras/uvloop.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/extras/uvloop.txt b/requirements/extras/uvloop.txt index a51c94482..8bb45b44c 100644 --- a/requirements/extras/uvloop.txt +++ b/requirements/extras/uvloop.txt @@ -1 +1 @@ -uvloop>=0.12.0 +uvloop>=0.12.0; sys_platform != "win32" From 9388f2854273b08a74c873578d4dea0d3c74b305 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 23 Apr 2022 09:33:18 +0100 Subject: [PATCH 307/727] bumped version --- CHANGES.rst | 8 ++++++++ piccolo/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 9d8835602..7cde47263 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,14 @@ Changes ======= +0.74.1 +------ + +When using ``pip install piccolo[all]`` on Windows it would fail because uvloop +isn't supported. Thanks to @jack1142 for reporting this issue. + +------------------------------------------------------------------------------- + 0.74.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index bfb3a167a..822c38762 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.74.0" +__VERSION__ = "0.74.1" From 8e6e4ff34126dc0e0f0f2dfad1766e7214af03b6 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 26 Apr 2022 11:35:09 +0100 Subject: [PATCH 308/727] add docs for read only Postgres databases (#495) --- piccolo/engine/postgres.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/piccolo/engine/postgres.py b/piccolo/engine/postgres.py index 2c9330e76..5b6bb6d54 100644 --- a/piccolo/engine/postgres.py +++ b/piccolo/engine/postgres.py @@ -220,7 +220,8 @@ class PostgresEngine(Engine): :param extensions: When the engine starts, it will try and create these extensions - in Postgres. + in Postgres. If you're using a read only database, set this value to an + empty tuple ``()``. :param log_queries: If ``True``, all SQL and DDL statements are printed out before being @@ -238,11 +239,14 @@ class PostgresEngine(Engine): config={ 'database': 'main_db', host: 'read_replicate.my_db.com' - } + }, + extensions=() ) } ) + Note how we set ``extensions=()``, because it's a read only database. + When executing a query, you can specify one of these nodes instead of the main database. For example:: @@ -265,12 +269,10 @@ class PostgresEngine(Engine): def __init__( self, config: t.Dict[str, t.Any], - extensions: t.Sequence[str] = None, + extensions: t.Sequence[str] = ("uuid-ossp",), log_queries: bool = False, extra_nodes: t.Dict[str, PostgresEngine] = None, ) -> None: - if extensions is None: - extensions = ["uuid-ossp"] if extra_nodes is None: extra_nodes = {} From 2c23a56aebba25eeb00fec1f9a30ffdab0c1e0ff Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 27 Apr 2022 19:12:04 +0100 Subject: [PATCH 309/727] return `None` if column isn't recognised (#497) --- piccolo/apps/migrations/auto/schema_differ.py | 3 +++ piccolo/apps/migrations/auto/serialisation.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/piccolo/apps/migrations/auto/schema_differ.py b/piccolo/apps/migrations/auto/schema_differ.py index 97dfd7d5c..2427c44a5 100644 --- a/piccolo/apps/migrations/auto/schema_differ.py +++ b/piccolo/apps/migrations/auto/schema_differ.py @@ -345,6 +345,7 @@ def alter_columns(self) -> AlterStatements: expect_conflict_with_global_name=getattr( UniqueGlobalNames, f"COLUMN_{alter_column.column_class.__name__.upper()}", # noqa: E501 + None, ), ) ) @@ -425,6 +426,7 @@ def add_columns(self) -> AlterStatements: expect_conflict_with_global_name=getattr( UniqueGlobalNames, f"COLUMN_{column_class.__name__.upper()}", + None, ), ) ) @@ -480,6 +482,7 @@ def new_table_columns(self) -> AlterStatements: expect_conflict_with_global_name=getattr( UniqueGlobalNames, f"COLUMN_{column.__class__.__name__.upper()}", + None, ), ) ) diff --git a/piccolo/apps/migrations/auto/serialisation.py b/piccolo/apps/migrations/auto/serialisation.py index ea2f06248..2b45dfba7 100644 --- a/piccolo/apps/migrations/auto/serialisation.py +++ b/piccolo/apps/migrations/auto/serialisation.py @@ -525,6 +525,7 @@ def serialise_params(params: t.Dict[str, t.Any]) -> SerialisedParams: expect_conflict_with_global_name=getattr( UniqueGlobalNames, f"COLUMN_{column_class_name.upper()}", + None, ), ) ) @@ -679,6 +680,7 @@ def serialise_params(params: t.Dict[str, t.Any]) -> SerialisedParams: expect_conflict_with_global_name=getattr( UniqueGlobalNames, f"COLUMN_{primary_key_class.__name__.upper()}", + None, ), ) ) From 32bd3b1d763e706c5b382c2e634c10ef1709ac57 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 27 Apr 2022 19:19:36 +0100 Subject: [PATCH 310/727] bumped version --- CHANGES.rst | 22 ++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 7cde47263..d964bad4c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,28 @@ Changes ======= +0.74.2 +------ + +If a user created a custom ``Column`` subclass, then migrations would fail. +For example: + +.. code-block:: python + + class CustomColumn(Varchar): + def __init__(self, custom_arg: str = '', *args, **kwargs): + self.custom_arg = custom_arg + super().__init__(*args, **kwargs) + + @property + def column_type(self): + return 'VARCHAR' + +See `PR 497 `_. Thanks to +@WintonLi for reporting this issue. + +------------------------------------------------------------------------------- + 0.74.1 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 822c38762..a417497d9 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.74.1" +__VERSION__ = "0.74.2" From 352df4f433fefdaa054f9cfa0e154bb46ecd06e6 Mon Sep 17 00:00:00 2001 From: WintonLi Date: Fri, 29 Apr 2022 23:43:51 +0100 Subject: [PATCH 311/727] =?UTF-8?q?=F0=9F=93=9D=20Add=20custom=20column=20?= =?UTF-8?q?type=20to=20doc=20(#499)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add custom column types to doc * change some wording * Add some dashes under the section title of custom column types to make they have the same length Co-authored-by: winton --- docs/src/piccolo/schema/advanced.rst | 55 ++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/docs/src/piccolo/schema/advanced.rst b/docs/src/piccolo/schema/advanced.rst index 510d4913c..3ca7c5d1f 100644 --- a/docs/src/piccolo/schema/advanced.rst +++ b/docs/src/piccolo/schema/advanced.rst @@ -231,3 +231,58 @@ is not present, it will be reflected and returned. .. hint:: Reflection will automatically create ``Table`` classes for referenced tables too. For example, if ``Table1`` references ``Table2``, then ``Table2`` will automatically be added to ``TableStorage``. + +------------------------------------------------------------------------------- + +How to create custom column types +--------------------------------- + +Sometimes, the column types shipped with Piccolo don't meet your requirements, and you +will need to define your own column types. + +Generally there are two ways to define your own column types: + +* Create a subclass of an existing column type; or +* Directly subclass the :ref:`Column ` class. + +Try to use the first method whenever possible because it is more straightforward and +can often save you some work. Otherwise, subclass :ref:`Column `. + +**Example** + +In this example, we create a column type called ``MyColumn``, which is fundamentally +an ``Integer`` type but has a custom attribute ``custom_attr``: + +.. code-block:: python + + from piccolo.columns import Integer + + class MyColumn(Integer): + def __init__(self, *args, custom_attr: str = '', **kwargs): + self.custom_attr = custom_attr + super().__init__(*args, **kwargs) + + @property + def column_type(self): + return 'INTEGER' + +.. hint:: It is **important** to specify the ``column_type`` property, which + tells the database engine the **actual** storage type of the custom + column. + +Now we can use ``MyColumn`` in our table: + +.. code-block:: python + + from piccolo.table import Table + + class MyTable(Table): + my_col = MyColumn(custom_attr='foo') + ... + +And later we can retrieve the value of the attribute: + +.. code-block:: python + + >>> MyTable.my_col.custom_attr + 'foo' \ No newline at end of file From 7f49ce643c17659019f5bfcf688a1515f0a158ad Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 30 Apr 2022 20:03:33 +0100 Subject: [PATCH 312/727] fix migrations for tables with arrays of `BigInt` (#501) --- piccolo/table.py | 4 ++++ .../auto/integration/test_migrations.py | 15 +++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/piccolo/table.py b/piccolo/table.py index b0e732271..9ff261023 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -10,6 +10,7 @@ from piccolo.columns.column_types import ( JSON, JSONB, + Array, ForeignKey, Secret, Serial, @@ -217,6 +218,9 @@ def __init_subclass__( column._meta._name = attribute_name column._meta._table = cls + if isinstance(column, Array): + column.base_column._meta._table = cls + if isinstance(column, Secret): secret_columns.append(column) diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index ae325e6b4..64178fc6f 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -598,6 +598,21 @@ def test_array_column_varchar(self): ), ) + def test_array_column_bigint(self): + """ + There was a bug with using an array of ``BigInt`` - see issue 500 on + GitHub. It's because ``BigInt`` requires access to the parent table to + determine what the column type is. + """ + self._test_migrations( + table_snapshots=[ + [self.table(column)] + for column in [ + Array(base_column=BigInt()), + ] + ] + ) + ########################################################################### # We deliberately don't test setting JSON or JSONB columns as indexes, as From 570b395d3adc81f344802484796c715aebdd39f3 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 30 Apr 2022 20:09:51 +0100 Subject: [PATCH 313/727] bumped version --- CHANGES.rst | 21 +++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index d964bad4c..5a447a554 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,27 @@ Changes ======= +0.74.3 +------ + +If you had a table containing an array of ``BigInt``, then migrations could +fail: + +.. code-block:: python + + from piccolo.table import Table + from piccolo.columns.column_types import Array, BigInt + + class MyTable(Table): + my_column = Array(base_column=BigInt()) + +It's because the ``BigInt`` base column needs access to the parent table to +know if it's targeting Postgres or SQLite. See `PR 501 `_. + +Thanks to @cheesycod for reporting this issue. + +------------------------------------------------------------------------------- + 0.74.2 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index a417497d9..638e15e25 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.74.2" +__VERSION__ = "0.74.3" From 1996a8c6ac6f1c27d500ec9ad108d60fd39215a0 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 13 May 2022 05:08:16 +0100 Subject: [PATCH 314/727] set `in_pool=False` by default for `run_sync` (#509) --- piccolo/query/base.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/piccolo/query/base.py b/piccolo/query/base.py index ba1b6ff58..535e98049 100644 --- a/piccolo/query/base.py +++ b/piccolo/query/base.py @@ -175,8 +175,9 @@ async def run(self, node: t.Optional[str] = None, in_pool: bool = True): If specified, run this query against another database node. Only available in Postgres. See :class:`PostgresEngine `. :param in_pool: - Whether to run this in a connection pool if one is running. This is - mostly just for debugging - use a connection pool where possible. + Whether to run this in a connection pool if one is available. This + is mostly just for debugging - use a connection pool where + possible. """ # noqa: E501 self._validate() @@ -211,16 +212,21 @@ async def run(self, node: t.Optional[str] = None, in_pool: bool = True): responses.append(await self._process_results(results)) return responses - def run_sync(self, timed=False, *args, **kwargs): + def run_sync(self, timed=False, in_pool=False, *args, **kwargs): """ A convenience method for running the coroutine synchronously. :param timed: If ``True``, the time taken to run the query is printed out. Useful for debugging. + :param in_pool: + Whether to run this in a connection pool if one is available. Set + to ``False`` by default, because if an app uses ``run`` and + ``run_sync`` in the same app, it can cause errors. See + `issue 505 `_. """ - coroutine = self.run(*args, **kwargs) + coroutine = self.run(in_pool=in_pool, *args, **kwargs) if not timed: return run_sync(coroutine) From 2fde648452cea8d5f4c7deaf0974989abeee86b3 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 13 May 2022 05:36:35 +0100 Subject: [PATCH 315/727] output warning if a matching trigger can't be found (#510) Before we were raising a `ValueError`, which stopped the schema generation. --- piccolo/apps/schema/commands/generate.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index ed7fbf9c0..566f5a547 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -162,7 +162,7 @@ def get_column_triggers(self, column_name: str) -> t.List[Trigger]: def get_column_ref_trigger( self, column_name: str, references_table: str - ) -> Trigger: + ) -> t.Optional[Trigger]: for trigger in self.triggers: if ( trigger.column_name == column_name @@ -170,7 +170,7 @@ def get_column_ref_trigger( ): return trigger - raise ValueError("No matching trigger found") + return None @dataclasses.dataclass @@ -244,6 +244,8 @@ class OutputSchema: e.g. ["some_table.some_column unrecognised_type"] :param index_warnings: Warnings if column indexes can't be parsed. + :param trigger_warnings: + Warnings if triggers for certain columns can't be found. :param tables: e.g. ["class MyTable(Table): ..."] """ @@ -251,6 +253,7 @@ class OutputSchema: imports: t.List[str] = dataclasses.field(default_factory=list) warnings: t.List[str] = dataclasses.field(default_factory=list) index_warnings: t.List[str] = dataclasses.field(default_factory=list) + trigger_warnings: t.List[str] = dataclasses.field(default_factory=list) tables: t.List[t.Type[Table]] = dataclasses.field(default_factory=list) def get_table_with_name(self, tablename: str) -> t.Optional[t.Type[Table]]: @@ -271,6 +274,7 @@ def __radd__(self, value: OutputSchema) -> OutputSchema: value.imports.extend(self.imports) value.warnings.extend(self.warnings) value.index_warnings.extend(self.index_warnings) + value.trigger_warnings.extend(self.trigger_warnings) value.tables.extend(self.tables) return value @@ -278,6 +282,7 @@ def __add__(self, value: OutputSchema) -> OutputSchema: self.imports.extend(value.imports) self.warnings.extend(value.warnings) self.index_warnings.extend(value.index_warnings) + self.trigger_warnings.extend(value.trigger_warnings) self.tables.extend(value.tables) return self @@ -736,6 +741,10 @@ async def create_table_class_from_db( if trigger: kwargs["on_update"] = ONUPDATE_MAP[trigger.on_update] kwargs["on_delete"] = ONDELETE_MAP[trigger.on_delete] + else: + output_schema.trigger_warnings.append( + f"{tablename}.{column_name}" + ) output_schema = sum( # type: ignore [output_schema, referenced_output_schema] # type: ignore @@ -900,6 +909,17 @@ async def generate(schema_name: str = "public"): output.append(warning_str) output.append('"""') + if output_schema.trigger_warnings: + warning_str = "\n".join(set(output_schema.trigger_warnings)) + + output.append('"""') + output.append( + "WARNING: Unable to find triggers for the following (used for " + "ON UPDATE, ON DELETE):" + ) + output.append(warning_str) + output.append('"""') + nicely_formatted = black.format_str( "\n".join(output), mode=black.FileMode(line_length=79) ) From 4284d131fd57484519d211d8db82a8acd2c94d0c Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 13 May 2022 09:09:08 +0100 Subject: [PATCH 316/727] tutorial for migrating an existing project to Piccolo (#511) --- docs/src/index.rst | 1 + docs/src/piccolo/tutorials/index.rst | 10 ++ .../tutorials/migrate_existing_project.rst | 129 ++++++++++++++++++ 3 files changed, 140 insertions(+) create mode 100644 docs/src/piccolo/tutorials/index.rst create mode 100644 docs/src/piccolo/tutorials/migrate_existing_project.rst diff --git a/docs/src/index.rst b/docs/src/index.rst index dba01f7b3..928664c75 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -23,6 +23,7 @@ batteries included. piccolo/playground/index piccolo/deployment/index piccolo/ecosystem/index + piccolo/tutorials/index piccolo/contributing/index piccolo/changes/index piccolo/help/index diff --git a/docs/src/piccolo/tutorials/index.rst b/docs/src/piccolo/tutorials/index.rst new file mode 100644 index 000000000..87ccf2ede --- /dev/null +++ b/docs/src/piccolo/tutorials/index.rst @@ -0,0 +1,10 @@ +Tutorials +========= + +These tutorials bring together information from across the documentation, to +help you solve common problems: + +.. toctree:: + :maxdepth: 1 + + ./migrate_existing_project diff --git a/docs/src/piccolo/tutorials/migrate_existing_project.rst b/docs/src/piccolo/tutorials/migrate_existing_project.rst new file mode 100644 index 000000000..7ead3dcbd --- /dev/null +++ b/docs/src/piccolo/tutorials/migrate_existing_project.rst @@ -0,0 +1,129 @@ +Migrate an existing project to Piccolo +====================================== + +Introduction +------------ + +If you have an existing project and Postgres database, and you want to use +Piccolo with it, these are the steps you need to take. + +Option 1 - ``piccolo asgi new`` +------------------------------- + +This is the recommended way of creating brand new projects. If this is your +first experience with Piccolo, then it's a good idea to create a test project: + +.. code-block:: bash + + mkdir test_project + cd test_project + piccolo asgi new + +You'll learn a lot about how Piccolo works by looking at the generated code. +You can then copy over the relevant files to your existing project if you like. + +Alternatively, doing it from scratch, you'll need to do the following: + +Option 2 - from scratch +----------------------- + +Create a Piccolo project file +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Create a new ``piccolo_conf.py`` file in the root of your project: + +.. code-block:: bash + + piccolo project new + +This contains your database details, and is used to register Piccolo apps. + +Create a new Piccolo app +~~~~~~~~~~~~~~~~~~~~~~~~ + +The app contains your ``Table`` classes and migrations. Run this command at the +root of your project: + +.. code-block:: bash + + # Replace 'my_app' with whatever you want to call your app + piccolo app new my_app + +Register the new Piccolo app +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Register this new app in ``piccolo_conf.py``. For example: + +.. code-block:: python + + APP_REGISTRY = AppRegistry( + apps=[ + "my_app.piccolo_app", + ] + ) + +While you're at it, make sure the database credentials are correct in +``piccolo_conf.py``. + +Make ``Table`` classes for your current database +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Now, if you run: + +.. code-block:: bash + + piccolo schema generate + +It will output Piccolo ``Table`` classes for your current database. Copy the +output into ``my_app/tables.py``. Double check that everything looks correct. + +In ``my_app/piccolo_app.py`` make sure it's tracking these tables for +migration purposes. + +.. code-block:: python + + from piccolo.conf.apps import AppConfig, table_finder + + APP_CONFIG = AppConfig( + table_classes=table_finder(["my_app.tables"], exclude_imported=True), + ... + ) + +Create an initial migration +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This will create a new file in ``my_app/piccolo_migrations``: + +.. code-block:: bash + + piccolo migrations new my_app --auto + +These tables already exist in the database, as it's an existing project, so +you need to fake apply this initial migration: + +.. code-block:: bash + + piccolo migrations forwards my_app --fake + +Making queries +~~~~~~~~~~~~~~ + +Now you're basically setup - to make database queries: + +.. code-block:: python + + from my_app.tables import MyTable + + async def my_endpoint(): + data = await MyTable.select() + return data + +Making new migrations +~~~~~~~~~~~~~~~~~~~~~ + +Just modify the files in ``tables.py``, and then run: + +.. code-block:: bash + + piccolo migrations new my_app --auto + piccolo migrations forwards my_app From a89c2fa57ba01f352e6830b99ff108b50b14c0e8 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 13 May 2022 09:57:32 +0100 Subject: [PATCH 317/727] bumped version --- CHANGES.rst | 16 ++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 5a447a554..c45e62d5e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,22 @@ Changes ======= +0.74.4 +------ + +``piccolo schema generate`` now outputs a warning when it can't detect the +``ON DELETE`` and ``ON UPDATE`` for a ``ForeignKey``, rather than raising an +exception. Thanks to @theelderbeever for reporting this issue. + +``run_sync`` doesn't use the connection pool by default anymore. It was causing +issues when an app contained sync and async code. Thanks to @WintonLi for +reporting this issue. + +Added a tutorial to the docs for using Piccolo with an existing project and +database. Thanks to @virajkanwade for reporting this issue. + +------------------------------------------------------------------------------- + 0.74.3 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 638e15e25..22d6db751 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.74.3" +__VERSION__ = "0.74.4" From bebb9728056e79e7317d80c54cb6d84ad2c2a943 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 24 May 2022 18:12:12 +0100 Subject: [PATCH 318/727] replace `get_event_loop` with `get_running_loop` (#528) `get_event_loop` is deprecated in Python 3.10, and emits a warning. --- piccolo/utils/sync.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/piccolo/utils/sync.py b/piccolo/utils/sync.py index 020b5899e..764281720 100644 --- a/piccolo/utils/sync.py +++ b/piccolo/utils/sync.py @@ -9,21 +9,19 @@ def run_sync(coroutine: t.Coroutine): """ Run the coroutine synchronously - trying to accommodate as many edge cases as possible. - 1. When called within a coroutine. 2. When called from ``python -m asyncio``, or iPython with %autoawait enabled, which means an event loop may already be running in the current thread. - """ try: - loop = asyncio.get_event_loop() + asyncio.get_running_loop() except RuntimeError: + # No current event loop, so it's safe to use: return asyncio.run(coroutine) else: - if not loop.is_running(): - return loop.run_until_complete(coroutine) - + # We're already inside a running event loop, so run the coroutine in a + # new thread: new_loop = asyncio.new_event_loop() with ThreadPoolExecutor(max_workers=1) as executor: From 1fa7bf087cd5f1c25b2bff45d5d8ad37256edfde Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 1 Jun 2022 17:53:55 +0100 Subject: [PATCH 319/727] improve docstring for `exclude_secrets` (#532) --- piccolo/table.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/piccolo/table.py b/piccolo/table.py index 9ff261023..a35d8cdb5 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -851,9 +851,11 @@ def select( await Band.select('name') :param exclude_secrets: - If ``True``, any password fields are omitted from the response. - Even though passwords are hashed, you still don't want them being - passed over the network if avoidable. + If ``True``, any columns with ``secret=True`` are omitted from the + response. For example, we use this for the password column of + :class:`BaseUser `. Even though + the passwords are hashed, you still don't want them being passed + over the network if avoidable. """ _columns = cls._process_column_args(*columns) From 5ffd22bf3575c5fe6790c516cf611742a315e66b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 2 Jun 2022 20:56:58 +0100 Subject: [PATCH 320/727] Docs for how to run async unit tests (#533) * updated the docstring for `ModelBuilder` * added `ModelBuilder` to the API reference * added docs for running async tests * update GitHub action versions * mention that `IsolatedAsyncioTestCase` is Python 3.8 and above --- .github/workflows/release.yaml | 2 +- .github/workflows/tests.yaml | 16 ++--- docs/src/piccolo/api_reference/index.rst | 13 ++++ docs/src/piccolo/testing/index.rst | 88 +++++++++++++++++++----- piccolo/testing/model_builder.py | 49 +++++++------ 5 files changed, 122 insertions(+), 46 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ee4569f63..c4c643bd9 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -11,7 +11,7 @@ jobs: runs-on: "ubuntu-latest" steps: - - uses: "actions/checkout@v2" + - uses: "actions/checkout@v3" - uses: "actions/setup-python@v1" with: python-version: 3.7 diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index f92e72fc1..0bb8b58d1 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -15,9 +15,9 @@ jobs: python-version: ["3.7", "3.8", "3.9", "3.10"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -51,9 +51,9 @@ jobs: ports: - 5432:5432 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -103,9 +103,9 @@ jobs: - 5432:5432 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -140,9 +140,9 @@ jobs: python-version: ["3.7", "3.8", "3.9", "3.10"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/docs/src/piccolo/api_reference/index.rst b/docs/src/piccolo/api_reference/index.rst index 22bce6688..deaf75b38 100644 --- a/docs/src/piccolo/api_reference/index.rst +++ b/docs/src/piccolo/api_reference/index.rst @@ -58,3 +58,16 @@ Date .. autoclass:: DateOffset :members: + +------------------------------------------------------------------------------- + +Testing +------- + +.. currentmodule:: piccolo.testing.model_builder + +ModelBuilder +~~~~~~~~~~~~ + +.. autoclass:: ModelBuilder + :members: diff --git a/docs/src/piccolo/testing/index.rst b/docs/src/piccolo/testing/index.rst index 6904f5d47..c2da500db 100644 --- a/docs/src/piccolo/testing/index.rst +++ b/docs/src/piccolo/testing/index.rst @@ -1,18 +1,32 @@ Testing ======= -Piccolo provides a few tools to make testing easier and decrease manual work. +Piccolo provides a few tools to make testing easier. + +------------------------------------------------------------------------------- + +Test runner +----------- + +Piccolo ships with a handy command for running your unit tests using pytest. +See the :ref:`tester app`. + +You can put your test files anywhere you like, but a good place is in a ``tests`` +folder within your Piccolo app. The test files should be named like +``test_*. py`` or ``*_test.py`` for pytest to recognise them. ------------------------------------------------------------------------------- Model Builder ------------- -When writing unit tests, it's usually required to have some data seeded into the database. -You can build and save the records manually or use ``ModelBuilder`` to generate random records for you. +When writing unit tests, it's usually required to have some data seeded into +the database. You can build and save the records manually or use +:class:`ModelBuilder ` to generate +random records for you. -This way you can randomize the fields you don't care about and specify important fields explicitly and -reduce the amount of manual work required. +This way you can randomize the fields you don't care about and specify +important fields explicitly and reduce the amount of manual work required. ``ModelBuilder`` currently supports all Piccolo column types and features. Let's say we have the following schema: @@ -28,7 +42,8 @@ Let's say we have the following schema: name = Varchar(length=50) manager = ForeignKey(Manager, null=True) -You can build a random ``Band`` which will also build and save a random ``Manager``: +You can build a random ``Band`` which will also build and save a random +``Manager``: .. code-block:: python @@ -70,7 +85,7 @@ To build objects without persisting them into the database: band = await ModelBuilder.build(Band, persist=False) -To build object with minimal attributes, leaving nullable fields empty: +To build objects with minimal attributes, leaving nullable fields empty: .. code-block:: python @@ -79,13 +94,6 @@ To build object with minimal attributes, leaving nullable fields empty: ------------------------------------------------------------------------------- -Test runner ------------ - -This runs your unit tests using pytest. See the :ref:`tester app`. - -------------------------------------------------------------------------------- - Creating the test schema ------------------------ @@ -119,19 +127,65 @@ Alternatively, you can run the migrations to setup the schema if you prefer: .. code-block:: python - import asyncio from unittest import TestCase from piccolo.apps.migrations.commands.backwards import run_backwards from piccolo.apps.migrations.commands.forwards import run_forwards + from piccolo.utils.sync import run_sync class TestApp(TestCase): def setUp(self): - asyncio.run(run_forwards("all")) + run_sync(run_forwards("all")) def tearDown(self): - asyncio.run(run_backwards("all", auto_agree=True)) + run_sync(run_backwards("all", auto_agree=True)) def test_app(self): # Do some testing ... pass + +------------------------------------------------------------------------------- + +Testing async code +------------------ + +There are a few options for testing async code using pytest. + +You can either call any async code using Piccolo's ``run_sync`` utility: + +.. code-block:: python + + from piccolo.utils.sync import run_sync + + async def get_data(): + ... + + def test_get_data(): + rows = run_sync(get_data()) + assert len(rows) == 1 + +Alternatively, you can make your tests natively async. + +If you prefer using pytest's function based tests, then take a look at +`pytest-asyncio `_. Simply +install it using ``pip install pytest-asyncio``, then you can then write tests +like this: + +.. code-block:: python + + async def test_select(): + rows = await MyTable.select() + assert len(rows) == 1 + +If you prefer class based tests, and are using Python 3.8 or above, then have +a look at :class:`IsolatedAsyncioTestCase ` +from Python's standard library. You can then write tests like this: + +.. code-block:: python + + from unittest import IsolatedAsyncioTestCase + + class MyTest(IsolatedAsyncioTestCase): + async def test_select(self): + rows = await MyTable.select() + assert len(rows) == 1 diff --git a/piccolo/testing/model_builder.py b/piccolo/testing/model_builder.py index 8d3fabb6b..dd5d1f833 100644 --- a/piccolo/testing/model_builder.py +++ b/piccolo/testing/model_builder.py @@ -33,18 +33,39 @@ async def build( minimal: bool = False, ) -> Table: """ - Build Table instance with random data and save async. - This can build relationships, supported data types and parameters. + Build a ``Table`` instance with random data and save async. + If the ``Table`` has any foreign keys, then the related rows are also + created automatically. :param table_class: Table class to randomize. + :param defaults: + Any values specified here will be used instead of random values. + :param persist: + Whether to save the new instance in the database. + :param minimal: + If ``True`` then any columns with ``null=True`` are assigned + a value of ``None``. - Examples: + Examples:: + + # Create a new instance with all random values: manager = await ModelBuilder.build(Manager) - manager = await ModelBuilder.build(Manager, name='Guido') - manager = await ModelBuilder(persist=False).build(Manager) - manager = await ModelBuilder(minimal=True).build(Manager) - band = await ModelBuilder.build(Band, manager=manager) + + # Create a new instance, with certain defaults: + manager = await ModelBuilder.build( + Manager, + {Manager.name: 'Guido'} + ) + + # Create a new instance, but don't save it in the database: + manager = await ModelBuilder.build(Manager, persist=False) + + # Create a new instance, with all null values set to None: + manager = await ModelBuilder.build(Manager, minimal=True) + + # We can pass other table instances in as default values: + band = await ModelBuilder.build(Band, {Band.manager: manager}) """ return await cls._build( @@ -63,19 +84,7 @@ def build_sync( minimal: bool = False, ) -> Table: """ - Build Table instance with random data and save sync. - This can build relationships, supported data types and parameters. - - :param table_class: - Table class to randomize. - - Examples: - manager = ModelBuilder.build_sync(Manager) - manager = ModelBuilder.build_sync(Manager, name='Guido') - manager = ModelBuilder(persist=False).build_sync(Manager) - manager = ModelBuilder(minimal=True).build_sync(Manager) - band = ModelBuilder.build_sync(Band, manager=manager) - + A sync wrapper around :meth:`build`. """ return run_sync( cls.build( From f2a2fb8cd91e29b1d0b90b3a8e01a43f2ae502dd Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 2 Jun 2022 22:19:47 +0100 Subject: [PATCH 321/727] add more in depth docs around Docker deployment (#534) --- docs/src/piccolo/deployment/index.rst | 69 ++++++++++++++++++- .../projects_and_apps/piccolo_projects.rst | 2 + 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/docs/src/piccolo/deployment/index.rst b/docs/src/piccolo/deployment/index.rst index 1fc595b9b..26475385b 100644 --- a/docs/src/piccolo/deployment/index.rst +++ b/docs/src/piccolo/deployment/index.rst @@ -4,13 +4,76 @@ Deployment Docker ------ +`Docker `_ is a very popular way of deploying +applications, using containers. + +Base image +~~~~~~~~~~ + Piccolo has several dependencies which are compiled (e.g. asyncpg, orjson), which is great for performance, but you may run into difficulties when using -Alpine Linux as your base Docker image. +Alpine Linux as your base Docker image. Alpine uses a different compiler +toolchain to most Linux distros. -Alpine uses a different compiler toolchain to most Linux distros. It's -highly recommended to use Debian as your base Docker image. Many Python packages +It's highly recommended to use Debian as your base Docker image. Many Python packages have prebuilt versions for Debian, meaning you don't have to compile them at all during install. The result is a much faster build process, and potentially even a smaller overall Docker image size (the size of Alpine quickly balloons after you've added all of the compilation dependencies). + +Environment variables +~~~~~~~~~~~~~~~~~~~~~ + +By using environment variables, we can inject the database credentials for +Piccolo. + +Example Dockerfile +~~~~~~~~~~~~~~~~~~ + +This is a very simple Dockerfile, and illustrates the basics: + +.. code-block:: dockerfile + + # Specify the base image: + FROM python:3.10-slim-bullseye + + # Install the pip requirements: + RUN pip install --upgrade pip + ADD app/requirements.txt / + RUN pip install -r /requirements.txt + + # Add the application code: + ADD app /app + + # Environment variables: + ENV PG_HOST=localhost + ENV PG_PORT=5432 + ENV PG_USER=my_database_user + ENV PG_PASSWORD="" + ENV PG_DATABASE=my_database + + CMD ["/usr/local/bin/python", "/app/main.py"] + +We can then modify our :ref:`piccolo_conf.py ` file to use these +environment variables: + +.. code-block:: python + + # piccolo_conf.py + + import os + + DB = PostgresEngine( + config={ + "port": int(os.environ.get("PG_PORT", "5432")), + "user": os.environ.get("PG_USER", "my_database_user"), + "password": os.environ.get("PG_PASSWORD", ""), + "database": os.environ.get("PG_DATABASE", "my_database"), + "host": os.environ.get("PG_HOST", "localhost"), + } + ) + +When we run the container (usually via `Kubernetes `_, +`Docker Compose `_, or similar), +we can specify the database credentials using environment variables, which will +be used by our application. diff --git a/docs/src/piccolo/projects_and_apps/piccolo_projects.rst b/docs/src/piccolo/projects_and_apps/piccolo_projects.rst index 945a0da4d..f85591ede 100644 --- a/docs/src/piccolo/projects_and_apps/piccolo_projects.rst +++ b/docs/src/piccolo/projects_and_apps/piccolo_projects.rst @@ -7,6 +7,8 @@ A Piccolo project is a collection of apps. ------------------------------------------------------------------------------- +.. _PiccoloConf: + piccolo_conf.py --------------- From a3e1c38b847b9e60ff124b8e0514b9c985b66166 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 3 Jun 2022 21:17:51 +0100 Subject: [PATCH 322/727] added some links from the docs to video tutorials (#535) --- docs/src/piccolo/projects_and_apps/included_apps.rst | 12 ++++++++++++ docs/src/piccolo/projects_and_apps/index.rst | 4 ++++ docs/src/piccolo/schema/m2m.rst | 4 ++++ docs/src/piccolo/serialization/index.rst | 4 ++++ 4 files changed, 24 insertions(+) diff --git a/docs/src/piccolo/projects_and_apps/included_apps.rst b/docs/src/piccolo/projects_and_apps/included_apps.rst index 1dfd48bec..a2590dd05 100644 --- a/docs/src/piccolo/projects_and_apps/included_apps.rst +++ b/docs/src/piccolo/projects_and_apps/included_apps.rst @@ -166,6 +166,10 @@ Here's an example of a generated image: .. image:: ./images/schema_graph_output.png :target: /_images/schema_graph_output.png +.. note:: + + There is a `video tutorial on YouTube `_. + ------------------------------------------------------------------------------- shell @@ -178,6 +182,10 @@ Launches an iPython shell, and automatically imports all of your registered piccolo shell run +.. note:: + + There is a `video tutorial on YouTube `_. + ------------------------------------------------------------------------------- sql_shell @@ -194,6 +202,10 @@ need to run raw SQL queries on your database. For it to work, the underlying command needs to be on the path (i.e. ``psql`` or ``sqlite3`` depending on which you're using). +.. note:: + + There is a `video tutorial on YouTube `_. + ------------------------------------------------------------------------------- .. _TesterApp: diff --git a/docs/src/piccolo/projects_and_apps/index.rst b/docs/src/piccolo/projects_and_apps/index.rst index 36f46f25b..c77a77a35 100644 --- a/docs/src/piccolo/projects_and_apps/index.rst +++ b/docs/src/piccolo/projects_and_apps/index.rst @@ -12,3 +12,7 @@ application. ./piccolo_projects ./piccolo_apps ./included_apps + +.. note:: + + There is a `video tutorial on YouTube `_. diff --git a/docs/src/piccolo/schema/m2m.rst b/docs/src/piccolo/schema/m2m.rst index c8536b2ba..5413389ec 100644 --- a/docs/src/piccolo/schema/m2m.rst +++ b/docs/src/piccolo/schema/m2m.rst @@ -4,6 +4,10 @@ M2M ### +.. note:: + + There is a `video tutorial on YouTube `_. + Sometimes in database design you need `many-to-many (M2M) `_ relationships. diff --git a/docs/src/piccolo/serialization/index.rst b/docs/src/piccolo/serialization/index.rst index c1f7f3f9b..18ac966aa 100644 --- a/docs/src/piccolo/serialization/index.rst +++ b/docs/src/piccolo/serialization/index.rst @@ -118,6 +118,10 @@ To populate a nested Pydantic model with data from the database: >>> model.manager.name 'Guido' +.. note:: + + There is a `video tutorial on YouTube `_. + include_default_columns ~~~~~~~~~~~~~~~~~~~~~~~ From 21f3c82981b61062552cccad83c18182344787be Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 5 Jun 2022 20:17:43 +0100 Subject: [PATCH 323/727] bumped version --- CHANGES.rst | 11 +++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index c45e62d5e..036946e5d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,17 @@ Changes ======= +0.75.0 +------ + +Changed how ``piccolo.utils.sync.run_sync`` works, to prevent a warning on +Python 3.10. Thanks to @Drapersniper for reporting this issue. + +Lots of documentation improvements - particularly around testing, and Docker +deployment. + +------------------------------------------------------------------------------- + 0.74.4 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 22d6db751..b59abc9ae 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.74.4" +__VERSION__ = "0.75.0" From e1d5456a42927bb1f9a79781548f5c3b92010bee Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 9 Jun 2022 14:46:44 +0100 Subject: [PATCH 324/727] create a `_validate_password` method in `BaseUser` (#537) --- piccolo/apps/user/tables.py | 46 +++++++++++++++++++++++----------- tests/apps/user/test_tables.py | 27 ++++++++++++++++++++ 2 files changed, 59 insertions(+), 14 deletions(-) diff --git a/piccolo/apps/user/tables.py b/piccolo/apps/user/tables.py index 7df949cad..43c5e9bda 100644 --- a/piccolo/apps/user/tables.py +++ b/piccolo/apps/user/tables.py @@ -71,6 +71,35 @@ def get_readable(cls) -> Readable: ########################################################################### + @classmethod + def _validate_password(cls, password: str): + """ + Validate the raw password. Used by :meth:`update_password` and + :meth:`create_user`. + + :param password: + The raw password e.g. ``'hello123'``. + :raises ValueError: + If the password fails any of the criteria. + + """ + if not password: + raise ValueError("A password must be provided.") + + if len(password) < cls._min_password_length: + raise ValueError("The password is too short.") + + if len(password) > cls._max_password_length: + raise ValueError("The password is too long.") + + if password.startswith("pbkdf2_sha256"): + logger.warning( + "Tried to create a user with an already hashed password." + ) + raise ValueError("Do not pass a hashed password.") + + ########################################################################### + @classmethod def update_password_sync(cls, user: t.Union[str, int], password: str): """ @@ -93,6 +122,8 @@ async def update_password(cls, user: t.Union[str, int], password: str): "The `user` arg must be a user id, or a username." ) + cls._validate_password(password=password) + password = cls.hash_password(password) await cls.update({cls.password: password}).where(clause).run() @@ -227,20 +258,7 @@ async def create_user( if not username: raise ValueError("A username must be provided.") - if not password: - raise ValueError("A password must be provided.") - - if len(password) < cls._min_password_length: - raise ValueError("The password is too short.") - - if len(password) > cls._max_password_length: - raise ValueError("The password is too long.") - - if password.startswith("pbkdf2_sha256"): - logger.warning( - "Tried to create a user with an already hashed password." - ) - raise ValueError("Do not pass a hashed password.") + cls._validate_password(password=password) user = cls(username=username, password=password, **extra_params) await user.save() diff --git a/tests/apps/user/test_tables.py b/tests/apps/user/test_tables.py index ac4926466..497a2df0a 100644 --- a/tests/apps/user/test_tables.py +++ b/tests/apps/user/test_tables.py @@ -112,6 +112,33 @@ def test_update_password(self): "The password is too long.", ) + # Test short passwords + short_password = "abc" + with self.assertRaises(ValueError) as manager: + BaseUser.update_password_sync(username, short_password) + self.assertEqual( + manager.exception.__str__(), + "The password is too short.", + ) + + # Test no password + empty_password = "" + with self.assertRaises(ValueError) as manager: + BaseUser.update_password_sync(username, empty_password) + self.assertEqual( + manager.exception.__str__(), + "A password must be provided.", + ) + + # Test hashed password + hashed_password = "pbkdf2_sha256$abc123" + with self.assertRaises(ValueError) as manager: + BaseUser.update_password_sync(username, hashed_password) + self.assertEqual( + manager.exception.__str__(), + "Do not pass a hashed password.", + ) + class TestCreateUserFromFixture(TestCase): def setUp(self): From 99a77ae385c2c0eaf370de075f4ba879df007624 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 13 Jun 2022 16:31:18 +0100 Subject: [PATCH 325/727] change `run_sync` (#539) * change `run_sync` * simplify `run_sync` - call `asyncio.run` first Trying `asyncio.run` first, as in 99% of cases this will work. Also, using asyncio.run in a thread when running in an event loop. * Create test-strict.sh This runs the tests in Python's development mode, which gives lots of useful information - for example, about deprecated features, and asyncio warnings. * close connection if exception is raised by asyncpg When running the tests in development mode, I realised that there were situations where the connection wasn't being closed (i.e. if an exception was raised by asyncpg due to a Postgres error). * added `create_db_tables` and `drop_db_tables` These are async first. They replace `create_tables` and `drop_tables` which were sync first, and had no async equivalent. --- docs/src/piccolo/api_reference/index.rst | 10 +++ docs/src/piccolo/query_types/alter.rst | 17 +++- docs/src/piccolo/query_types/create_table.rst | 16 +++- docs/src/piccolo/testing/index.rst | 13 ++- piccolo/engine/postgres.py | 16 +++- piccolo/table.py | 86 +++++++++++++++++-- piccolo/utils/sync.py | 13 +-- scripts/test-strict.sh | 12 +++ .../auto/integration/test_migrations.py | 8 +- .../columns/foreign_key/test_target_column.py | 10 +-- tests/columns/test_m2m.py | 14 +-- .../instance/test_get_related_readable.py | 6 +- tests/table/instance/test_save.py | 6 +- tests/table/test_create_db_tables.py | 29 +++++++ tests/table/test_create_tables.py | 15 ---- tests/table/test_drop_db_tables.py | 37 ++++++++ tests/table/test_drop_tables.py | 21 ----- 17 files changed, 240 insertions(+), 89 deletions(-) create mode 100755 scripts/test-strict.sh create mode 100644 tests/table/test_create_db_tables.py delete mode 100644 tests/table/test_create_tables.py create mode 100644 tests/table/test_drop_db_tables.py delete mode 100644 tests/table/test_drop_tables.py diff --git a/docs/src/piccolo/api_reference/index.rst b/docs/src/piccolo/api_reference/index.rst index deaf75b38..30eb0f2b0 100644 --- a/docs/src/piccolo/api_reference/index.rst +++ b/docs/src/piccolo/api_reference/index.rst @@ -71,3 +71,13 @@ ModelBuilder .. autoclass:: ModelBuilder :members: + +.. currentmodule:: piccolo.table + +create_db_tables / drop_db_tables +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autofunction:: create_db_tables +.. autofunction:: create_db_tables_sync +.. autofunction:: drop_db_tables +.. autofunction:: drop_db_tables_sync diff --git a/docs/src/piccolo/query_types/alter.rst b/docs/src/piccolo/query_types/alter.rst index 4361ec68c..2542641f6 100644 --- a/docs/src/piccolo/query_types/alter.rst +++ b/docs/src/piccolo/query_types/alter.rst @@ -40,14 +40,23 @@ Used to drop the table - use with caution! await Band.alter().drop_table() -If you have several tables which you want to drop, you can use ``drop_tables`` -instead. It will drop them in the correct order. +drop_db_tables / drop_db_tables_sync +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you have several tables which you want to drop, you can use +:func:`drop_db_tables ` or +:func:`drop_db_tables_sync `. The tables +will be dropped in the correct order based on their foreign keys. .. code-block:: python - from piccolo.table import drop_tables + # async version + >>> from piccolo.table import drop_db_tables + >>> await drop_db_tables(Band, Manager) - drop_tables(Band, Manager) + # sync version + >>> from piccolo.table import drop_db_tables_sync + >>> drop_db_tables_sync(Band, Manager) ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/query_types/create_table.rst b/docs/src/piccolo/query_types/create_table.rst index 2e6e98cc5..569462d3c 100644 --- a/docs/src/piccolo/query_types/create_table.rst +++ b/docs/src/piccolo/query_types/create_table.rst @@ -20,12 +20,20 @@ To prevent an error from being raised if the table already exists: >>> await Band.create_table(if_not_exists=True) [] -Also, you can create multiple tables at once. +create_db_tables / create_db_tables_sync +---------------------------------------- -This function will automatically sort tables based on their foreign keys so they're created in the right order: +You can create multiple tables at once. + +This function will automatically sort tables based on their foreign keys so +they're created in the right order: .. code-block:: python - >>> from piccolo.table import create_tables - >>> create_tables(Band, Manager, if_not_exists=True) + # async version + >>> from piccolo.table import create_db_tables + >>> await create_db_tables(Band, Manager, if_not_exists=True) + # sync version + >>> from piccolo.table import create_db_tables_sync + >>> create_db_tables_sync(Band, Manager, if_not_exists=True) diff --git a/docs/src/piccolo/testing/index.rst b/docs/src/piccolo/testing/index.rst index c2da500db..39791ab0d 100644 --- a/docs/src/piccolo/testing/index.rst +++ b/docs/src/piccolo/testing/index.rst @@ -101,23 +101,28 @@ When running your unit tests, you usually start with a blank test database, create the tables, and then install test data. To create the tables, there are a few different approaches you can take. Here -we use ``create_tables`` and ``drop_tables``: +we use :func:`create_db_tables_sync ` and +:func:`drop_db_tables_sync `. + +.. note:: + The async equivalents are :func:`create_db_tables ` + and :func:`drop_db_tables `. .. code-block:: python from unittest import TestCase - from piccolo.table import create_tables, drop_tables + from piccolo.table import create_db_tables_sync, drop_db_tables_sync from piccolo.conf.apps import Finder TABLES = Finder().get_table_classes() class TestApp(TestCase): def setUp(self): - create_tables(*TABLES) + create_db_tables_sync(*TABLES) def tearDown(self): - drop_tables(*TABLES) + drop_db_tables_sync(*TABLES) def test_app(self): # Do some testing ... diff --git a/piccolo/engine/postgres.py b/piccolo/engine/postgres.py index 5b6bb6d54..3eb40992f 100644 --- a/piccolo/engine/postgres.py +++ b/piccolo/engine/postgres.py @@ -126,7 +126,13 @@ async def _run_in_pool(self): async def _run_in_new_connection(self): connection = await asyncpg.connect(**self.engine.config) - await self._run_queries(connection) + try: + await self._run_queries(connection) + except asyncpg.exceptions.PostgresError as exception: + await connection.close() + raise exception + + await connection.close() async def run(self, in_pool=True): if in_pool and self.engine.pool: @@ -410,7 +416,13 @@ async def _run_in_new_connection( if args is None: args = [] connection = await self.get_new_connection() - results = await connection.fetch(query, *args) + + try: + results = await connection.fetch(query, *args) + except asyncpg.exceptions.PostgresError as exception: + await connection.close() + raise exception + await connection.close() return results diff --git a/piccolo/table.py b/piccolo/table.py index a35d8cdb5..d057c0133 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -46,6 +46,8 @@ from piccolo.utils import _camel_to_snake from piccolo.utils.graphlib import TopologicalSorter from piccolo.utils.sql_values import convert_to_sql_value +from piccolo.utils.sync import run_sync +from piccolo.utils.warnings import colored_warning if t.TYPE_CHECKING: from piccolo.columns import Selectable @@ -1171,10 +1173,23 @@ def create_table_class( ) -def create_tables(*tables: t.Type[Table], if_not_exists: bool = False) -> None: +############################################################################### +# Quickly create or drop database tables from Piccolo `Table` clases. + + +async def create_db_tables( + *tables: t.Type[Table], if_not_exists: bool = False +) -> None: """ - Creates the tables passed to it in the correct order, based on their - foreign keys. + Creates the database table for each ``Table`` class passed in. The tables + are created in the correct order, based on their foreign keys. + + :param tables: + The tables to create in the database. + :param if_not_exists: + No errors will be raised if any of the tables already exist in the + database. + """ if tables: engine = tables[0]._meta.db @@ -1190,13 +1205,42 @@ def create_tables(*tables: t.Type[Table], if_not_exists: bool = False) -> None: for table in sorted_table_classes ] ) - atomic.run_sync() + await atomic.run() -def drop_tables(*tables: t.Type[Table]) -> None: +def create_db_tables_sync( + *tables: t.Type[Table], if_not_exists: bool = False +) -> None: + """ + A sync wrapper around :func:`create_db_tables`. + """ + run_sync(create_db_tables(*tables, if_not_exists=if_not_exists)) + + +def create_tables(*tables: t.Type[Table], if_not_exists: bool = False) -> None: + """ + This original implementation has been replaced, because it was synchronous, + and felt at odds with the rest of the Piccolo codebase which is async + first. + """ + colored_warning( + "`create_tables` is deprecated and will be removed in v1 of Piccolo. " + "Use `await create_db_tables(...)` or `create_db_tables_sync(...)` " + "instead.", + category=DeprecationWarning, + ) + + return create_db_tables_sync(*tables, if_not_exists=if_not_exists) + + +async def drop_db_tables(*tables: t.Type[Table]) -> None: """ - Drops the tables passed to it in the correct order, based on their foreign - keys. + Drops the database table for each ``Table`` class passed in. The tables + are dropped in the correct order, based on their foreign keys. + + :param tables: + The tables to delete from the database. + """ if tables: engine = tables[0]._meta.db @@ -1223,7 +1267,33 @@ def drop_tables(*tables: t.Type[Table]) -> None: ] ) - atomic.run_sync() + await atomic.run() + + +def drop_db_tables_sync(*tables: t.Type[Table]) -> None: + """ + A sync wrapper around :func:`drop_db_tables`. + """ + run_sync(drop_db_tables(*tables)) + + +def drop_tables(*tables: t.Type[Table]) -> None: + """ + This original implementation has been replaced, because it was synchronous, + and felt at odds with the rest of the Piccolo codebase which is async + first. + """ + colored_warning( + "`drop_tables` is deprecated and will be removed in v1 of Piccolo. " + "Use `await drop_db_tables(...)` or `drop_db_tables_sync(...)` " + "instead.", + category=DeprecationWarning, + ) + + return drop_db_tables_sync(*tables) + + +############################################################################### def sort_table_classes( diff --git a/piccolo/utils/sync.py b/piccolo/utils/sync.py index 764281720..55b44e499 100644 --- a/piccolo/utils/sync.py +++ b/piccolo/utils/sync.py @@ -15,15 +15,10 @@ def run_sync(coroutine: t.Coroutine): current thread. """ try: - asyncio.get_running_loop() - except RuntimeError: - # No current event loop, so it's safe to use: + # We try this first, as in most situations this will work. return asyncio.run(coroutine) - else: - # We're already inside a running event loop, so run the coroutine in a - # new thread: - new_loop = asyncio.new_event_loop() - + except RuntimeError: + # An event loop already exists. with ThreadPoolExecutor(max_workers=1) as executor: - future = executor.submit(new_loop.run_until_complete, coroutine) + future = executor.submit(asyncio.run, coroutine) return future.result() diff --git a/scripts/test-strict.sh b/scripts/test-strict.sh new file mode 100755 index 000000000..08cc48a2b --- /dev/null +++ b/scripts/test-strict.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# This runs the tests in Python's development mode: +# https://docs.python.org/3/library/devmode.html +# It shows us deprecation warnings, and asyncio warnings. + +# To run all in a folder tests/ +# To run all in a file tests/test_foo.py +# To run all in a class tests/test_foo.py::TestFoo +# To run a single test tests/test_foo.py::TestFoo::test_foo + +export PICCOLO_CONF="tests.postgres_conf" +python -X dev -m pytest -m "not integration" -s $@ diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index 64178fc6f..d7261a074 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -45,7 +45,7 @@ from piccolo.columns.m2m import M2M from piccolo.columns.reference import LazyTableReference from piccolo.conf.apps import AppConfig -from piccolo.table import Table, create_table_class, drop_tables +from piccolo.table import Table, create_table_class, drop_db_tables_sync from piccolo.utils.sync import run_sync from tests.base import DBTestCase, postgres_only @@ -836,7 +836,7 @@ def setUp(self): pass def tearDown(self): - drop_tables(Migration, Band, Genre, GenreToBand) + drop_db_tables_sync(Migration, Band, Genre, GenreToBand) def test_m2m(self): """ @@ -872,7 +872,7 @@ def setUp(self): pass def tearDown(self): - drop_tables(Migration, TableA, TableC) + drop_db_tables_sync(Migration, TableA, TableC) def test_target_column(self): """ @@ -910,7 +910,7 @@ def setUp(self): pass def tearDown(self): - drop_tables(Migration, TableA, TableB) + drop_db_tables_sync(Migration, TableA, TableB) def test_target_column(self): """ diff --git a/tests/columns/foreign_key/test_target_column.py b/tests/columns/foreign_key/test_target_column.py index 4e059dff8..e9a0c4460 100644 --- a/tests/columns/foreign_key/test_target_column.py +++ b/tests/columns/foreign_key/test_target_column.py @@ -1,7 +1,7 @@ from unittest import TestCase from piccolo.columns import ForeignKey, Varchar -from piccolo.table import Table, create_tables, drop_tables +from piccolo.table import Table, create_db_tables_sync, drop_db_tables_sync class Manager(Table): @@ -20,10 +20,10 @@ class TestTargetColumnWithString(TestCase): """ def setUp(self): - create_tables(Manager, Band) + create_db_tables_sync(Manager, Band) def tearDown(self): - drop_tables(Manager, Band) + drop_db_tables_sync(Manager, Band) def test_queries(self): manager_1 = Manager.objects().create(name="Guido").run_sync() @@ -63,10 +63,10 @@ class TestTargetColumnWithColumnRef(TestCase): """ def setUp(self): - create_tables(ManagerA, BandA) + create_db_tables_sync(ManagerA, BandA) def tearDown(self): - drop_tables(ManagerA, BandA) + drop_db_tables_sync(ManagerA, BandA) def test_queries(self): manager_1 = ManagerA.objects().create(name="Guido").run_sync() diff --git a/tests/columns/test_m2m.py b/tests/columns/test_m2m.py index 47f034e3a..d3374cfff 100644 --- a/tests/columns/test_m2m.py +++ b/tests/columns/test_m2m.py @@ -33,7 +33,7 @@ Varchar, ) from piccolo.columns.m2m import M2M -from piccolo.table import Table, create_tables, drop_tables +from piccolo.table import Table, create_db_tables_sync, drop_db_tables_sync class Band(Table): @@ -57,7 +57,7 @@ class GenreToBand(Table): class TestM2M(TestCase): def setUp(self): - create_tables(*SIMPLE_SCHEMA, if_not_exists=True) + create_db_tables_sync(*SIMPLE_SCHEMA, if_not_exists=True) Band.insert( Band(name="Pythonistas"), @@ -80,7 +80,7 @@ def setUp(self): ).run_sync() def tearDown(self): - drop_tables(*SIMPLE_SCHEMA) + drop_db_tables_sync(*SIMPLE_SCHEMA) def test_select_name(self): response = Band.select( @@ -414,7 +414,7 @@ class TestM2MCustomPrimaryKey(TestCase): """ def setUp(self): - create_tables(*CUSTOM_PK_SCHEMA, if_not_exists=True) + create_db_tables_sync(*CUSTOM_PK_SCHEMA, if_not_exists=True) bob = Customer.objects().create(name="Bob").run_sync() sally = Customer.objects().create(name="Sally").run_sync() @@ -433,7 +433,7 @@ def setUp(self): ).run_sync() def tearDown(self): - drop_tables(*CUSTOM_PK_SCHEMA) + drop_db_tables_sync(*CUSTOM_PK_SCHEMA) def test_select(self): response = Customer.select( @@ -555,7 +555,7 @@ class TestM2MComplexSchema(TestCase): """ def setUp(self): - create_tables(*COMPLEX_SCHEMA, if_not_exists=True) + create_db_tables_sync(*COMPLEX_SCHEMA, if_not_exists=True) small_table = SmallTable(varchar_col="Test") small_table.save().run_sync() @@ -589,7 +589,7 @@ def setUp(self): self.mega_table = mega_table def tearDown(self): - drop_tables(*COMPLEX_SCHEMA) + drop_db_tables_sync(*COMPLEX_SCHEMA) def test_select_all(self): """ diff --git a/tests/table/instance/test_get_related_readable.py b/tests/table/instance/test_get_related_readable.py index 3a07efed6..3c66f6ad4 100644 --- a/tests/table/instance/test_get_related_readable.py +++ b/tests/table/instance/test_get_related_readable.py @@ -1,7 +1,7 @@ import decimal from unittest import TestCase -from piccolo.table import create_tables, drop_tables +from piccolo.table import create_db_tables_sync, drop_db_tables_sync from tests.example_apps.music.tables import ( Band, Concert, @@ -15,7 +15,7 @@ class TestGetRelatedReadable(TestCase): def setUp(self): - create_tables(*TABLES) + create_db_tables_sync(*TABLES) manager_1 = Manager.objects().create(name="Guido").run_sync() manager_2 = Manager.objects().create(name="Graydon").run_sync() @@ -45,7 +45,7 @@ def setUp(self): ).run_sync() def tearDown(self): - drop_tables(*TABLES) + drop_db_tables_sync(*TABLES) def test_get_related_readable(self): """ diff --git a/tests/table/instance/test_save.py b/tests/table/instance/test_save.py index e1bc49d1d..e97df0d54 100644 --- a/tests/table/instance/test_save.py +++ b/tests/table/instance/test_save.py @@ -1,15 +1,15 @@ from unittest import TestCase -from piccolo.table import create_tables, drop_tables +from piccolo.table import create_db_tables_sync, drop_db_tables_sync from tests.example_apps.music.tables import Band, Manager class TestSave(TestCase): def setUp(self): - create_tables(Manager, Band) + create_db_tables_sync(Manager, Band) def tearDown(self): - drop_tables(Manager, Band) + drop_db_tables_sync(Manager, Band) def test_save_new(self): """ diff --git a/tests/table/test_create_db_tables.py b/tests/table/test_create_db_tables.py new file mode 100644 index 000000000..fdcf2a5d5 --- /dev/null +++ b/tests/table/test_create_db_tables.py @@ -0,0 +1,29 @@ +from unittest import TestCase + +from piccolo.table import ( + create_db_tables_sync, + create_tables, + drop_db_tables_sync, +) +from tests.example_apps.music.tables import Band, Manager + + +class TestCreateDBTables(TestCase): + def tearDown(self) -> None: + drop_db_tables_sync(Manager, Band) + + def test_create_db_tables(self): + """ + Make sure the tables are created in the database. + """ + create_db_tables_sync(Manager, Band, if_not_exists=False) + self.assertTrue(Manager.table_exists().run_sync()) + self.assertTrue(Band.table_exists().run_sync()) + + def test_create_tables(self): + """ + This is a deprecated function, which just acts as a proxy. + """ + create_tables(Manager, Band, if_not_exists=False) + self.assertTrue(Manager.table_exists().run_sync()) + self.assertTrue(Band.table_exists().run_sync()) diff --git a/tests/table/test_create_tables.py b/tests/table/test_create_tables.py deleted file mode 100644 index 0e3c94237..000000000 --- a/tests/table/test_create_tables.py +++ /dev/null @@ -1,15 +0,0 @@ -from unittest import TestCase - -from piccolo.table import create_tables -from tests.example_apps.music.tables import Band, Manager - - -class TestCreateTables(TestCase): - def tearDown(self) -> None: - Band.alter().drop_table(if_exists=True).run_sync() - Manager.alter().drop_table(if_exists=True).run_sync() - - def test_create_tables(self): - create_tables(Manager, Band, if_not_exists=False) - self.assertTrue(Manager.table_exists().run_sync()) - self.assertTrue(Band.table_exists().run_sync()) diff --git a/tests/table/test_drop_db_tables.py b/tests/table/test_drop_db_tables.py new file mode 100644 index 000000000..320b374dd --- /dev/null +++ b/tests/table/test_drop_db_tables.py @@ -0,0 +1,37 @@ +from unittest import TestCase + +from piccolo.table import ( + create_db_tables_sync, + drop_db_tables_sync, + drop_tables, +) +from tests.example_apps.music.tables import Band, Manager + + +class TestDropTables(TestCase): + def setUp(self): + create_db_tables_sync(Band, Manager) + + def test_drop_db_tables(self): + """ + Make sure the tables are dropped. + """ + self.assertEqual(Manager.table_exists().run_sync(), True) + self.assertEqual(Band.table_exists().run_sync(), True) + + drop_db_tables_sync(Manager, Band) + + self.assertEqual(Manager.table_exists().run_sync(), False) + self.assertEqual(Band.table_exists().run_sync(), False) + + def test_drop_tables(self): + """ + This is a deprecated function, which just acts as a proxy. + """ + self.assertEqual(Manager.table_exists().run_sync(), True) + self.assertEqual(Band.table_exists().run_sync(), True) + + drop_tables(Manager, Band) + + self.assertEqual(Manager.table_exists().run_sync(), False) + self.assertEqual(Band.table_exists().run_sync(), False) diff --git a/tests/table/test_drop_tables.py b/tests/table/test_drop_tables.py deleted file mode 100644 index 8e9dffb0a..000000000 --- a/tests/table/test_drop_tables.py +++ /dev/null @@ -1,21 +0,0 @@ -from unittest import TestCase - -from piccolo.table import create_tables, drop_tables -from tests.example_apps.music.tables import Band, Manager - - -class TestDropTables(TestCase): - def setUp(self): - create_tables(Band, Manager) - - def test_drop_tables(self): - """ - Make sure the tables are dropped. - """ - self.assertEqual(Manager.table_exists().run_sync(), True) - self.assertEqual(Band.table_exists().run_sync(), True) - - drop_tables(Manager, Band) - - self.assertEqual(Manager.table_exists().run_sync(), False) - self.assertEqual(Band.table_exists().run_sync(), False) From 32aae5eccdf0cc5b940c3db7ea7599da87d01c50 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 13 Jun 2022 17:02:46 +0100 Subject: [PATCH 326/727] bumped version --- CHANGES.rst | 35 +++++++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 036946e5d..04012648b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,41 @@ Changes ======= +0.76.0 +------ + +create_db_tables / drop_db_tables +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Added ``create_db_tables`` and ``create_db_tables_sync`` to replace +``create_tables``. The problem was ``create_tables`` was sync only, and was +inconsistent with the rest of Piccolo's API, which is async first. +``create_tables`` will continue to work for now, but is deprecated, and will be +removed in version 1.0. + +Likewise, ``drop_db_tables`` and ``drop_db_tables_sync`` have replaced +``drop_tables``. + +When calling ``create_tables`` / ``drop_tables`` within other async libraries +(such as `ward `_) it was sometimes +unreliable - the best solution was just to make async versions of these +functions. Thanks to @backwardspy for reporting this issue. + +``BaseUser`` password validation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We centralised the password validation logic in ``BaseUser`` into a method +called ``_validate_password``. This is needed by Piccolo API, but also makes it +easier for users to override this logic if subclassing ``BaseUser``. + +More ``run_sync`` refinements +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``run_sync``, which is the main utility function which Piccolo uses to run +async code, has been further simplified for Python > v3.10 compatibility. + +------------------------------------------------------------------------------- + 0.75.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index b59abc9ae..b15a78f1d 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.75.0" +__VERSION__ = "0.76.0" From cfd8085b244fd02d0d40ade45c92c337e56b413a Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 14 Jun 2022 13:39:10 +0100 Subject: [PATCH 327/727] make `Atomic` work with a connection pool (#544) There was a typo in `Atomic` meaning it wouldn't work when a connection pool was active. Added unit tests to cover this. --- piccolo/engine/postgres.py | 10 ++---- tests/engine/test_transaction.py | 54 ++++++++++++++++++++++++++------ 2 files changed, 47 insertions(+), 17 deletions(-) diff --git a/piccolo/engine/postgres.py b/piccolo/engine/postgres.py index 3eb40992f..9ca867d18 100644 --- a/piccolo/engine/postgres.py +++ b/piccolo/engine/postgres.py @@ -114,15 +114,11 @@ async def _run_queries(self, connection): self.queries = [] async def _run_in_pool(self): - pool = await self.engine.get_pool() - connection = await pool.acquire() + if not self.engine.pool: + raise ValueError("No pool is currently active.") - try: + async with self.engine.pool.acquire() as connection: await self._run_queries(connection) - except Exception: - pass - finally: - await pool.release(connection) async def _run_in_new_connection(self): connection = await asyncpg.connect(**self.engine.config) diff --git a/tests/engine/test_transaction.py b/tests/engine/test_transaction.py index a2cd38a43..1cf4538fb 100644 --- a/tests/engine/test_transaction.py +++ b/tests/engine/test_transaction.py @@ -1,6 +1,9 @@ import asyncio from unittest import TestCase +from piccolo.engine.postgres import Atomic +from piccolo.table import drop_db_tables_sync +from piccolo.utils.sync import run_sync from tests.example_apps.music.tables import Band, Manager from ..base import postgres_only @@ -11,31 +14,62 @@ def test_error(self): """ Make sure queries in a transaction aren't committed if a query fails. """ - transaction = Band._meta.db.atomic() - transaction.add( + atomic = Band._meta.db.atomic() + atomic.add( Manager.create_table(), Band.create_table(), Band.raw("MALFORMED QUERY ... SHOULD ERROR"), ) try: - transaction.run_sync() + atomic.run_sync() except Exception: pass self.assertTrue(not Band.table_exists().run_sync()) self.assertTrue(not Manager.table_exists().run_sync()) def test_succeeds(self): - transaction = Band._meta.db.atomic() - transaction.add(Manager.create_table(), Band.create_table()) - transaction.run_sync() + """ + Make sure that when atomic is run successfully the database is modified + accordingly. + """ + atomic = Band._meta.db.atomic() + atomic.add(Manager.create_table(), Band.create_table()) + atomic.run_sync() self.assertTrue(Band.table_exists().run_sync()) self.assertTrue(Manager.table_exists().run_sync()) - transaction.add( - Band.alter().drop_table(), Manager.alter().drop_table() - ) - transaction.run_sync() + drop_db_tables_sync(Band, Manager) + + @postgres_only + def test_pool(self): + """ + Make sure atomic works correctly when a connection pool is active. + """ + + async def run(): + """ + We have to run this async function, so we can use a connection + pool. + """ + engine = Band._meta.db + await engine.start_connection_pool() + + atomic: Atomic = engine.atomic() + atomic.add( + Manager.create_table(), + Band.create_table(), + ) + + await atomic.run() + await engine.close_connection_pool() + + run_sync(run()) + + self.assertTrue(Band.table_exists().run_sync()) + self.assertTrue(Manager.table_exists().run_sync()) + + drop_db_tables_sync(Band, Manager) class TestTransaction(TestCase): From 36746e948ccbb6bed0bc7d382b51d57894ad7048 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 14 Jun 2022 13:43:19 +0100 Subject: [PATCH 328/727] bumped version --- CHANGES.rst | 18 ++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 04012648b..99a382fbb 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,24 @@ Changes ======= +0.76.1 +------ + +Fixed a bug with ``atomic`` when run async with a connection pool. + +For example: + +.. code-block:: python + + atomic = Band._meta.db.atomic() + atomic.add(query_1, query_1) + # This was failing: + await atomic.run() + +Thanks to @Anton-Karpenko for reporting this issue. + +------------------------------------------------------------------------------- + 0.76.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index b15a78f1d..66e972524 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.76.0" +__VERSION__ = "0.76.1" From ba6c70567cef3323c0ec76890ada90d6eeb5536f Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 15 Jun 2022 09:28:52 +0100 Subject: [PATCH 329/727] added `refresh` method to `Table` (#545) * added `refresh` method to `Table` It re-fetches the data from the db. * make sure primary key is excluded * simplify refresh API There's no need for the `include_columns` and `exclude_columns` arguments, because to exclude you can do `band.refresh(columns=Band.all_columns(exclude=Band.name]))`. * tweak docstring for `refresh` --- docs/src/piccolo/api_reference/index.rst | 10 +++ docs/src/piccolo/asgi/index.rst | 3 +- docs/src/piccolo/query_types/objects.rst | 17 ++++ piccolo/query/base.py | 4 +- piccolo/query/methods/__init__.py | 1 + piccolo/query/methods/insert.py | 2 +- piccolo/query/methods/refresh.py | 102 +++++++++++++++++++++++ piccolo/table.py | 35 ++++++-- tests/table/test_refresh.py | 84 +++++++++++++++++++ 9 files changed, 249 insertions(+), 9 deletions(-) create mode 100644 piccolo/query/methods/refresh.py create mode 100644 tests/table/test_refresh.py diff --git a/docs/src/piccolo/api_reference/index.rst b/docs/src/piccolo/api_reference/index.rst index 30eb0f2b0..1e1395174 100644 --- a/docs/src/piccolo/api_reference/index.rst +++ b/docs/src/piccolo/api_reference/index.rst @@ -11,6 +11,16 @@ Table ------------------------------------------------------------------------------- +Refresh +------- + +.. currentmodule:: piccolo.query.methods.refresh + +.. autoclass:: Refresh + :members: + +------------------------------------------------------------------------------- + LazyTableReference ------------------ diff --git a/docs/src/piccolo/asgi/index.rst b/docs/src/piccolo/asgi/index.rst index b930392d0..1da8d7a64 100644 --- a/docs/src/piccolo/asgi/index.rst +++ b/docs/src/piccolo/asgi/index.rst @@ -33,7 +33,8 @@ Which to use? ============= All are great choices. FastAPI is built on top of Starlette, so they're -very similar. FastAPI is useful if you want to document a REST API. +very similar. FastAPI and BlackSheep are great if you want to document a REST +API, as they have built-in OpenAPI support. ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/query_types/objects.rst b/docs/src/piccolo/query_types/objects.rst index f33f03922..839317a19 100644 --- a/docs/src/piccolo/query_types/objects.rst +++ b/docs/src/piccolo/query_types/objects.rst @@ -253,6 +253,23 @@ the columns: ------------------------------------------------------------------------------- +refresh +------- + +If you have an object which has gotten stale, and want to refresh it, so it +has the latest data from the database, you can use the +:meth:`refresh ` method. + +.. code-block:: python + + # If we have an instance: + band = await Band.objects().first() + + # And it has gotten stale, we can refresh it: + await band.refresh() + +------------------------------------------------------------------------------- + Query clauses ------------- diff --git a/piccolo/query/base.py b/piccolo/query/base.py index 535e98049..591c8b20a 100644 --- a/piccolo/query/base.py +++ b/piccolo/query/base.py @@ -61,8 +61,8 @@ async def _process_results(self, results): # noqa: C901 else: raw = [] - if hasattr(self, "run_callback"): - self.run_callback(raw) + if hasattr(self, "_raw_response_callback"): + self._raw_response_callback(raw) output: t.Optional[OutputDelegate] = getattr( self, "output_delegate", None diff --git a/piccolo/query/methods/__init__.py b/piccolo/query/methods/__init__.py index aaf1d41b1..6c1854381 100644 --- a/piccolo/query/methods/__init__.py +++ b/piccolo/query/methods/__init__.py @@ -8,6 +8,7 @@ from .insert import Insert from .objects import Objects from .raw import Raw +from .refresh import Refresh from .select import Avg, Max, Min, Select, Sum from .table_exists import TableExists from .update import Update diff --git a/piccolo/query/methods/insert.py b/piccolo/query/methods/insert.py index 313c7e1c1..0a9825216 100644 --- a/piccolo/query/methods/insert.py +++ b/piccolo/query/methods/insert.py @@ -22,7 +22,7 @@ def add(self, *instances: Table) -> Insert: self.add_delegate.add(*instances, table_class=self.table) return self - def run_callback(self, results): + def _raw_response_callback(self, results): """ Assign the ids of the created rows to the model instances. """ diff --git a/piccolo/query/methods/refresh.py b/piccolo/query/methods/refresh.py new file mode 100644 index 000000000..9152638e9 --- /dev/null +++ b/piccolo/query/methods/refresh.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import typing as t +from dataclasses import dataclass + +from piccolo.utils.sync import run_sync + +if t.TYPE_CHECKING: # pragma: no cover + from piccolo.columns import Column + from piccolo.table import Table + + +@dataclass +class Refresh: + """ + Used to refresh :class:`Table ` instances with the + latest data data from the database. Accessible via + :meth:`refresh `. + + :param instance: + The instance to refresh. + :param columns: + Which columns to refresh - it not specified, then all columns are + refreshed. + + """ + + instance: Table + columns: t.Optional[t.Sequence[Column]] = None + + @property + def _columns(self) -> t.Sequence[Column]: + """ + Works out which columns the user wants to refresh. + """ + if self.columns: + return self.columns + + return [ + i for i in self.instance._meta.columns if not i._meta.primary_key + ] + + async def run(self) -> Table: + """ + Run it asynchronously. For example:: + + await my_instance.refresh().run() + + # or for convenience: + await my_instance.refresh() + + Modifies the instance in place, but also returns it as a convenience. + + """ + + instance = self.instance + + if not instance._exists_in_db: + raise ValueError("The instance doesn't exist in the database.") + + pk_column = instance._meta.primary_key + + primary_key_value = getattr(instance, pk_column._meta.name, None) + + if primary_key_value is None: + raise ValueError("The instance's primary key value isn't defined.") + + columns = self._columns + if not columns: + raise ValueError("No columns to fetch.") + + updated_values = ( + await instance.select(*columns) + .where(pk_column == primary_key_value) + .first() + ) + + if updated_values is None: + raise ValueError( + "The object doesn't exist in the database any more." + ) + + for key, value in updated_values.items(): + setattr(instance, key, value) + + return instance + + def __await__(self): + """ + If the user doesn't explicity call :meth:`run`, proxy to it as a + convenience. + """ + return self.run().__await__() + + def run_sync(self) -> Table: + """ + Run it synchronously. For example:: + + my_instance.refresh().run_sync() + + """ + return run_sync(self.run()) diff --git a/piccolo/table.py b/piccolo/table.py index d057c0133..6306f2a03 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -42,6 +42,7 @@ ) from piccolo.query.methods.create_index import CreateIndex from piccolo.query.methods.indexes import Indexes +from piccolo.query.methods.refresh import Refresh from piccolo.querystring import QueryString, Unquoted from piccolo.utils import _camel_to_snake from piccolo.utils.graphlib import TopologicalSorter @@ -338,7 +339,7 @@ def from_dict(cls, data: t.Dict[str, t.Any]) -> Table: ########################################################################### def save( - self, columns: t.Optional[t.List[t.Union[Column, str]]] = None + self, columns: t.Optional[t.Sequence[t.Union[Column, str]]] = None ) -> t.Union[Insert, Update]: """ A proxy to an insert or update query. @@ -365,9 +366,7 @@ def save( # Pre-existing row - update if columns is None: column_instances = [ - i - for i in cls._meta.columns - if i._meta.name != self._meta.primary_key._meta.name + i for i in cls._meta.columns if not i._meta.primary_key ] else: column_instances = [ @@ -403,6 +402,32 @@ def remove(self) -> Delete: self.__class__._meta.primary_key == primary_key_value ) + def refresh( + self, columns: t.Optional[t.Sequence[Column]] = None + ) -> Refresh: + """ + Used to fetch the latest data for this instance from the database. + Modifies the instance in place, but also returns it as a convenience. + + :param columns: + If you only want to refresh certain columns, specify them here. + Otherwise all columns are refreshed. + + Example usage:: + + # Get an instance from the database. + instance = await Band.objects.first() + + # Later on we can refresh this instance with the latest data + # from the database, in case it has gotten stale. + await instance.refresh() + + # Alternatively, running it synchronously: + instance.refresh().run_sync() + + """ + return Refresh(instance=self, columns=columns) + def get_related(self, foreign_key: t.Union[ForeignKey, str]) -> Objects: """ Used to fetch a ``Table`` instance, for the target of a foreign key. @@ -720,7 +745,7 @@ def all_related( @classmethod def all_columns( - cls, exclude: t.List[t.Union[str, Column]] = None + cls, exclude: t.Sequence[t.Union[str, Column]] = None ) -> t.List[Column]: """ Used in conjunction with ``select`` queries. Just as we can use diff --git a/tests/table/test_refresh.py b/tests/table/test_refresh.py new file mode 100644 index 000000000..22e01d8f7 --- /dev/null +++ b/tests/table/test_refresh.py @@ -0,0 +1,84 @@ +from tests.base import DBTestCase +from tests.example_apps.music.tables import Band + + +class TestRefresh(DBTestCase): + def setUp(self): + super().setUp() + self.insert_rows() + + def test_refresh(self): + """ + Make sure ``refresh`` works, with not columns specified. + """ + # Fetch an instance from the database. + band: Band = Band.objects().get(Band.name == "Pythonistas").run_sync() + initial_data = band.to_dict() + + # Modify the data in the database. + Band.update( + {Band.name: Band.name + "!!!", Band.popularity: 8000} + ).where(Band.name == "Pythonistas").run_sync() + + # Refresh `band`, and make sure it has the correct data. + band.refresh().run_sync() + + self.assertTrue(band.name == "Pythonistas!!!") + self.assertTrue(band.popularity == 8000) + self.assertTrue(band.id == initial_data["id"]) + + def test_columns(self): + """ + Make sure ``refresh`` works, when columns are specified. + """ + # Fetch an instance from the database. + band: Band = Band.objects().get(Band.name == "Pythonistas").run_sync() + initial_data = band.to_dict() + + # Modify the data in the database. + Band.update( + {Band.name: Band.name + "!!!", Band.popularity: 8000} + ).where(Band.name == "Pythonistas").run_sync() + + # Refresh `band`, and make sure it has the correct data. + query = band.refresh(columns=[Band.name]) + self.assertEqual( + [i._meta.name for i in query._columns], + ["name"], + ) + query.run_sync() + + self.assertTrue(band.name == "Pythonistas!!!") + self.assertTrue(band.popularity == initial_data["popularity"]) + self.assertTrue(band.id == initial_data["id"]) + + def test_error_when_not_in_db(self): + """ + Make sure we can't refresh an instance which hasn't been saved in the + database. + """ + band = Band() + + with self.assertRaises(ValueError) as manager: + band.refresh().run_sync() + + self.assertEqual( + "The instance doesn't exist in the database.", + str(manager.exception), + ) + + def test_error_when_pk_in_none(self): + """ + Make sure we can't refresh an instance when the primary key value isn't + set. + """ + band: Band = Band.objects().first().run_sync() + band.id = None + + with self.assertRaises(ValueError) as manager: + band.refresh().run_sync() + + self.assertEqual( + "The instance's primary key value isn't defined.", + str(manager.exception), + ) From 079665c70e8176ef6670605422b5d38309e8291f Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 15 Jun 2022 09:32:35 +0100 Subject: [PATCH 330/727] bumped version --- CHANGES.rst | 19 +++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 99a382fbb..94c5bffe5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,25 @@ Changes ======= +0.77.0 +------ + +Added the ``refresh`` method. If you have an object which has gotten stale, and +want to refresh it, so it has the latest data from the database, you can now do +this: + +.. code-block:: python + + # If we have an instance: + band = await Band.objects().first() + + # And it has gotten stale, we can refresh it: + await band.refresh() + +Thanks to @trondhindenes for suggesting this feature. + +------------------------------------------------------------------------------- + 0.76.1 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 66e972524..313fa56e8 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.76.1" +__VERSION__ = "0.77.0" From ccb6d6685e16a3d8908106b9b351f7af17c3ca10 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 15 Jun 2022 10:36:19 +0100 Subject: [PATCH 331/727] add docs for using joins in `where` clauses (#547) --- docs/src/piccolo/query_clauses/where.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/src/piccolo/query_clauses/where.rst b/docs/src/piccolo/query_clauses/where.rst index b80c955b0..245f3b88a 100644 --- a/docs/src/piccolo/query_clauses/where.rst +++ b/docs/src/piccolo/query_clauses/where.rst @@ -252,3 +252,15 @@ untrusted source, otherwise it could lead to a SQL injection attack. await Band.select().where( WhereRaw("name = 'Pythonistas'") | (Band.popularity > 1000) ) + +------------------------------------------------------------------------------- + +Joins +----- + +The ``where`` clause has full support for joins. For example: + +.. code-block:: python + + >>> await Band.select(Band.name).where(Band.manager.name == 'Guido') + [{'name': 'Pythonistas'}] From 3b23eed3777430e7aedc1002229f54e737866be0 Mon Sep 17 00:00:00 2001 From: backwardspy Date: Thu, 16 Jun 2022 13:01:18 +0100 Subject: [PATCH 332/727] add callbacks to queries (#540) * add callback for query success * accept multiple callbacks, remove success flag * add support for different kinds of callbacks * allow callbacks to transform results * store callbacks more efficiently * add callback delegate to objects query * document callback clauses for select and objects queries * fix callback tests on python 3.7 --- docs/src/piccolo/query_clauses/callback.rst | 68 ++++++ docs/src/piccolo/query_clauses/index.rst | 1 + docs/src/piccolo/query_types/objects.rst | 5 + docs/src/piccolo/query_types/select.rst | 5 + piccolo/query/base.py | 26 ++- piccolo/query/methods/objects.py | 13 ++ piccolo/query/methods/select.py | 13 ++ piccolo/query/mixins.py | 58 +++++ tests/base.py | 7 + tests/table/test_callback.py | 228 ++++++++++++++++++++ 10 files changed, 420 insertions(+), 4 deletions(-) create mode 100644 docs/src/piccolo/query_clauses/callback.rst create mode 100644 tests/table/test_callback.py diff --git a/docs/src/piccolo/query_clauses/callback.rst b/docs/src/piccolo/query_clauses/callback.rst new file mode 100644 index 000000000..da2510311 --- /dev/null +++ b/docs/src/piccolo/query_clauses/callback.rst @@ -0,0 +1,68 @@ +.. _callback: + +callback +======== + +You can use ``callback`` clauses with the following queries: + +* :ref:`Select` +* :ref:`Objects` + +Callbacks are used to run arbitrary code after a query completes. + +Callback handlers +----------------- + +A callback handler is a function or coroutine that takes query results as +its only parameter. + +For example, you can automatically print the result of a select query using +``print`` as a callback handler: + +.. code-block:: python + + >>> await Band.select(Band.name).callback(print) + [{'name': 'Pythonistas'}] + +Likewise for an objects query: + +.. code-block:: python + + >>> await Band.objects().callback(print) + [] + +Transforming results +-------------------- + +Callback handlers are able to modify the results of a query by returning a +value. Note that in the previous examples, the queries returned ``None`` since +``print`` itself returns ``None``. + +To modify query results with a custom callback handler: + +.. code-block:: python + + >>> def uppercase_name(band): + return band.name.upper() + + >>> await Band.objects().first().callback(uppercase_name) + 'PYTHONISTAS' + +Multiple callbacks +------------------ + +You can add as many callbacks to a query as you like. This can be done in two +ways. + +Passing a list of callbacks: + +.. code-block:: python + + Band.select(Band.name).callback([handler_a, handler_b]) + +Chaining ``callback`` clauses: + +.. code-block:: python + + Band.select(Band.name).callback(handler_a).callback(handler_b) + diff --git a/docs/src/piccolo/query_clauses/index.rst b/docs/src/piccolo/query_clauses/index.rst index e4129c83a..0fe8bc52f 100644 --- a/docs/src/piccolo/query_clauses/index.rst +++ b/docs/src/piccolo/query_clauses/index.rst @@ -16,6 +16,7 @@ by modifying the return values. ./offset ./order_by ./output + ./callback ./where ./batch ./freeze diff --git a/docs/src/piccolo/query_types/objects.rst b/docs/src/piccolo/query_types/objects.rst index 839317a19..1028c0f21 100644 --- a/docs/src/piccolo/query_types/objects.rst +++ b/docs/src/piccolo/query_types/objects.rst @@ -303,6 +303,11 @@ output See :ref:`output`. +callback +~~~~~~~~ + +See :ref:`callback`. + where ~~~~~ diff --git a/docs/src/piccolo/query_types/select.rst b/docs/src/piccolo/query_types/select.rst index 0c85500d5..28cd364a9 100644 --- a/docs/src/piccolo/query_types/select.rst +++ b/docs/src/piccolo/query_types/select.rst @@ -330,6 +330,11 @@ output See :ref:`output`. +callback +~~~~~~~~ + +See :ref:`callback`. + where ~~~~~ diff --git a/piccolo/query/base.py b/piccolo/query/base.py index 591c8b20a..b1857df87 100644 --- a/piccolo/query/base.py +++ b/piccolo/query/base.py @@ -5,14 +5,14 @@ from time import time from piccolo.columns.column_types import JSON, JSONB -from piccolo.query.mixins import ColumnsDelegate +from piccolo.query.mixins import CallbackType, ColumnsDelegate from piccolo.querystring import QueryString from piccolo.utils.encoding import dump_json, load_json from piccolo.utils.objects import make_nested_object from piccolo.utils.sync import run_sync if t.TYPE_CHECKING: # pragma: no cover - from piccolo.query.mixins import OutputDelegate + from piccolo.query.mixins import CallbackDelegate, OutputDelegate from piccolo.table import Table # noqa @@ -198,18 +198,36 @@ async def run(self, node: t.Optional[str] = None, in_pool: bool = True): querystrings = self.querystrings + callback: t.Optional[CallbackDelegate] = getattr( + self, "callback_delegate", None + ) + if len(querystrings) == 1: results = await engine.run_querystring( querystrings[0], in_pool=in_pool ) - return await self._process_results(results) + processed_results = await self._process_results(results) + + if callback: + processed_results = await callback.invoke( + processed_results, kind=CallbackType.success + ) + + return processed_results else: responses = [] for querystring in querystrings: results = await engine.run_querystring( querystring, in_pool=in_pool ) - responses.append(await self._process_results(results)) + processed_results = await self._process_results(results) + + if callback: + processed_results = await callback.invoke( + processed_results, kind=CallbackType.success + ) + + responses.append(processed_results) return responses def run_sync(self, timed=False, in_pool=False, *args, **kwargs): diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index 050edd169..a6c082691 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -9,6 +9,8 @@ from piccolo.engine.base import Batch from piccolo.query.base import Query from piccolo.query.mixins import ( + CallbackDelegate, + CallbackType, LimitDelegate, OffsetDelegate, OrderByDelegate, @@ -124,6 +126,7 @@ class Objects(Query): "offset_delegate", "order_by_delegate", "output_delegate", + "callback_delegate", "prefetch_delegate", "where_delegate", ) @@ -140,6 +143,7 @@ def __init__( self.order_by_delegate = OrderByDelegate() self.output_delegate = OutputDelegate() self.output_delegate._output.as_objects = True + self.callback_delegate = CallbackDelegate() self.prefetch_delegate = PrefetchDelegate() self.prefetch(*prefetch) self.where_delegate = WhereDelegate() @@ -150,6 +154,15 @@ def output(self, load_json: bool = False) -> Objects: ) return self + def callback( + self, + callbacks: t.Union[t.Callable, t.List[t.Callable]], + *, + on: CallbackType = CallbackType.success, + ) -> Objects: + self.callback_delegate.callback(callbacks, on=on) + return self + def limit(self, number: int) -> Objects: self.limit_delegate.limit(number) return self diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 9b05f85b8..0c31e8373 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -12,6 +12,8 @@ from piccolo.engine.base import Batch from piccolo.query.base import Query from piccolo.query.mixins import ( + CallbackDelegate, + CallbackType, ColumnsDelegate, DistinctDelegate, GroupByDelegate, @@ -221,6 +223,7 @@ class Select(Query): "offset_delegate", "order_by_delegate", "output_delegate", + "callback_delegate", "where_delegate", ) @@ -243,6 +246,7 @@ def __init__( self.offset_delegate = OffsetDelegate() self.order_by_delegate = OrderByDelegate() self.output_delegate = OutputDelegate() + self.callback_delegate = CallbackDelegate() self.where_delegate = WhereDelegate() self.columns(*columns_list) @@ -461,6 +465,15 @@ def output( ) return self + def callback( + self, + callbacks: t.Union[t.Callable, t.List[t.Callable]], + *, + on: CallbackType = CallbackType.success, + ) -> Select: + self.callback_delegate.callback(callbacks, on=on) + return self + def where(self, *where: Combinable) -> Select: self.where_delegate.where(*where) return self diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index af8197ced..75318aee3 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -1,7 +1,9 @@ from __future__ import annotations +import asyncio import typing as t from dataclasses import dataclass, field +from enum import Enum, auto from piccolo.columns import And, Column, Or, Where from piccolo.columns.column_types import ForeignKey @@ -92,6 +94,16 @@ def copy(self) -> Output: ) +class CallbackType(Enum): + success = auto() + + +@dataclass +class Callback: + kind: CallbackType + target: t.Callable + + @dataclass class WhereDelegate: @@ -237,6 +249,52 @@ def copy(self) -> OutputDelegate: return self.__class__(_output=_output) +@dataclass +class CallbackDelegate: + """ + Example usage: + + .callback(my_handler_function) + .callback(print, on=CallbackType.success) + .callback(my_handler_coroutine) + .callback([handler1, handler2]) + """ + + _callbacks: t.Dict[CallbackType, t.List[Callback]] = field( + default_factory=lambda: {kind: [] for kind in CallbackType} + ) + + def callback( + self, + callbacks: t.Union[t.Callable, t.List[t.Callable]], + *, + on: CallbackType, + ): + if isinstance(callbacks, list): + self._callbacks[on].extend( + Callback(kind=on, target=callback) for callback in callbacks + ) + else: + self._callbacks[on].append(Callback(kind=on, target=callbacks)) + + async def invoke(self, results: t.Any, *, kind: CallbackType) -> t.Any: + """ + Utility function that invokes the registered callbacks in the correct + way, handling both sync and async callbacks. Only callbacks of the + given kind are invoked. + Results are passed through the callbacks in the order they were added, + with each callback able to transform them. This function returns the + transformed results. + """ + for callback in self._callbacks[kind]: + if asyncio.iscoroutinefunction(callback.target): + results = await callback.target(results) + else: + results = callback.target(results) + + return results + + @dataclass class PrefetchDelegate: """ diff --git a/tests/base.py b/tests/base.py index 560da9b90..5c0375cff 100644 --- a/tests/base.py +++ b/tests/base.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import sys import typing as t from unittest import TestCase @@ -45,6 +46,12 @@ class AsyncMock(MagicMock): Python 3.7. """ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # this makes asyncio.iscoroutinefunction(AsyncMock()) return True + self._is_coroutine = asyncio.coroutines._is_coroutine + async def __call__(self, *args, **kwargs): return super(AsyncMock, self).__call__(*args, **kwargs) diff --git a/tests/table/test_callback.py b/tests/table/test_callback.py new file mode 100644 index 000000000..46ea60f97 --- /dev/null +++ b/tests/table/test_callback.py @@ -0,0 +1,228 @@ +from unittest.mock import Mock + +from tests.base import AsyncMock, DBTestCase +from tests.example_apps.music.tables import Band + + +def identity(x): + """Returns the input. Used as the side effect for mock callbacks.""" + return x + + +def get_name(results): + return results["name"] + + +async def uppercase(name): + """Async to ensure coroutines are called correctly.""" + return name.upper() + + +def limit(name): + return name[:6] + + +class TestNoCallbackSelect(DBTestCase): + def test_no_callback(self): + """ + Just check we don't get any "NoneType is not callable" kind of errors + when we run a select query without setting any callbacks. + """ + self.insert_row() + Band.select(Band.name).run_sync() + + +class TestNoCallbackObjects(DBTestCase): + def test_no_callback(self): + """ + Just check we don't get any "NoneType is not callable" kind of errors + when we run an objects query without setting any callbacks. + """ + self.insert_row() + Band.objects().run_sync() + + +class TestCallbackSuccessesSelect(DBTestCase): + def test_callback_sync(self): + self.insert_row() + + callback_handler = Mock(return_value="it worked") + result = Band.select(Band.name).callback(callback_handler).run_sync() + callback_handler.assert_called_once_with([{"name": "Pythonistas"}]) + self.assertEqual(result, "it worked") + + def test_callback_async(self): + self.insert_row() + + callback_handler = AsyncMock(return_value="it worked") + result = Band.select(Band.name).callback(callback_handler).run_sync() + callback_handler.assert_called_once_with([{"name": "Pythonistas"}]) + self.assertEqual(result, "it worked") + + +class TestCallbackSuccessesObjects(DBTestCase): + def test_callback_sync(self): + self.insert_row() + + callback_handler = Mock(return_value="it worked") + result = Band.objects().callback(callback_handler).run_sync() + callback_handler.assert_called_once() + + args = callback_handler.call_args[0][0] + self.assertIsInstance(args, list) + self.assertIsInstance(args[0], Band) + self.assertEqual(args[0].name, "Pythonistas") + self.assertEqual(result, "it worked") + + def test_callback_async(self): + self.insert_row() + + callback_handler = AsyncMock(return_value="it worked") + result = Band.objects().callback(callback_handler).run_sync() + callback_handler.assert_called_once() + + args = callback_handler.call_args[0][0] + self.assertIsInstance(args, list) + self.assertIsInstance(args[0], Band) + self.assertEqual(args[0].name, "Pythonistas") + self.assertEqual(result, "it worked") + + +class TestMultipleCallbacksSelect(DBTestCase): + def test_all_sync(self): + self.insert_row() + + handlers = [ + Mock(side_effect=identity), + Mock(side_effect=identity), + Mock(side_effect=identity), + ] + Band.select(Band.name).callback(handlers).run_sync() + + for handler in handlers: + handler.assert_called_once_with([{"name": "Pythonistas"}]) + + def test_all_sync_chained(self): + self.insert_row() + + handlers = [ + Mock(side_effect=identity), + Mock(side_effect=identity), + Mock(side_effect=identity), + ] + + ( + Band.select(Band.name) + .callback(handlers[0]) + .callback(handlers[1]) + .callback(handlers[2]) + .run_sync() + ) + + for handler in handlers: + handler.assert_called_once_with([{"name": "Pythonistas"}]) + + def test_all_async(self): + self.insert_row() + + handlers = [ + AsyncMock(side_effect=identity), + AsyncMock(side_effect=identity), + AsyncMock(side_effect=identity), + ] + Band.select(Band.name).callback(handlers).run_sync() + + for handler in handlers: + handler.assert_called_once_with([{"name": "Pythonistas"}]) + + def test_all_async_chained(self): + self.insert_row() + + handlers = [ + AsyncMock(side_effect=identity), + AsyncMock(side_effect=identity), + AsyncMock(side_effect=identity), + ] + ( + Band.select(Band.name) + .callback(handlers[0]) + .callback(handlers[1]) + .callback(handlers[2]) + .run_sync() + ) + for handler in handlers: + handler.assert_called_once_with([{"name": "Pythonistas"}]) + + def test_mixed(self): + self.insert_row() + + handlers = [ + Mock(side_effect=identity), + AsyncMock(side_effect=identity), + Mock(side_effect=identity), + ] + Band.select(Band.name).callback(handlers).run_sync() + + for handler in handlers: + handler.assert_called_once_with([{"name": "Pythonistas"}]) + + def test_mixed_chained(self): + self.insert_row() + + handlers = [ + Mock(side_effect=identity), + AsyncMock(side_effect=identity), + Mock(side_effect=identity), + ] + + ( + Band.select(Band.name) + .callback(handlers[0]) + .callback(handlers[1]) + .callback(handlers[2]) + .run_sync() + ) + + for handler in handlers: + handler.assert_called_once_with([{"name": "Pythonistas"}]) + + +class TestCallbackTransformDataSelect(DBTestCase): + def test_transform(self): + self.insert_row() + + result = ( + Band.select(Band.name) + .first() + .callback([get_name, uppercase, limit]) + .run_sync() + ) + + self.assertEqual(result, "PYTHON") + + def test_transform_chain(self): + self.insert_row() + + result = ( + Band.select(Band.name) + .first() + .callback(get_name) + .callback(uppercase) + .callback(limit) + .run_sync() + ) + + self.assertEqual(result, "PYTHON") + + def test_transform_mixed(self): + self.insert_row() + + result = ( + Band.select(Band.name) + .first() + .callback([get_name, uppercase]) + .callback(limit) + .run_sync() + ) + + self.assertEqual(result, "PYTHON") From 3426794fc23c3db107132bd030baa386d6ac1394 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 16 Jun 2022 14:11:17 +0100 Subject: [PATCH 333/727] ordered query clauses alphabetically --- docs/src/piccolo/query_types/objects.rst | 22 +++++++++++----------- docs/src/piccolo/query_types/select.rst | 19 +++++++++---------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/docs/src/piccolo/query_types/objects.rst b/docs/src/piccolo/query_types/objects.rst index 1028c0f21..b9a0a15b2 100644 --- a/docs/src/piccolo/query_types/objects.rst +++ b/docs/src/piccolo/query_types/objects.rst @@ -274,10 +274,20 @@ Query clauses ------------- batch -~~~~~~~ +~~~~~ See :ref:`batch`. +callback +~~~~~~~~ + +See :ref:`callback`. + +first +~~~~~ + +See :ref:`first`. + limit ~~~~~ @@ -288,11 +298,6 @@ offset See :ref:`offset`. -first -~~~~~ - -See :ref:`first`. - order_by ~~~~~~~~ @@ -303,11 +308,6 @@ output See :ref:`output`. -callback -~~~~~~~~ - -See :ref:`callback`. - where ~~~~~ diff --git a/docs/src/piccolo/query_types/select.rst b/docs/src/piccolo/query_types/select.rst index 28cd364a9..643aa789e 100644 --- a/docs/src/piccolo/query_types/select.rst +++ b/docs/src/piccolo/query_types/select.rst @@ -263,6 +263,11 @@ batch See :ref:`batch`. +callback +~~~~~~~~ + +See :ref:`callback`. + columns ~~~~~~~ @@ -294,6 +299,10 @@ columns. # Or just define it one go: await Band.select().columns(Band.name, Band.manager) +distinct +~~~~~~~~ + +See :ref:`distinct`. first ~~~~~ @@ -315,11 +324,6 @@ offset See :ref:`offset`. -distinct -~~~~~~~~ - -See :ref:`distinct`. - order_by ~~~~~~~~ @@ -330,11 +334,6 @@ output See :ref:`output`. -callback -~~~~~~~~ - -See :ref:`callback`. - where ~~~~~ From ed239215256fc91182dcba02eaf943c8d251838b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 16 Jun 2022 14:11:27 +0100 Subject: [PATCH 334/727] bumped version --- CHANGES.rst | 48 +++++++++++++++++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 94c5bffe5..27fab9918 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,54 @@ Changes ======= +0.78.0 +------ + +Added the ``callback`` clause to ``select`` and ``objects`` queries (courtesy +@backwardspy). For example: + +.. code-block:: python + + >>> await Band.select().callback(my_callback) + +The callback can be a normal function or async function, which is called when +the query is successful. The callback can be used to modify the query's output. + +It allows for some interesting and powerful code. Here's a very simple example +where we modify the query's output: + +.. code-block:: python + + >>> def get_uppercase_names() -> Select: + ... def make_uppercase(response): + ... return [{'name': i['name'].upper()} for i in response] + ... + ... return Band.select(Band.name).callback(make_uppercase) + + >>> await get_uppercase_names().where(Band.name == 'Pythonistas') + [{'name': 'PYTHONISTAS'}] + +Here's another example, where we perform validation on the query's output: + +.. code-block:: python + + >>> def get_concerts() -> Select: + ... def check_length(response): + ... if len(response) == 0: + ... raise ValueError('No concerts!') + ... return response + ... + ... return Concert.select().callback(check_length) + + >>> await get_concerts().where(Concert.band_1.name == 'Terrible Band') + ValueError: No concerts! + +At the moment, callbacks are just triggered when a query is successful, but in +the future other callbacks will be added, to hook into more of Piccolo's +internals. + +------------------------------------------------------------------------------- + 0.77.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 313fa56e8..c382b92e0 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.77.0" +__VERSION__ = "0.78.0" From c014204cb157233373c1c7ef6fc0d29deb65d700 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 16 Jun 2022 14:17:44 +0100 Subject: [PATCH 335/727] tweaked release notes for 0.78.0 - added a better example --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 27fab9918..c0774adad 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -25,7 +25,7 @@ where we modify the query's output: ... ... return Band.select(Band.name).callback(make_uppercase) - >>> await get_uppercase_names().where(Band.name == 'Pythonistas') + >>> await get_uppercase_names().where(Band.manager.name == 'Guido') [{'name': 'PYTHONISTAS'}] Here's another example, where we perform validation on the query's output: From 355de578d66b17aeb205a108364c3ae1c75ea19b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 17 Jun 2022 16:07:44 +0100 Subject: [PATCH 336/727] override `Table` repr (#550) * override `Table`` repr * fix grammar in docstring --- piccolo/table.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/piccolo/table.py b/piccolo/table.py index 6306f2a03..3c5a5dd07 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -139,6 +139,31 @@ class TableMetaclass(type): def __str__(cls): return cls._table_str() + def __repr__(cls): + """ + We override this, because by default Python will output something + like:: + + >>> repr(MyTable) + + + It's a very common pattern in Piccolo and its sister libraries to + have ``Table`` class types as default values:: + + # `SessionsBase` is a `Table` subclass: + def session_auth( + session_table: t.Type[SessionsBase] = SessionsBase + ): + ... + + This looks terrible in Sphinx's autodoc output, as Python's default + repr contains angled brackets, which breaks the HTML output. So we just + output the name instead. The user can still easily find which module a + ``Table`` subclass belongs to by using ``MyTable.__module__``. + + """ + return cls.__name__ + class Table(metaclass=TableMetaclass): """ From 228859684736973e42d791ef8e248c597148590a Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 17 Jun 2022 20:18:59 +0100 Subject: [PATCH 337/727] bumped version --- CHANGES.rst | 10 ++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index c0774adad..59ee15494 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,16 @@ Changes ======= +0.79.0 +------ + +Added a custom ``__repr__`` method to ``Table``'s metaclass. It's needed to +improve the appearance of our Sphinx docs. See +`issue 549 `_ for more +details. + +------------------------------------------------------------------------------- + 0.78.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index c382b92e0..039899e69 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.78.0" +__VERSION__ = "0.79.0" From aa4795d6aa6965e4f2c566fc8bf344f9c4709761 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 21 Jun 2022 16:02:21 +0100 Subject: [PATCH 338/727] `JSONB` `as_alias` fix when using joins (#553) * added a choices example to the playground * simplified `get_select_string` logic --- piccolo/apps/playground/commands/run.py | 7 ++ piccolo/columns/base.py | 109 +++++++++++++++--------- piccolo/columns/column_types.py | 40 ++++----- piccolo/columns/m2m.py | 2 +- piccolo/columns/readable.py | 4 +- piccolo/query/base.py | 9 +- piccolo/query/methods/select.py | 60 ++++++------- piccolo/query/mixins.py | 4 +- piccolo/table.py | 3 +- tests/columns/test_jsonb.py | 39 ++++++++- tests/conftest.py | 1 + 11 files changed, 174 insertions(+), 104 deletions(-) diff --git a/piccolo/apps/playground/commands/run.py b/piccolo/apps/playground/commands/run.py index 598fb7e1d..582a87633 100644 --- a/piccolo/apps/playground/commands/run.py +++ b/piccolo/apps/playground/commands/run.py @@ -6,6 +6,7 @@ import sys import uuid from decimal import Decimal +from enum import Enum from piccolo.columns import ( JSON, @@ -47,8 +48,14 @@ class Concert(Table): class Ticket(Table): + class TicketType(Enum): + sitting = "sitting" + standing = "standing" + premium = "premium" + concert = ForeignKey(Concert) price = Numeric(digits=(5, 2)) + ticket_type = Varchar(choices=TicketType, default=TicketType.standing) class DiscountCode(Table): diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 5a92322fb..0cd435d3a 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -203,6 +203,8 @@ def table(self) -> t.Type[Table]: call_chain: t.List["ForeignKey"] = field(default_factory=list) table_alias: t.Optional[str] = None + ########################################################################### + @property def engine_type(self) -> str: engine = self.table._meta.db @@ -234,21 +236,52 @@ def get_choices_dict(self) -> t.Optional[t.Dict[str, t.Any]]: return output + ########################################################################### + + def get_default_alias(self): + column_name = self.db_column_name + + if self.call_chain: + column_name = ( + "$".join( + t.cast(str, i._meta.db_column_name) + for i in self.call_chain + ) + + f"${column_name}" + ) + + return column_name + + def _get_path(self, include_quotes: bool = False): + column_name = self.db_column_name + + if self.call_chain: + table_alias = self.call_chain[-1]._meta.table_alias + if include_quotes: + return f'"{table_alias}"."{column_name}"' + else: + return f"{table_alias}.{column_name}" + else: + if include_quotes: + return f'"{self.table._meta.tablename}"."{column_name}"' + else: + return f"{self.table._meta.tablename}.{column_name}" + def get_full_name( - self, just_alias: bool = False, include_quotes: bool = False + self, with_alias: bool = True, include_quotes: bool = True ) -> str: """ Returns the full column name, taking into account joins. - :param just_alias: + :param with_alias: Examples: .. code-block python:: - >>> Band.manager.name._meta.get_full_name(just_alias=True) + >>> Band.manager.name._meta.get_full_name(with_alias=False) 'band$manager.name' - >>> Band.manager.name._meta.get_full_name(just_alias=False) + >>> Band.manager.name._meta.get_full_name(with_alias=True) 'band$manager.name AS "manager$name"' :param include_quotes: @@ -267,27 +300,16 @@ def get_full_name( """ - column_name = self.db_column_name + full_name = self._get_path(include_quotes=include_quotes) - if not self.call_chain: + if with_alias and self.call_chain: + alias = self.get_default_alias() if include_quotes: - return f'"{self.table._meta.tablename}"."{column_name}"' + full_name += f' AS "{alias}"' else: - return f"{self.table._meta.tablename}.{column_name}" - - column_name = ( - "$".join( - t.cast(str, i._meta.db_column_name) for i in self.call_chain - ) - + f"${column_name}" - ) - - if include_quotes: - alias = f'"{self.call_chain[-1]._meta.table_alias}"."{self.name}"' - else: - alias = f"{self.call_chain[-1]._meta.table_alias}.{self.name}" + full_name += f" AS {alias}" - return alias if just_alias else f'{alias} AS "{column_name}"' + return full_name ########################################################################### @@ -321,22 +343,24 @@ def __deepcopy__(self, memo) -> ColumnMeta: class Selectable(metaclass=ABCMeta): - alias: t.Optional[str] + _alias: t.Optional[str] @abstractmethod - def get_select_string(self, engine_type: str, just_alias=False) -> str: + def get_select_string( + self, engine_type: str, with_alias: bool = True + ) -> str: """ In a query, what to output after the select statement - could be a column name, a sub query, a function etc. For a column it will be the column name. """ - pass + raise NotImplementedError() def as_alias(self, alias: str) -> Selectable: """ Allows column names to be changed in the result of a select. """ - self.alias = alias + self._alias = alias return self @@ -472,7 +496,7 @@ def __init__( secret=secret, ) - self.alias: t.Optional[str] = None + self._alias: t.Optional[str] = None def _validate_default( self, @@ -694,7 +718,7 @@ def as_alias(self, name: str) -> Column: """ column = copy.deepcopy(self) - column.alias = name + column._alias = name return column def get_default_value(self) -> t.Any: @@ -709,21 +733,30 @@ def get_default_value(self) -> t.Any: return default() if is_callable else default # type: ignore return None - def get_select_string(self, engine_type: str, just_alias=False) -> str: + def get_select_string( + self, engine_type: str, with_alias: bool = True + ) -> str: """ - How to refer to this column in a SQL query. + How to refer to this column in a SQL query, taking account of any joins + and aliases. """ - if self.alias is None: - return self._meta.get_full_name( - just_alias=just_alias, include_quotes=True - ) - original_name = self._meta.get_full_name( - just_alias=True, include_quotes=True - ) - return f"{original_name} AS {self.alias}" + if with_alias: + if self._alias: + original_name = self._meta.get_full_name( + with_alias=False, + ) + return f'{original_name} AS "{self._alias}"' + else: + return self._meta.get_full_name( + with_alias=True, + ) + + return self._meta.get_full_name(with_alias=False) def get_where_string(self, engine_type: str) -> str: - return self.get_select_string(engine_type=engine_type, just_alias=True) + return self.get_select_string( + engine_type=engine_type, with_alias=False + ) def get_sql_value(self, value: t.Any) -> t.Any: """ diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 84ee1db8e..2fc76a722 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -2227,21 +2227,19 @@ def arrow(self, key: str) -> JSONB: instance.json_operator = f"-> '{key}'" return instance - def get_select_string(self, engine_type: str, just_alias=False) -> str: - select_string = self._meta.get_full_name( - just_alias=just_alias, include_quotes=True - ) + def get_select_string( + self, engine_type: str, with_alias: bool = True + ) -> str: + select_string = self._meta.get_full_name(with_alias=False) + if self.json_operator is not None: - return ( - f"{select_string} {self.json_operator}" - if self.alias is None - else f"{select_string} {self.json_operator} AS {self.alias}" - ) + select_string += f" {self.json_operator}" - if self.alias is None: - return select_string - else: - return f"{select_string} AS {self.alias}" + if with_alias: + alias = self._alias or self._meta.get_default_alias() + select_string += f' AS "{alias}"' + + return select_string def eq(self, value) -> Where: """ @@ -2489,15 +2487,17 @@ def __getitem__(self, value: int) -> Array: else: raise ValueError("Only integers can be used for indexing.") - def get_select_string(self, engine_type: str, just_alias=False) -> str: - select_string = self._meta.get_full_name( - just_alias=just_alias, include_quotes=True - ) + def get_select_string(self, engine_type: str, with_alias=True) -> str: + select_string = self._meta.get_full_name(with_alias=False) if isinstance(self.index, int): - return f"{select_string}[{self.index}]" - else: - return select_string + select_string += f"[{self.index}]" + + if with_alias: + alias = self._alias or self._meta.get_default_alias() + select_string += f' AS "{alias}"' + + return select_string def any(self, value: t.Any) -> Where: """ diff --git a/piccolo/columns/m2m.py b/piccolo/columns/m2m.py index 57f403073..db0d61d77 100644 --- a/piccolo/columns/m2m.py +++ b/piccolo/columns/m2m.py @@ -55,7 +55,7 @@ def __init__( for column in columns ) - def get_select_string(self, engine_type: str, just_alias=False) -> str: + def get_select_string(self, engine_type: str, with_alias=True) -> str: m2m_table_name = self.m2m._meta.resolved_joining_table._meta.tablename m2m_relationship_name = self.m2m._meta.name diff --git a/piccolo/columns/readable.py b/piccolo/columns/readable.py index 651f5b1a5..630299f95 100644 --- a/piccolo/columns/readable.py +++ b/piccolo/columns/readable.py @@ -24,7 +24,7 @@ class Readable(Selectable): @property def _columns_string(self) -> str: return ", ".join( - i._meta.get_full_name(just_alias=True) for i in self.columns + i._meta.get_full_name(with_alias=False) for i in self.columns ) def _get_string(self, operator: str) -> str: @@ -41,7 +41,7 @@ def sqlite_string(self) -> str: def postgres_string(self) -> str: return self._get_string(operator="FORMAT") - def get_select_string(self, engine_type: str, just_alias=False) -> str: + def get_select_string(self, engine_type: str, with_alias=True) -> str: try: return getattr(self, f"{engine_type}_string") except AttributeError as e: diff --git a/piccolo/query/base.py b/piccolo/query/base.py index b1857df87..8e6bc1459 100644 --- a/piccolo/query/base.py +++ b/piccolo/query/base.py @@ -86,13 +86,10 @@ async def _process_results(self, results): # noqa: C901 json_column_names = [] for column in json_columns: - if column.alias is not None: - json_column_names.append(column.alias) + if column._alias is not None: + json_column_names.append(column._alias) elif column.json_operator is not None: - # If no alias is specified, then the default column name - # that Postgres gives when using the `->` operator is - # `?column?`. - json_column_names.append("?column?") + json_column_names.append(column._meta.name) elif len(column._meta.call_chain) > 0: json_column_names.append( column.get_select_string( diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 0c31e8373..45fa675f0 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -62,13 +62,13 @@ def __init__(self, column: Column, alias: str = "avg"): self.column = column else: raise ValueError("Column type must be numeric to run the query.") - self.alias = alias + self._alias = alias - def get_select_string(self, engine_type: str, just_alias=False) -> str: - column_name = self.column._meta.get_full_name( - just_alias=just_alias, include_quotes=True - ) - return f"AVG({column_name}) AS {self.alias}" + def get_select_string( + self, engine_type: str, with_alias: bool = True + ) -> str: + column_name = self.column._meta.get_full_name(with_alias=False) + return f'AVG({column_name}) AS "{self._alias}"' class Count(Selectable): @@ -100,16 +100,16 @@ def __init__( self, column: t.Optional[Column] = None, alias: str = "count" ): self.column = column - self.alias = alias + self._alias = alias - def get_select_string(self, engine_type: str, just_alias=False) -> str: + def get_select_string( + self, engine_type: str, with_alias: bool = True + ) -> str: if self.column is None: column_name = "*" else: - column_name = self.column._meta.get_full_name( - just_alias=just_alias, include_quotes=True - ) - return f"COUNT({column_name}) AS {self.alias}" + column_name = self.column._meta.get_full_name(with_alias=False) + return f'COUNT({column_name}) AS "{self._alias}"' class Max(Selectable): @@ -136,13 +136,13 @@ class Max(Selectable): def __init__(self, column: Column, alias: str = "max"): self.column = column - self.alias = alias + self._alias = alias - def get_select_string(self, engine_type: str, just_alias=False) -> str: - column_name = self.column._meta.get_full_name( - just_alias=just_alias, include_quotes=True - ) - return f"MAX({column_name}) AS {self.alias}" + def get_select_string( + self, engine_type: str, with_alias: bool = True + ) -> str: + column_name = self.column._meta.get_full_name(with_alias=False) + return f'MAX({column_name}) AS "{self._alias}"' class Min(Selectable): @@ -167,13 +167,13 @@ class Min(Selectable): def __init__(self, column: Column, alias: str = "min"): self.column = column - self.alias = alias + self._alias = alias - def get_select_string(self, engine_type: str, just_alias=False) -> str: - column_name = self.column._meta.get_full_name( - just_alias=just_alias, include_quotes=True - ) - return f"MIN({column_name}) AS {self.alias}" + def get_select_string( + self, engine_type: str, with_alias: bool = True + ) -> str: + column_name = self.column._meta.get_full_name(with_alias=False) + return f'MIN({column_name}) AS "{self._alias}"' class Sum(Selectable): @@ -203,13 +203,13 @@ def __init__(self, column: Column, alias: str = "sum"): self.column = column else: raise ValueError("Column type must be numeric to run the query.") - self.alias = alias + self._alias = alias - def get_select_string(self, engine_type: str, just_alias=False) -> str: - column_name = self.column._meta.get_full_name( - just_alias=just_alias, include_quotes=True - ) - return f"SUM({column_name}) AS {self.alias}" + def get_select_string( + self, engine_type: str, with_alias: bool = True + ) -> str: + column_name = self.column._meta.get_full_name(with_alias=False) + return f'SUM({column_name}) AS "{self._alias}"' class Select(Query): diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index 75318aee3..fe33092fc 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -66,7 +66,7 @@ class OrderBy: def querystring(self) -> QueryString: order = "ASC" if self.ascending else "DESC" columns_names = ", ".join( - i._meta.get_full_name(just_alias=True) for i in self.columns + i._meta.get_full_name(with_alias=False) for i in self.columns ) return QueryString(f" ORDER BY {columns_names} {order}") @@ -438,7 +438,7 @@ class GroupBy: @property def querystring(self) -> QueryString: columns_names = ", ".join( - i._meta.get_full_name(just_alias=True) for i in self.columns + i._meta.get_full_name(with_alias=False) for i in self.columns ) return QueryString(f" GROUP BY {columns_names}") diff --git a/piccolo/table.py b/piccolo/table.py index 3c5a5dd07..208708380 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -618,8 +618,7 @@ def to_dict(self, *columns: Column) -> t.Dict[str, t.Any]: break alias_names = { - column._meta.name: getattr(column, "alias", None) - for column in filtered_columns + column._meta.name: column._alias for column in filtered_columns } output = {} diff --git a/tests/columns/test_jsonb.py b/tests/columns/test_jsonb.py index fab2b7d10..fd4191792 100644 --- a/tests/columns/test_jsonb.py +++ b/tests/columns/test_jsonb.py @@ -1,6 +1,6 @@ from unittest import TestCase -from piccolo.columns.column_types import JSONB, Varchar +from piccolo.columns.column_types import JSONB, ForeignKey, Varchar from piccolo.table import Table from tests.base import postgres_only @@ -10,12 +10,19 @@ class RecordingStudio(Table): facilities = JSONB(null=True) +class Instrument(Table): + name = Varchar() + studio = ForeignKey(RecordingStudio) + + @postgres_only class TestJSONB(TestCase): def setUp(self): RecordingStudio.create_table().run_sync() + Instrument.create_table().run_sync() def tearDown(self): + Instrument.alter().drop_table().run_sync() RecordingStudio.alter().drop_table().run_sync() def test_json(self): @@ -88,6 +95,32 @@ def test_where(self): [{"name": "Abbey Road"}], ) + def test_as_alias_join(self): + """ + Make sure that ``as_alias`` performs correctly when used via a join. + """ + studio = ( + RecordingStudio.objects() + .create(name="Abbey Road", facilities={"mixing_desk": True}) + .run_sync() + ) + + Instrument.objects().create(name="Guitar", studio=studio).run_sync() + + response = ( + Instrument.select( + Instrument.name, + Instrument.studio.facilities.as_alias("studio_facilities"), + ) + .output(load_json=True) + .run_sync() + ) + + self.assertListEqual( + response, + [{"name": "Guitar", "studio_facilities": {"mixing_desk": True}}], + ) + def test_arrow(self): """ Test using the arrow function to retrieve a subset of the JSON. @@ -103,7 +136,7 @@ def test_arrow(self): .first() .run_sync() ) - self.assertEqual(row["?column?"], "true") + self.assertEqual(row["facilities"], "true") row = ( RecordingStudio.select( @@ -113,7 +146,7 @@ def test_arrow(self): .first() .run_sync() ) - self.assertEqual(row["?column?"], True) + self.assertEqual(row["facilities"], True) def test_arrow_as_alias(self): """ diff --git a/tests/conftest.py b/tests/conftest.py index 9ee3ff0c6..72c20d275 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,6 +18,7 @@ async def drop_tables(): "my_table", "recording_studio", "shirt", + "instrument", "mega_table", "small_table", ]: From c8e3f1d421c7303d5418cf5695db45e8c3d77547 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 21 Jun 2022 16:19:25 +0100 Subject: [PATCH 339/727] bumped version --- CHANGES.rst | 32 ++++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 59ee15494..3a88fd4dc 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,38 @@ Changes ======= +0.80.0 +------ + +There was a bug when doing joins with a ``JSONB`` column with ``as_alias``. + +.. code-block:: python + + class User(Table, tablename="my_user"): + name = Varchar(length=120) + config = JSONB(default={}) + + + class Subscriber(Table, tablename="subscriber"): + name = Varchar(length=120) + user = ForeignKey(references=User) + + + async def main(): + # This was failing: + await Subscriber.select( + Subscriber.name, + Subscriber.user.config.as_alias("config") + ) + +Thanks to @Anton-Karpenko for reporting this issue. + +Even though this is a bug fix, the minor version number has been bumped because +the fix resulted in some refactoring of Piccolo's internals, so is a fairly big +change. + +------------------------------------------------------------------------------- + 0.79.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 039899e69..0861a38d5 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.79.0" +__VERSION__ = "0.80.0" From cdd34f2026f6eff65821c6fde1e4cc61b02d57dd Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 28 Jun 2022 22:44:24 +0100 Subject: [PATCH 340/727] add `readable` to each `Table` in the playground (#555) --- piccolo/apps/playground/commands/run.py | 58 +++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/piccolo/apps/playground/commands/run.py b/piccolo/apps/playground/commands/run.py index 582a87633..6f5d3fb02 100644 --- a/piccolo/apps/playground/commands/run.py +++ b/piccolo/apps/playground/commands/run.py @@ -4,6 +4,7 @@ """ import datetime import sys +import typing as t import uuid from decimal import Decimal from enum import Enum @@ -19,6 +20,7 @@ Timestamp, Varchar, ) +from piccolo.columns.readable import Readable from piccolo.engine import PostgresEngine, SQLiteEngine from piccolo.engine.base import Engine from piccolo.table import Table @@ -27,17 +29,38 @@ class Manager(Table): name = Varchar(length=50) + @classmethod + def get_readable(cls) -> Readable: + return Readable( + template="%s", + columns=[cls.name], + ) + class Band(Table): name = Varchar(length=50) manager = ForeignKey(references=Manager, null=True) popularity = Integer() + @classmethod + def get_readable(cls) -> Readable: + return Readable( + template="%s", + columns=[cls.name], + ) + class Venue(Table): name = Varchar(length=100) capacity = Integer(default=0) + @classmethod + def get_readable(cls) -> Readable: + return Readable( + template="%s", + columns=[cls.name], + ) + class Concert(Table): band_1 = ForeignKey(Band) @@ -46,6 +69,17 @@ class Concert(Table): starts = Timestamp() duration = Interval() + @classmethod + def get_readable(cls) -> Readable: + return Readable( + template="%s and %s at %s", + columns=[ + cls.band_1.name, + cls.band_2.name, + cls.venue.name, + ], + ) + class Ticket(Table): class TicketType(Enum): @@ -57,16 +91,40 @@ class TicketType(Enum): price = Numeric(digits=(5, 2)) ticket_type = Varchar(choices=TicketType, default=TicketType.standing) + @classmethod + def get_readable(cls) -> Readable: + return Readable( + template="%s - %s", + columns=[ + t.cast(t.Type[Venue], cls.concert.venue).name, + cls.ticket_type, + ], + ) + class DiscountCode(Table): code = UUID() active = Boolean(default=True, null=True) + @classmethod + def get_readable(cls) -> Readable: + return Readable( + template="%s - %s", + columns=[cls.code, cls.active], + ) + class RecordingStudio(Table): name = Varchar(length=100) facilities = JSON(null=True) + @classmethod + def get_readable(cls) -> Readable: + return Readable( + template="%s", + columns=[cls.name], + ) + TABLES = (Manager, Band, Venue, Concert, Ticket, DiscountCode, RecordingStudio) From 7bd694d9e23462620f81e96f75113d8b03fb3130 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Sun, 3 Jul 2022 09:28:37 +0200 Subject: [PATCH 341/727] fix _get_related_readable typo (#557) --- piccolo/table.py | 2 +- .../instance/test_get_related_readable.py | 76 ++++++++++++++++++- 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/piccolo/table.py b/piccolo/table.py index 208708380..1c707b585 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -654,7 +654,7 @@ def _get_related_readable(cls, column: ForeignKey) -> Readable: for readable_column in readable.columns: output_column = column for fk in readable_column._meta.call_chain: - output_column = getattr(column, fk._meta.name) + output_column = getattr(output_column, fk._meta.name) output_column = getattr(output_column, readable_column._meta.name) output_columns.append(output_column) diff --git a/tests/table/instance/test_get_related_readable.py b/tests/table/instance/test_get_related_readable.py index 3c66f6ad4..1e182a679 100644 --- a/tests/table/instance/test_get_related_readable.py +++ b/tests/table/instance/test_get_related_readable.py @@ -1,7 +1,9 @@ import decimal from unittest import TestCase -from piccolo.table import create_db_tables_sync, drop_db_tables_sync +from piccolo.columns import ForeignKey, Varchar +from piccolo.columns.readable import Readable +from piccolo.table import Table, create_db_tables_sync, drop_db_tables_sync from tests.example_apps.music.tables import ( Band, Concert, @@ -10,7 +12,48 @@ Venue, ) -TABLES = [Band, Concert, Manager, Venue, Ticket] + +class ThingOne(Table): + name = Varchar(length=300, null=False) + + +class ThingTwo(Table): + name = Varchar(length=300, null=False) + thing_one = ForeignKey(references=ThingOne) + + +class ThingThree(Table): + name = Varchar(length=300, null=False) + thing_two = ForeignKey(references=ThingTwo) + + @classmethod + def get_readable(cls): + return Readable( + template="three name: %s - two name: %s - one name: %s", + columns=[ + cls.name, + cls.thing_two.name, + cls.thing_two.thing_one.name, + ], + ) + + +class ThingFour(Table): + name = Varchar(length=300, null=False) + thing_three = ForeignKey(references=ThingThree) + + +TABLES = [ + Band, + Concert, + Manager, + Venue, + Ticket, + ThingOne, + ThingTwo, + ThingThree, + ThingFour, +] class TestGetRelatedReadable(TestCase): @@ -44,6 +87,17 @@ def setUp(self): price=decimal.Decimal(50.0), concert=concert ).run_sync() + thing_one = ThingOne.insert(ThingOne(name="thing_one")).run_sync() + thing_two = ThingTwo.insert( + ThingTwo(name="thing_two", thing_one=thing_one[0]["id"]) + ).run_sync() + thing_three = ThingThree.insert( + ThingThree(name="thing_three", thing_two=thing_two[0]["id"]) + ).run_sync() + ThingFour.insert( + ThingFour(name="thing_four", thing_three=thing_three[0]["id"]) + ).run_sync() + def tearDown(self): drop_db_tables_sync(*TABLES) @@ -66,7 +120,8 @@ def test_get_related_readable(self): # Now try something much more complex. response = Ticket.select( - Ticket.id, Ticket._get_related_readable(Ticket.concert) + Ticket.id, + Ticket._get_related_readable(Ticket.concert), ).run_sync() self.assertEqual( response, @@ -80,3 +135,18 @@ def test_get_related_readable(self): } ], ) + + # A really complex references chain from Piccolo Admin issue #170 + response = ThingFour.select( + ThingFour._get_related_readable(ThingFour.thing_three) + ).run_sync() + self.assertEqual( + response, + [ + { + "thing_three_readable": ( + "three name: thing_three - two name: thing_two - one name: thing_one" # noqa: E501 + ) + } + ], + ) From d02331fc54df930f0ed83a9f6e24c20acee5b7d8 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 3 Jul 2022 09:26:14 +0100 Subject: [PATCH 342/727] bumped version --- CHANGES.rst | 10 ++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 3a88fd4dc..f4a7ec1a7 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,16 @@ Changes ======= +0.80.1 +------ + +Fixed a bug with Piccolo Admin and ``_get_related_readable``, which is used +to show a human friendly identifier for a row, rather than just the ID. + +Thanks to @ethagnawl and @sinisaos for their help with this. + +------------------------------------------------------------------------------- + 0.80.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 0861a38d5..a86c1d470 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.80.0" +__VERSION__ = "0.80.1" From a6e79c3774d9dbf818dc9615db19862129c45634 Mon Sep 17 00:00:00 2001 From: Patrick Forringer Date: Wed, 6 Jul 2022 15:51:50 -0500 Subject: [PATCH 343/727] Return querystring string cast when casting Combination (#558) Ran into this problem locally while trying to debug an And combination. --- piccolo/columns/combination.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piccolo/columns/combination.py b/piccolo/columns/combination.py index 6162aac72..a753957d3 100644 --- a/piccolo/columns/combination.py +++ b/piccolo/columns/combination.py @@ -41,7 +41,7 @@ def querystring(self) -> QueryString: ) def __str__(self): - self.querystring.__str__() + return self.querystring.__str__() class And(Combination): From cacf7c507f20d153bd40306bf250816e6edb0512 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 6 Jul 2022 21:57:02 +0100 Subject: [PATCH 344/727] bumped version --- CHANGES.rst | 9 +++++++++ piccolo/__init__.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index f4a7ec1a7..23320b9a1 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,15 @@ Changes ======= +0.80.2 +------ + +Fixed a bug with ``Combination.__str__``, which meant that when printing out a +query for debugging purposes it was wasn't showing correctly (courtesy +@destos). + +------------------------------------------------------------------------------- + 0.80.1 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index a86c1d470..7554ae4c4 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.80.1" +__VERSION__ = "0.80.2" From 55bde6ed1fa8decf0448437317ae001fe5d04d10 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 18 Jul 2022 18:28:59 +0100 Subject: [PATCH 345/727] Returning clause for update (#563) * add returning clause to update query * fix tests * don't run tests for old SQLite versions * added docs for `returning` clause --- docs/src/piccolo/query_clauses/index.rst | 1 + docs/src/piccolo/query_clauses/returning.rst | 24 +++++++++ docs/src/piccolo/query_types/update.rst | 6 +++ piccolo/query/methods/update.py | 51 ++++++++++++++++---- piccolo/query/mixins.py | 32 ++++++++++++ tests/base.py | 5 ++ tests/table/test_update.py | 47 +++++++++++++++++- 7 files changed, 155 insertions(+), 11 deletions(-) create mode 100644 docs/src/piccolo/query_clauses/returning.rst diff --git a/docs/src/piccolo/query_clauses/index.rst b/docs/src/piccolo/query_clauses/index.rst index 0fe8bc52f..4b5200ab0 100644 --- a/docs/src/piccolo/query_clauses/index.rst +++ b/docs/src/piccolo/query_clauses/index.rst @@ -20,3 +20,4 @@ by modifying the return values. ./where ./batch ./freeze + ./returning diff --git a/docs/src/piccolo/query_clauses/returning.rst b/docs/src/piccolo/query_clauses/returning.rst new file mode 100644 index 000000000..b1d8f6765 --- /dev/null +++ b/docs/src/piccolo/query_clauses/returning.rst @@ -0,0 +1,24 @@ +.. _returning: + +returning +========= + +You can use the ``returning`` clause with the following queries: + +* :ref:`Update` + +By default, an update query returns an empty list, but using the ``returning`` +clause you can retrieve values from the updated rows. + +.. code-block:: python + + >>> await Band.update({ + ... Band.name: 'Pythonistas Tribute Band' + ... }).where( + ... Band.name == 'Pythonistas' + ... ).returning(Band.id, Band.name) + [{'id': 1, 'name': 'Pythonistas Tribute Band'}] + +.. warning:: This works for all versions of Postgres, but only + `SQLite 3.35.0 `_ and above + support the returning clause. diff --git a/docs/src/piccolo/query_types/update.rst b/docs/src/piccolo/query_types/update.rst index 24a6892f3..80c25a62f 100644 --- a/docs/src/piccolo/query_types/update.rst +++ b/docs/src/piccolo/query_types/update.rst @@ -212,6 +212,12 @@ you prefer: Query clauses ------------- +returning +~~~~~~~~~ + +See :ref:`Returning`. + + where ~~~~~ diff --git a/piccolo/query/methods/update.py b/piccolo/query/methods/update.py index e9c9dbadb..6a09c7b9d 100644 --- a/piccolo/query/methods/update.py +++ b/piccolo/query/methods/update.py @@ -4,7 +4,11 @@ from piccolo.custom_types import Combinable from piccolo.query.base import Query -from piccolo.query.mixins import ValuesDelegate, WhereDelegate +from piccolo.query.mixins import ( + ReturningDelegate, + ValuesDelegate, + WhereDelegate, +) from piccolo.querystring import QueryString if t.TYPE_CHECKING: # pragma: no cover @@ -18,14 +22,23 @@ class UpdateError(Exception): class Update(Query): - __slots__ = ("force", "values_delegate", "where_delegate") + __slots__ = ( + "force", + "returning_delegate", + "values_delegate", + "where_delegate", + ) def __init__(self, table: t.Type[Table], force: bool = False, **kwargs): super().__init__(table, **kwargs) self.force = force + self.returning_delegate = ReturningDelegate() self.values_delegate = ValuesDelegate(table=table) self.where_delegate = WhereDelegate() + ########################################################################### + # Clauses + def values( self, values: t.Dict[t.Union[Column, str], t.Any] = None, **kwargs ) -> Update: @@ -39,7 +52,17 @@ def where(self, *where: Combinable) -> Update: self.where_delegate.where(*where) return self + def returning(self, *columns: Column) -> Update: + self.returning_delegate.returning(columns) + return self + + ########################################################################### + def _validate(self): + """ + Called at the start of :meth:`piccolo.query.base.Query.run` to make + sure the user has configured the query correctly before running it. + """ if len(self.values_delegate._values) == 0: raise ValueError("No values were specified to update.") @@ -57,6 +80,8 @@ def _validate(self): f"`{classname}.update`. Otherwise, add a where clause." ) + ########################################################################### + @property def default_querystrings(self) -> t.Sequence[QueryString]: columns_str = ", ".join( @@ -70,12 +95,18 @@ def default_querystrings(self) -> t.Sequence[QueryString]: query, *self.values_delegate.get_sql_values() ) - if not self.where_delegate._where: - return [querystring] + if self.where_delegate._where: + querystring = QueryString( + "{} WHERE {}", + querystring, + self.where_delegate._where.querystring, + ) - where_querystring = QueryString( - "{} WHERE {}", - querystring, - self.where_delegate._where.querystring, - ) - return [where_querystring] + if self.returning_delegate._returning: + querystring = QueryString( + "{}{}", + querystring, + self.returning_delegate._returning.querystring, + ) + + return [querystring] diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index fe33092fc..b872c52c2 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -75,6 +75,30 @@ def __str__(self): return self.querystring.__str__() +@dataclass +class Returning: + __slots__ = ("columns",) + + columns: t.Iterable[Column] + + @property + def querystring(self) -> QueryString: + column_names = [] + for column in self.columns: + column_names.append( + f'"{column._meta.db_column_name}" AS "{column._alias}"' + if column._alias + else f'"{column._meta.db_column_name}"' + ) + + columns_string = ", ".join(column_names) + + return QueryString(f" RETURNING {columns_string}") + + def __str__(self): + return self.querystring.__str__() + + @dataclass class Output: @@ -178,6 +202,14 @@ def distinct(self): self._distinct = True +@dataclass +class ReturningDelegate: + _returning: t.Optional[Returning] = None + + def returning(self, columns: t.Sequence[Column]): + self._returning = Returning(columns=columns) + + @dataclass class CountDelegate: diff --git a/tests/base.py b/tests/base.py index 5c0375cff..b6612a124 100644 --- a/tests/base.py +++ b/tests/base.py @@ -13,10 +13,15 @@ from piccolo.engine.postgres import PostgresEngine from piccolo.engine.sqlite import SQLiteEngine from piccolo.table import Table, create_table_class +from piccolo.utils.sync import run_sync ENGINE = engine_finder() +def engine_version_lt(version: float): + return ENGINE and run_sync(ENGINE.get_version()) < version + + def is_running_postgres(): return isinstance(ENGINE, PostgresEngine) diff --git a/tests/table/test_update.py b/tests/table/test_update.py index 60cb787af..8a70159f3 100644 --- a/tests/table/test_update.py +++ b/tests/table/test_update.py @@ -3,6 +3,8 @@ import typing as t from unittest import TestCase +import pytest + from piccolo.columns.base import Column from piccolo.columns.column_types import ( Date, @@ -15,7 +17,12 @@ ) from piccolo.querystring import QueryString from piccolo.table import Table -from tests.base import DBTestCase, sqlite_only +from tests.base import ( + DBTestCase, + engine_version_lt, + is_running_sqlite, + sqlite_only, +) from tests.example_apps.music.tables import Band @@ -105,6 +112,44 @@ def test_update_values_with_kwargs(self): self.check_response() + @pytest.mark.skipif( + is_running_sqlite() and engine_version_lt(3.35), + reason="SQLite version not supported", + ) + def test_update_returning(self): + """ + Make sure update works with the `returning` clause. + """ + self.insert_rows() + + response = ( + Band.update({Band.name: "Pythonistas 2"}) + .where(Band.name == "Pythonistas") + .returning(Band.name) + .run_sync() + ) + + self.assertEqual(response, [{"name": "Pythonistas 2"}]) + + @pytest.mark.skipif( + is_running_sqlite() and engine_version_lt(3.35), + reason="SQLite version not supported", + ) + def test_update_returning_alias(self): + """ + Make sure update works with the `returning` clause. + """ + self.insert_rows() + + response = ( + Band.update({Band.name: "Pythonistas 2"}) + .where(Band.name == "Pythonistas") + .returning(Band.name.as_alias("band name")) + .run_sync() + ) + + self.assertEqual(response, [{"band name": "Pythonistas 2"}]) + ############################################################################### # Test operators From d3a056c63e66cde770c445889e69edcd0af12163 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 18 Jul 2022 23:48:58 +0100 Subject: [PATCH 346/727] add support for `returning` to insert (#564) * add support for `returning` to insert * add table to querystring * pass in table * use `db_column_name` * wrap tablename in quotes * add `log_queries` to `SQLiteEngine` * more test fixes for unsupported SQLite versions * remove unused import * add docs for setting up SQLite * organise query clauses index page - separating into essential and advanced * update docs for returning clause - include `insert` queries --- docs/src/piccolo/getting_started/index.rst | 1 + .../piccolo/getting_started/setup_sqlite.rst | 28 +++++++ docs/src/piccolo/query_clauses/index.rst | 17 +++-- docs/src/piccolo/query_clauses/returning.rst | 16 +++- piccolo/engine/base.py | 4 + piccolo/engine/postgres.py | 3 + piccolo/engine/sqlite.py | 49 ++++++++----- piccolo/query/methods/insert.py | 73 +++++++++++-------- piccolo/query/mixins.py | 4 +- piccolo/querystring.py | 17 ++++- piccolo/table.py | 4 +- tests/engine/test_pool.py | 39 +++++----- tests/table/test_alter.py | 13 +++- tests/table/test_insert.py | 36 ++++++++- 14 files changed, 222 insertions(+), 82 deletions(-) create mode 100644 docs/src/piccolo/getting_started/setup_sqlite.rst diff --git a/docs/src/piccolo/getting_started/index.rst b/docs/src/piccolo/getting_started/index.rst index c53234319..e31ce6619 100644 --- a/docs/src/piccolo/getting_started/index.rst +++ b/docs/src/piccolo/getting_started/index.rst @@ -10,5 +10,6 @@ Getting Started ./installing_piccolo ./playground ./setup_postgres + ./setup_sqlite ./example_schema ./sync_and_async diff --git a/docs/src/piccolo/getting_started/setup_sqlite.rst b/docs/src/piccolo/getting_started/setup_sqlite.rst new file mode 100644 index 000000000..f6f47361d --- /dev/null +++ b/docs/src/piccolo/getting_started/setup_sqlite.rst @@ -0,0 +1,28 @@ +.. _set_up_sqlite: + +Setup SQLite +============ + +Installation +------------ + +The good news is SQLite is good to go out of the box with Python. + +Some Piccolo features are only available with newer SQLite versions. + +.. _check_sqlite_version: + +Check version +------------- + +To check which SQLite version you're using, simply open a Python terminal, and +do the following: + +.. code-block:: python + + >>> import sqlite3 + >>> sqlite3.sqlite_version + '3.39.0' + +The easiest way to upgrade your SQLite version is to install the latest version +of Python. diff --git a/docs/src/piccolo/query_clauses/index.rst b/docs/src/piccolo/query_clauses/index.rst index 4b5200ab0..19c4cfbae 100644 --- a/docs/src/piccolo/query_clauses/index.rst +++ b/docs/src/piccolo/query_clauses/index.rst @@ -6,18 +6,25 @@ Query Clauses Query clauses are used to modify a query by making it more specific, or by modifying the return values. + .. toctree:: :maxdepth: 1 + :caption: Essential ./first - ./distinct - ./group_by ./limit - ./offset ./order_by - ./output - ./callback ./where + +.. toctree:: + :maxdepth: 1 + :caption: Advanced + ./batch + ./callback + ./distinct ./freeze + ./group_by + ./offset + ./output ./returning diff --git a/docs/src/piccolo/query_clauses/returning.rst b/docs/src/piccolo/query_clauses/returning.rst index b1d8f6765..113554090 100644 --- a/docs/src/piccolo/query_clauses/returning.rst +++ b/docs/src/piccolo/query_clauses/returning.rst @@ -5,6 +5,7 @@ returning You can use the ``returning`` clause with the following queries: +* :ref:`Insert` * :ref:`Update` By default, an update query returns an empty list, but using the ``returning`` @@ -19,6 +20,19 @@ clause you can retrieve values from the updated rows. ... ).returning(Band.id, Band.name) [{'id': 1, 'name': 'Pythonistas Tribute Band'}] +Similarly, for an insert query - we can retrieve some of the values from the +inserted rows: + +.. code-block:: python + + >>> await Manager.insert( + ... Manager(name="Maz"), + ... Manager(name="Graydon") + ... ).returning(Manager.id, Manager.name) + + [{'id': 1, 'name': 'Maz'}, {'id': 1, 'name': 'Graydon'}] + .. warning:: This works for all versions of Postgres, but only `SQLite 3.35.0 `_ and above - support the returning clause. + support the returning clause. See the :ref:`docs ` on + how to check your SQLite version. diff --git a/piccolo/engine/base.py b/piccolo/engine/base.py index 4ae7b7201..2f98e9f42 100644 --- a/piccolo/engine/base.py +++ b/piccolo/engine/base.py @@ -41,6 +41,10 @@ def min_version_number(self) -> float: async def get_version(self) -> float: pass + @abstractmethod + def get_version_sync(self) -> float: + pass + @abstractmethod async def prep_database(self): pass diff --git a/piccolo/engine/postgres.py b/piccolo/engine/postgres.py index 9ca867d18..73ba8f127 100644 --- a/piccolo/engine/postgres.py +++ b/piccolo/engine/postgres.py @@ -321,6 +321,9 @@ async def get_version(self) -> float: version_string=version_string ) + def get_version_sync(self) -> float: + return run_sync(self.get_version()) + async def prep_database(self): for extension in self.extensions: try: diff --git a/piccolo/engine/sqlite.py b/piccolo/engine/sqlite.py index 317514513..9f9ba0e81 100644 --- a/piccolo/engine/sqlite.py +++ b/piccolo/engine/sqlite.py @@ -347,9 +347,13 @@ class SQLiteEngine(Engine): See the `SQLite docs `_ for more info. + :param log_queries: + If ``True``, all SQL and DDL statements are printed out before being + run. Useful for debugging. + """ # noqa: E501 - __slots__ = ("connection_kwargs", "transaction_connection") + __slots__ = ("connection_kwargs", "transaction_connection", "log_queries") engine_type = "sqlite" min_version_number = 3.25 @@ -357,6 +361,7 @@ class SQLiteEngine(Engine): def __init__( self, path: str = "piccolo.sqlite", + log_queries: bool = False, detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES, isolation_level=None, **connection_kwargs, @@ -368,6 +373,7 @@ def __init__( "isolation_level": isolation_level, } ) + self.log_queries = log_queries self.connection_kwargs = connection_kwargs self.transaction_connection = contextvars.ContextVar( @@ -385,9 +391,9 @@ def path(self, value: str): self.connection_kwargs["database"] = value async def get_version(self) -> float: - """ - Warn if the version of SQLite isn't supported. - """ + return self.get_version_sync() + + def get_version_sync(self) -> float: major, minor, _ = sqlite3.sqlite_version_info return float(f"{major}.{minor}") @@ -443,13 +449,12 @@ async def _get_inserted_pk(self, cursor, table: t.Type[Table]) -> t.Any: different types. Need to query by `lastrowid` to get `pk`s in SQLite prior to 3.35.0. """ - # TODO: Add RETURNING clause for sqlite > 3.35.0 await cursor.execute( - f"SELECT {table._meta.primary_key._meta.name} FROM " + f"SELECT {table._meta.primary_key._meta.db_column_name} FROM " f"{table._meta.tablename} WHERE ROWID = {cursor.lastrowid}" ) response = await cursor.fetchone() - return response[table._meta.primary_key._meta.name] + return response[table._meta.primary_key._meta.db_column_name] async def _run_in_new_connection( self, @@ -467,13 +472,15 @@ async def _run_in_new_connection( async with connection.execute(query, args) as cursor: await connection.commit() - if query_type != "insert": + if query_type == "insert" and self.get_version_sync() < 3.35: + # We can't use the RETURNING clause on older versions + # of SQLite. + assert table is not None + pk = await self._get_inserted_pk(cursor, table) + return [{table._meta.primary_key._meta.db_column_name: pk}] + else: return await cursor.fetchall() - assert table is not None - pk = await self._get_inserted_pk(cursor, table) - return [{table._meta.primary_key._meta.name: pk}] - async def _run_in_existing_connection( self, connection, @@ -493,13 +500,15 @@ async def _run_in_existing_connection( async with connection.execute(query, args) as cursor: response = await cursor.fetchall() - if query_type != "insert": + if query_type == "insert" and self.get_version_sync() < 3.35: + # We can't use the RETURNING clause on older versions + # of SQLite. + assert table is not None + pk = await self._get_inserted_pk(cursor, table) + return [{table._meta.primary_key._meta.db_column_name: pk}] + else: return response - assert table is not None - pk = await self._get_inserted_pk(cursor, table) - return [{table._meta.primary_key._meta.name: pk}] - async def run_querystring( self, querystring: QueryString, in_pool: bool = False ): @@ -507,6 +516,9 @@ async def run_querystring( Connection pools aren't currently supported - the argument is there for consistency with other engines. """ + if self.log_queries: + print(querystring) + query, query_args = querystring.compile_string( engine_type=self.engine_type ) @@ -534,6 +546,9 @@ async def run_ddl(self, ddl: str, in_pool: bool = False): Connection pools aren't currently supported - the argument is there for consistency with other engines. """ + if self.log_queries: + print(ddl) + # If running inside a transaction: connection = self.transaction_connection.get() if connection: diff --git a/piccolo/query/methods/insert.py b/piccolo/query/methods/insert.py index 0a9825216..31b9cd134 100644 --- a/piccolo/query/methods/insert.py +++ b/piccolo/query/methods/insert.py @@ -3,25 +3,36 @@ import typing as t from piccolo.query.base import Query -from piccolo.query.mixins import AddDelegate +from piccolo.query.mixins import AddDelegate, ReturningDelegate from piccolo.querystring import QueryString if t.TYPE_CHECKING: # pragma: no cover + from piccolo.columns.base import Column from piccolo.table import Table class Insert(Query): - __slots__ = ("add_delegate",) + __slots__ = ("add_delegate", "returning_delegate") def __init__(self, table: t.Type[Table], *instances: Table, **kwargs): super().__init__(table, **kwargs) self.add_delegate = AddDelegate() + self.returning_delegate = ReturningDelegate() self.add(*instances) + ########################################################################### + # Clauses + def add(self, *instances: Table) -> Insert: self.add_delegate.add(*instances, table_class=self.table) return self + def returning(self, *columns: Column) -> Insert: + self.returning_delegate.returning(columns) + return self + + ########################################################################### + def _raw_response_callback(self, results): """ Assign the ids of the created rows to the model instances. @@ -31,42 +42,42 @@ def _raw_response_callback(self, results): setattr( table_instance, self.table._meta.primary_key._meta.name, - row[self.table._meta.primary_key._meta.name], + row.get( + self.table._meta.primary_key._meta.db_column_name, None + ), ) table_instance._exists_in_db = True @property - def sqlite_querystrings(self) -> t.Sequence[QueryString]: - base = f"INSERT INTO {self.table._meta.tablename}" + def default_querystrings(self) -> t.Sequence[QueryString]: + base = f'INSERT INTO "{self.table._meta.tablename}"' columns = ",".join( f'"{i._meta.db_column_name}"' for i in self.table._meta.columns ) values = ",".join("{}" for _ in self.add_delegate._add) query = f"{base} ({columns}) VALUES {values}" - return [ - QueryString( - query, - *[i.querystring for i in self.add_delegate._add], - query_type="insert", - table=self.table, - ) - ] - - @property - def postgres_querystrings(self) -> t.Sequence[QueryString]: - base = f"INSERT INTO {self.table._meta.tablename}" - columns = ",".join( - f'"{i._meta.db_column_name}"' for i in self.table._meta.columns - ) - values = ",".join("{}" for _ in self.add_delegate._add) - primary_key_name = self.table._meta.primary_key._meta.name - query = ( - f"{base} ({columns}) VALUES {values} RETURNING {primary_key_name}" + querystring = QueryString( + query, + *[i.querystring for i in self.add_delegate._add], + query_type="insert", + table=self.table, ) - return [ - QueryString( - query, - *[i.querystring for i in self.add_delegate._add], - query_type="insert", - ) - ] + + engine_type = self.engine_type + + if engine_type == "postgres" or ( + engine_type == "sqlite" + and self.table._meta.db.get_version_sync() >= 3.35 + ): + if self.returning_delegate._returning: + return [ + QueryString( + "{}{}", + querystring, + self.returning_delegate._returning.querystring, + query_type="insert", + table=self.table, + ) + ] + + return [querystring] diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index b872c52c2..e65ac8630 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -79,7 +79,7 @@ def __str__(self): class Returning: __slots__ = ("columns",) - columns: t.Iterable[Column] + columns: t.List[Column] @property def querystring(self) -> QueryString: @@ -207,7 +207,7 @@ class ReturningDelegate: _returning: t.Optional[Returning] = None def returning(self, columns: t.Sequence[Column]): - self._returning = Returning(columns=columns) + self._returning = Returning(columns=list(columns)) @dataclass diff --git a/piccolo/querystring.py b/piccolo/querystring.py index eeccbf5ad..6aa8fd50c 100644 --- a/piccolo/querystring.py +++ b/piccolo/querystring.py @@ -66,10 +66,21 @@ def __init__( table: t.Optional[t.Type[Table]] = None, ) -> None: """ - Example template: "WHERE {} = {}" + :param template: + The SQL query, with curly brackets as placeholders for any values:: + + "WHERE {} = {}" + + :param args: + The values to insert (one value is needed for each set of curly + braces in the template). + :param query_type: + The query type is sometimes used by the engine to modify how the + query is run. For example, INSERT queries on old SQLite versions. + :param table: + Sometimes the ``piccolo.engine.base.Engine`` needs access to the + table that the query is being run on. - The query type is sometimes used by the engine to modify how the query - is run. """ self.template = template self.args = args diff --git a/piccolo/table.py b/piccolo/table.py index 1c707b585..30e702be7 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -386,7 +386,7 @@ def save( cls = self.__class__ if not self._exists_in_db: - return cls.insert().add(self) + return cls.insert(self).returning(cls._meta.primary_key) # Pre-existing row - update if columns is None: @@ -846,7 +846,7 @@ def insert(cls, *rows: "Table") -> Insert: ) """ - query = Insert(table=cls) + query = Insert(table=cls).returning(cls._meta.primary_key) if rows: query.add(*rows) return query diff --git a/tests/engine/test_pool.py b/tests/engine/test_pool.py index 21c23870a..1c3e80b07 100644 --- a/tests/engine/test_pool.py +++ b/tests/engine/test_pool.py @@ -2,7 +2,7 @@ import os import tempfile from unittest import TestCase -from unittest.mock import MagicMock, call, patch +from unittest.mock import call, patch from piccolo.engine.postgres import PostgresEngine from piccolo.engine.sqlite import SQLiteEngine @@ -89,28 +89,29 @@ class TestConnectionPoolWarning(TestCase): async def _create_pool(self): sqlite_file = os.path.join(tempfile.gettempdir(), "engine.sqlite") engine = SQLiteEngine(path=sqlite_file) - await engine.start_connection_pool() - await engine.close_connection_pool() - @patch("piccolo.engine.base.colored_warning") - def test_warnings(self, colored_warning: MagicMock): + with patch("piccolo.engine.base.colored_warning") as colored_warning: + await engine.start_connection_pool() + await engine.close_connection_pool() + + self.assertEqual( + colored_warning.call_args_list, + [ + call( + "Connection pooling is not supported for sqlite.", + stacklevel=3, + ), + call( + "Connection pooling is not supported for sqlite.", + stacklevel=3, + ), + ], + ) + + def test_warnings(self): """ Make sure that when trying to start and close a connection pool with SQLite, a warning is printed out, as connection pools aren't currently supported. """ asyncio.run(self._create_pool()) - - self.assertEqual( - colored_warning.call_args_list, - [ - call( - "Connection pooling is not supported for sqlite.", - stacklevel=3, - ), - call( - "Connection pooling is not supported for sqlite.", - stacklevel=3, - ), - ], - ) diff --git a/tests/table/test_alter.py b/tests/table/test_alter.py index 5b37c8c90..cfc637258 100644 --- a/tests/table/test_alter.py +++ b/tests/table/test_alter.py @@ -3,14 +3,25 @@ import typing as t from unittest import TestCase +import pytest + from piccolo.columns import BigInt, Integer, Numeric, Varchar from piccolo.columns.base import Column from piccolo.columns.column_types import ForeignKey, Text from piccolo.table import Table -from tests.base import DBTestCase, postgres_only +from tests.base import ( + DBTestCase, + engine_version_lt, + is_running_sqlite, + postgres_only, +) from tests.example_apps.music.tables import Band, Manager +@pytest.mark.skipif( + is_running_sqlite() and engine_version_lt(3.25), + reason="SQLite version not supported", +) class TestRenameColumn(DBTestCase): def _test_rename( self, diff --git a/tests/table/test_insert.py b/tests/table/test_insert.py index 7822811e6..474497f25 100644 --- a/tests/table/test_insert.py +++ b/tests/table/test_insert.py @@ -1,4 +1,6 @@ -from tests.base import DBTestCase +import pytest + +from tests.base import DBTestCase, engine_version_lt, is_running_sqlite from tests.example_apps.music.tables import Band, Manager @@ -42,3 +44,35 @@ def test_insert_curly_braces(self): names = [i["name"] for i in response] self.assertIn("{}", names) + + @pytest.mark.skipif( + is_running_sqlite() and engine_version_lt(3.35), + reason="SQLite version not supported", + ) + def test_insert_returning(self): + """ + Make sure update works with the `returning` clause. + """ + response = ( + Manager.insert(Manager(name="Maz")) + .returning(Manager.name) + .run_sync() + ) + + self.assertListEqual(response, [{"name": "Maz"}]) + + @pytest.mark.skipif( + is_running_sqlite() and engine_version_lt(3.35), + reason="SQLite version not supported", + ) + def test_insert_returning_alias(self): + """ + Make sure update works with the `returning` clause. + """ + response = ( + Manager.insert(Manager(name="Maz")) + .returning(Manager.name.as_alias("manager_name")) + .run_sync() + ) + + self.assertListEqual(response, [{"manager_name": "Maz"}]) From 7b0bb681c859560a6af7219c5476809eab4f660c Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 19 Jul 2022 11:28:07 +0100 Subject: [PATCH 347/727] bumped version --- CHANGES.rst | 23 +++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 23320b9a1..612b32d02 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,29 @@ Changes ======= +0.81.0 +------ + +Added the ``returning`` clause to ``insert`` and ``update`` queries. + +This can be used to retrieve data from the inserted / modified rows. + +Here's an example, where we update the unpopular bands, and retrieve their +names, in a single query: + +.. code-block:: python + + >>> await Band.update({ + ... Band.popularity: Band.popularity + 5 + ... }).where( + ... Band.popularity < 10 + ... ).returning( + ... Band.name + ... ) + [{'name': 'Bad sound band'}, {'name': 'Tone deaf band'}] + +------------------------------------------------------------------------------- + 0.80.2 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 7554ae4c4..5ed674066 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.80.2" +__VERSION__ = "0.81.0" From 05da48d2bccb193d27d216f317d29e5f5ff77a77 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 19 Jul 2022 11:35:41 +0100 Subject: [PATCH 348/727] `Table.__init__` `_data` argument (#565) * add `_data` argument to `Table.__init__` * `exists_in_db` -> `_exists_in_db` * add tests * add docs * improve `Table.__init__` docstring --- docs/src/piccolo/query_types/objects.rst | 12 +++++- piccolo/query/base.py | 4 +- piccolo/table.py | 50 ++++++++++++++++++------ piccolo/testing/model_builder.py | 2 +- tests/table/test_constructor.py | 28 +++++++++++++ 5 files changed, 80 insertions(+), 16 deletions(-) create mode 100644 tests/table/test_constructor.py diff --git a/docs/src/piccolo/query_types/objects.rst b/docs/src/piccolo/query_types/objects.rst index b9a0a15b2..456e6717c 100644 --- a/docs/src/piccolo/query_types/objects.rst +++ b/docs/src/piccolo/query_types/objects.rst @@ -52,12 +52,22 @@ columns. Creating objects ---------------- +You can pass the column values using kwargs: + .. code-block:: python >>> band = Band(name="C-Sharps", popularity=100) >>> await band.save() -This can also be done like this: +Alternatively, you can pass in a dictionary, which is friendlier to static +analysis tools like Mypy (it can easily detect typos in the column names): + +.. code-block:: python + + >>> band = Band({Band.name: "C-Sharps", Band.popularity: 100}) + >>> await band.save() + +We also have this shortcut which combines the above into a single line: .. code-block:: python diff --git a/piccolo/query/base.py b/piccolo/query/base.py index 8e6bc1459..dd3ae9fac 100644 --- a/piccolo/query/base.py +++ b/piccolo/query/base.py @@ -126,14 +126,14 @@ async def _process_results(self, results): # noqa: C901 ] else: raw = [ - self.table(**columns, exists_in_db=True) + self.table(**columns, _exists_in_db=True) for columns in raw ] elif raw is not None: if output._output.nested: raw = make_nested_object(raw, self.table) else: - raw = self.table(**raw, exists_in_db=True) + raw = self.table(**raw, _exists_in_db=True) elif type(raw) is list: if output._output.as_list: if len(raw) == 0: diff --git a/piccolo/table.py b/piccolo/table.py index 30e702be7..a792db105 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -300,29 +300,55 @@ def __init_subclass__( def __init__( self, - ignore_missing: bool = False, - exists_in_db: bool = False, + _data: t.Dict[Column, t.Any] = None, + _ignore_missing: bool = False, + _exists_in_db: bool = False, **kwargs, ): """ - Assigns any default column values to the class. + The constructor can be used to assign column values. - :param ignore_missing: + .. note:: + The ``_data``, ``_ignore_missing``, and ``_exists_in_db`` + arguments are prefixed with an underscore to help prevent a clash + with a column name which might be passed in via kwargs. + + :param _data: + There's two ways of passing in the data for each column. Firstly, + you can use kwargs:: + + Band(name="Pythonistas") + + Secondly, you can pass in a dictionary which maps column classes to + values:: + + Band({Band.name: 'Pythonistas'}) + + The advantage of this second approach is it's more strongly typed, + and linters such as flake8 or MyPy will more easily detect typos. + + :param _ignore_missing: If ``False`` a ``ValueError`` will be raised if any column values haven't been provided. - :param exists_in_db: + :param _exists_in_db: Used internally to track whether this row exists in the database. """ - self._exists_in_db = exists_in_db + _data = _data or {} + + self._exists_in_db = _exists_in_db for column in self._meta.columns: - value = kwargs.pop(column._meta.name, ...) + value = _data.get(column, ...) - if value is ...: - value = kwargs.pop( - t.cast(str, column._meta.db_column_name), ... - ) + if kwargs: + if value is ...: + value = kwargs.pop(column._meta.name, ...) + + if value is ...: + value = kwargs.pop( + t.cast(str, column._meta.db_column_name), ... + ) if value is ...: value = column.get_default_value() @@ -333,7 +359,7 @@ def __init__( if ( (value is None) and (not column._meta.null) - and not ignore_missing + and not _ignore_missing ): raise ValueError(f"{column._meta.name} wasn't provided") diff --git a/piccolo/testing/model_builder.py b/piccolo/testing/model_builder.py index dd5d1f833..a4d6d9410 100644 --- a/piccolo/testing/model_builder.py +++ b/piccolo/testing/model_builder.py @@ -103,7 +103,7 @@ async def _build( minimal: bool = False, persist: bool = True, ) -> Table: - model = table_class(ignore_missing=True) + model = table_class(_ignore_missing=True) defaults = {} if not defaults else defaults for column, value in defaults.items(): diff --git a/tests/table/test_constructor.py b/tests/table/test_constructor.py new file mode 100644 index 000000000..78263e423 --- /dev/null +++ b/tests/table/test_constructor.py @@ -0,0 +1,28 @@ +from unittest import TestCase + +from tests.example_apps.music.tables import Band + + +class TestConstructor(TestCase): + def test_data_parameter(self): + """ + Make sure the _data parameter works. + """ + band = Band({Band.name: "Pythonistas"}) + self.assertEqual(band.name, "Pythonistas") + + def test_kwargs(self): + """ + Make sure kwargs works. + """ + band = Band(name="Pythonistas") + self.assertEqual(band.name, "Pythonistas") + + def test_mix(self): + """ + Make sure the _data paramter and kwargs works together (it's unlikely + people will do this, but just in case). + """ + band = Band({Band.name: "Pythonistas"}, popularity=1000) + self.assertEqual(band.name, "Pythonistas") + self.assertEqual(band.popularity, 1000) From 0c44bb22be451e27318ddc2594e7e5a775494f80 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 19 Jul 2022 11:42:05 +0100 Subject: [PATCH 349/727] bumped version --- CHANGES.rst | 24 ++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 612b32d02..8ebc47a48 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,27 @@ Changes ======= +0.82.0 +------ + +Traditionally, when instantiating a ``Table``, you passed in column values +using kwargs: + +.. code-block:: python + + >>> await Manager(name='Guido').save() + +You can now pass in a dictionary instead, which makes it easier for static +typing analysis tools like Mypy to detect typos. + +.. code-block:: python + + >>> await Manager({Manager.name: 'Guido'}).save() + +See `PR 565 `_ for more info. + +------------------------------------------------------------------------------- + 0.81.0 ------ @@ -22,6 +43,9 @@ names, in a single query: ... ) [{'name': 'Bad sound band'}, {'name': 'Tone deaf band'}] +See `PR 564 `_ and +`PR 563 `_ for more info. + ------------------------------------------------------------------------------- 0.80.2 diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 5ed674066..5cdb6dfa7 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.81.0" +__VERSION__ = "0.82.0" From 90388f6e149b6a47682a912a1bc6e8113b4b9b6e Mon Sep 17 00:00:00 2001 From: sinisaos Date: Thu, 11 Aug 2022 23:07:17 +0200 Subject: [PATCH 350/727] add node argument to batch method (#579) * add node argument to batch method * add tests * update test dependencies * exclude sqlite from batch tests * another try to pass GH tests * mention creating the database in contributing docs * remove some of the coverage ignores This is a good idea, but I worry that it might lead us to miss some issues. * break up `batch` docs into sections * expose `node` argument in `Select.batch` and `Objects.batch` * different testing approach I changed this because we were modifying the global DB variable - this was we're just using a local DB variable. * try `AsyncMock` * fix something I removed by accident from `exclude_lines` * add back 'pass' to 'exclude_lines' * make `batch` signature consistent for SQLite and Postgres * make tests run under SQLite Co-authored-by: Daniel Townsend --- docs/src/piccolo/contributing/index.rst | 2 +- docs/src/piccolo/query_clauses/batch.rst | 22 +++++++++++++ piccolo/apps/fixtures/commands/shared.py | 2 +- piccolo/engine/base.py | 7 +++- piccolo/engine/postgres.py | 19 +++++++++-- piccolo/engine/sqlite.py | 13 +++++++- piccolo/query/methods/objects.py | 7 +++- piccolo/query/methods/select.py | 7 +++- piccolo/querystring.py | 2 +- piccolo/table.py | 2 +- piccolo/utils/objects.py | 2 +- piccolo/utils/sql_values.py | 2 +- pyproject.toml | 18 ++++++++-- tests/table/test_batch.py | 42 ++++++++++++++++++++++-- 14 files changed, 131 insertions(+), 16 deletions(-) diff --git a/docs/src/piccolo/contributing/index.rst b/docs/src/piccolo/contributing/index.rst index a9d20f0de..7c7f0af28 100644 --- a/docs/src/piccolo/contributing/index.rst +++ b/docs/src/piccolo/contributing/index.rst @@ -17,7 +17,7 @@ Get the tests running * Install default dependencies: ``pip install -r requirements/requirements.txt`` * Install development dependencies: ``pip install -r requirements/dev-requirements.txt`` * Install test dependencies: ``pip install -r requirements/test-requirements.txt`` -* Setup Postgres +* Setup Postgres, and make sure a database called ``piccolo`` exists (see ``tests/postgres_conf.py``). * Run the automated code linting/formatting tools: ``./scripts/lint.sh`` * Run the test suite with Postgres: ``./scripts/test-postgres.sh`` * Run the test suite with Sqlite: ``./scripts/test-sqlite.sh`` diff --git a/docs/src/piccolo/query_clauses/batch.rst b/docs/src/piccolo/query_clauses/batch.rst index 1fd0829de..139a360b9 100644 --- a/docs/src/piccolo/query_clauses/batch.rst +++ b/docs/src/piccolo/query_clauses/batch.rst @@ -8,6 +8,9 @@ You can use ``batch`` clauses with the following queries: * :ref:`Objects` * :ref:`Select` +Example +------- + By default, a query will return as many rows as you ask it for. The problem is when you have a table containing millions of rows - you might not want to load them all into memory at once. To get around this, you can batch the @@ -20,6 +23,25 @@ responses. async for _batch in batch: print(_batch) +Node +---- + +If you're using ``extra_nodes`` with :class:`PostgresEngine `, +you can specify which node to query: + +.. code-block:: python + + # Returns 100 rows at a time from read_replica_db + async with await Manager.select().batch( + batch_size=100, + node="read_replica_db", + ) as batch: + async for _batch in batch: + print(_batch) + +Synchronous version +------------------- + There's currently no synchronous version. However, it's easy enough to achieve: .. code-block:: python diff --git a/piccolo/apps/fixtures/commands/shared.py b/piccolo/apps/fixtures/commands/shared.py index 9290605a7..99915a85a 100644 --- a/piccolo/apps/fixtures/commands/shared.py +++ b/piccolo/apps/fixtures/commands/shared.py @@ -8,7 +8,7 @@ from piccolo.conf.apps import Finder from piccolo.utils.pydantic import create_pydantic_model -if t.TYPE_CHECKING: +if t.TYPE_CHECKING: # pragma: no cover from piccolo.table import Table diff --git a/piccolo/engine/base.py b/piccolo/engine/base.py index 2f98e9f42..43b317ab4 100644 --- a/piccolo/engine/base.py +++ b/piccolo/engine/base.py @@ -50,7 +50,12 @@ async def prep_database(self): pass @abstractmethod - async def batch(self, query: Query, batch_size: int = 100) -> Batch: + async def batch( + self, + query: Query, + batch_size: int = 100, + node: t.Optional[str] = None, + ) -> Batch: pass @abstractmethod diff --git a/piccolo/engine/postgres.py b/piccolo/engine/postgres.py index 73ba8f127..fb76714f0 100644 --- a/piccolo/engine/postgres.py +++ b/piccolo/engine/postgres.py @@ -390,8 +390,23 @@ async def get_new_connection(self) -> Connection: ########################################################################### - async def batch(self, query: Query, batch_size: int = 100) -> AsyncBatch: - connection = await self.get_new_connection() + async def batch( + self, + query: Query, + batch_size: int = 100, + node: t.Optional[str] = None, + ) -> AsyncBatch: + """ + :param query: + The database query to run. + :param batch_size: + How many rows to fetch on each iteration. + :param node: + Which node to run the query on (see ``extra_nodes``). If not + specified, it runs on the main Postgres node. + """ + engine: t.Any = self.extra_nodes.get(node) if node else self + connection = await engine.get_new_connection() return AsyncBatch( connection=connection, query=query, batch_size=batch_size ) diff --git a/piccolo/engine/sqlite.py b/piccolo/engine/sqlite.py index 9f9ba0e81..f7fae03e2 100644 --- a/piccolo/engine/sqlite.py +++ b/piccolo/engine/sqlite.py @@ -427,7 +427,18 @@ def create_db(self, migrate=False): ########################################################################### - async def batch(self, query: Query, batch_size: int = 100) -> AsyncBatch: + async def batch( + self, query: Query, batch_size: int = 100, node: t.Optional[str] = None + ) -> AsyncBatch: + """ + :param query: + The database query to run. + :param batch_size: + How many rows to fetch on each iteration. + :param node: + This is ignored currently, as SQLite runs off a single node. The + value is here so the API is consistent with Postgres. + """ connection = await self.get_connection() return AsyncBatch( connection=connection, query=query, batch_size=batch_size diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index a6c082691..6750e6b30 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -207,10 +207,15 @@ def where(self, *where: Combinable) -> Objects: return self async def batch( - self, batch_size: t.Optional[int] = None, **kwargs + self, + batch_size: t.Optional[int] = None, + node: t.Optional[str] = None, + **kwargs, ) -> Batch: if batch_size: kwargs.update(batch_size=batch_size) + if node: + kwargs.update(node=node) return await self.table._meta.db.batch(self, **kwargs) async def response_handler(self, response): diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 45fa675f0..623230ada 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -479,10 +479,15 @@ def where(self, *where: Combinable) -> Select: return self async def batch( - self, batch_size: t.Optional[int] = None, **kwargs + self, + batch_size: t.Optional[int] = None, + node: t.Optional[str] = None, + **kwargs, ) -> Batch: if batch_size: kwargs.update(batch_size=batch_size) + if node: + kwargs.update(node=node) return await self.table._meta.db.batch(self, **kwargs) ########################################################################### diff --git a/piccolo/querystring.py b/piccolo/querystring.py index 6aa8fd50c..1bc8ee26b 100644 --- a/piccolo/querystring.py +++ b/piccolo/querystring.py @@ -6,7 +6,7 @@ from importlib.util import find_spec from string import Formatter -if t.TYPE_CHECKING: +if t.TYPE_CHECKING: # pragma: no cover from piccolo.table import Table from uuid import UUID diff --git a/piccolo/table.py b/piccolo/table.py index a792db105..604f630fd 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -50,7 +50,7 @@ from piccolo.utils.sync import run_sync from piccolo.utils.warnings import colored_warning -if t.TYPE_CHECKING: +if t.TYPE_CHECKING: # pragma: no cover from piccolo.columns import Selectable PROTECTED_TABLENAMES = ("user",) diff --git a/piccolo/utils/objects.py b/piccolo/utils/objects.py index 1faf5b41b..24e6e3e32 100644 --- a/piccolo/utils/objects.py +++ b/piccolo/utils/objects.py @@ -4,7 +4,7 @@ from piccolo.columns.column_types import ForeignKey -if t.TYPE_CHECKING: +if t.TYPE_CHECKING: # pragma: no cover from piccolo.table import Table diff --git a/piccolo/utils/sql_values.py b/piccolo/utils/sql_values.py index 6c13da99c..9dde4a5a0 100644 --- a/piccolo/utils/sql_values.py +++ b/piccolo/utils/sql_values.py @@ -5,7 +5,7 @@ from piccolo.utils.encoding import dump_json -if t.TYPE_CHECKING: +if t.TYPE_CHECKING: # pragma: no cover from piccolo.columns import Column diff --git a/pyproject.toml b/pyproject.toml index 6b4f9a201..ad8732598 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,8 @@ module = [ "dateutil", "IPython", "jinja2", - "orjson" + "orjson", + "aiosqlite" ] ignore_missing_imports = true @@ -25,4 +26,17 @@ markers = [ ] [tool.coverage.run] -omit = ["*.jinja", "**/piccolo_migrations/*", "**/piccolo_app.py"] +omit = [ + "*.jinja", + "**/piccolo_migrations/*", + "**/piccolo_app.py" +] + +[tool.coverage.report] +# Note, we have to re-specify "pragma: no cover" +# https://coverage.readthedocs.io/en/6.3.3/excluding.html#advanced-exclusion +exclude_lines = [ + "raise NotImplementedError", + "pragma: no cover", + "pass" +] diff --git a/tests/table/test_batch.py b/tests/table/test_batch.py index a0a5b34ee..69f7a027c 100644 --- a/tests/table/test_batch.py +++ b/tests/table/test_batch.py @@ -1,7 +1,13 @@ import asyncio import math - -from tests.base import DBTestCase +from unittest import TestCase + +from piccolo.columns import Varchar +from piccolo.engine.finder import engine_finder +from piccolo.engine.postgres import AsyncBatch, PostgresEngine +from piccolo.table import Table +from piccolo.utils.sync import run_sync +from tests.base import AsyncMock, DBTestCase, postgres_only from tests.example_apps.music.tables import Manager @@ -87,3 +93,35 @@ def test_batch(self): self.assertEqual(_row_count, row_count) self.assertEqual(iterations, _iterations) + + +@postgres_only +class TestBatchNodeArg(TestCase): + def test_batch_extra_node(self): + """ + Make sure the batch methods can accept a node argument. + """ + + # Get the test database credentials: + test_engine = engine_finder() + + EXTRA_NODE = AsyncMock(spec=PostgresEngine(config=test_engine.config)) + + DB = PostgresEngine( + config=test_engine.config, + extra_nodes={"read_1": EXTRA_NODE}, + ) + + class Manager(Table, db=DB): + name = Varchar() + + # Testing `select` + response = run_sync(Manager.select().batch(node="read_1")) + self.assertIsInstance(response, AsyncBatch) + self.assertTrue(EXTRA_NODE.get_new_connection.called) + EXTRA_NODE.reset_mock() + + # Testing `objects` + response = run_sync(Manager.objects().batch(node="read_1")) + self.assertIsInstance(response, AsyncBatch) + self.assertTrue(EXTRA_NODE.get_new_connection.called) From fbaa412bf75c92f793f489d0ecb185241d63d420 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 11 Aug 2022 22:15:11 +0100 Subject: [PATCH 351/727] bumped version --- CHANGES.rst | 20 ++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 8ebc47a48..fd2ee9e47 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,26 @@ Changes ======= +0.83.0 +------ + +We added support for Postgres read-slaves a few releases ago, but the ``batch`` +clause didn't support it until now. Thanks to @guruvignesh01 for reporting +this issue, and @sinisaos for help implementing it. + +.. code-block:: python + + # Returns 100 rows at a time from read_replica_db + async with await Manager.select().batch( + batch_size=100, + node="read_replica_db", + ) as batch: + async for _batch in batch: + print(_batch) + + +------------------------------------------------------------------------------- + 0.82.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 5cdb6dfa7..b5ddf8235 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.82.0" +__VERSION__ = "0.83.0" From 3d04c3bcac1672e1f6b89e5b98d5fe04511f322b Mon Sep 17 00:00:00 2001 From: ali sayyah Date: Fri, 12 Aug 2022 16:29:36 +0430 Subject: [PATCH 352/727] migration DDL preview functionality (#584) * Initial groundwork for preview functionality * query will be printed in the preview mode * lint * flake8 * wrong return value * simple test * better query output for preview * added preview option to the CLI command * added preview option tests * enhanced the tests * more tests for MigrationManager with preview mode * updated docs with the preview mode * lint * added a test for multi query scenario --- docs/src/piccolo/migrations/running.rst | 2 + .../apps/migrations/auto/migration_manager.py | 224 ++++++++++-------- piccolo/apps/migrations/commands/backwards.py | 25 +- piccolo/apps/migrations/commands/forwards.py | 45 +++- .../migrations/auto/test_migration_manager.py | 87 +++++-- .../commands/test_forwards_backwards.py | 39 ++- 6 files changed, 286 insertions(+), 136 deletions(-) diff --git a/docs/src/piccolo/migrations/running.rst b/docs/src/piccolo/migrations/running.rst index 29069c8b7..e7b38289a 100644 --- a/docs/src/piccolo/migrations/running.rst +++ b/docs/src/piccolo/migrations/running.rst @@ -3,6 +3,8 @@ Running migrations .. hint:: To see all available options for these commands, use the ``--help`` flag, for example ``piccolo migrations forwards --help``. +.. hint:: To see the SQL queries of a migration without actually running them , use the ``--preview`` + flag, for example: ``piccolo migrations forwards my_app --preview`` or ``piccolo migrations backwards 2018-09-04T19:44:09 --preview``. Forwards -------- diff --git a/piccolo/apps/migrations/auto/migration_manager.py b/piccolo/apps/migrations/auto/migration_manager.py index a03407efb..b9c585f2b 100644 --- a/piccolo/apps/migrations/auto/migration_manager.py +++ b/piccolo/apps/migrations/auto/migration_manager.py @@ -15,6 +15,8 @@ from piccolo.columns import Column, column_types from piccolo.columns.column_types import Serial from piccolo.engine import engine_finder +from piccolo.query import Query +from piccolo.query.base import DDL from piccolo.table import Table, create_table_class, sort_table_classes from piccolo.utils.warnings import colored_warning @@ -129,6 +131,7 @@ class MigrationManager: migration_id: str = "" app_name: str = "" description: str = "" + preview: bool = False add_tables: t.List[DiffableTable] = field(default_factory=list) drop_tables: t.List[DiffableTable] = field(default_factory=list) rename_tables: t.List[RenameTable] = field(default_factory=list) @@ -310,7 +313,7 @@ def add_raw_backwards(self, raw: t.Union[t.Callable, t.Coroutine]): ########################################################################### - async def get_table_from_snaphot( + async def get_table_from_snapshot( self, table_class_name: str, app_name: t.Optional[str], @@ -344,6 +347,23 @@ async def get_table_from_snaphot( ########################################################################### + @staticmethod + async def _print_query(query: t.Union[DDL, Query]): + if isinstance(query, DDL): + print("\n", ";".join(query.ddl) + ";") + else: + print(str(query)) + + async def _run_query(self, query: t.Union[DDL, Query]): + """ + If MigrationManager is not in the preview mode, + executes the queries. else, prints the query. + """ + if self.preview: + await self._print_query(query) + else: + await query.run() + async def _run_alter_columns(self, backwards=False): for table_class_name in self.alter_columns.table_class_names: alter_columns = self.alter_columns.for_table_class_name( @@ -415,9 +435,9 @@ async def _run_alter_columns(self, backwards=False): # something which can be cast to the new type, # it will also fail. Drop the default value for # now - the proper default is set later on. - await _Table.alter().drop_default( - old_column - ).run() + await self._run_query( + _Table.alter().drop_default(old_column) + ) using_expression = "{}::{}".format( alter_column.db_column_name, @@ -435,25 +455,31 @@ async def _run_alter_columns(self, backwards=False): "vice versa. This must be done manually." ) else: - await _Table.alter().set_column_type( - old_column=old_column, - new_column=new_column, - using_expression=using_expression, - ).run() + await self._run_query( + _Table.alter().set_column_type( + old_column=old_column, + new_column=new_column, + using_expression=using_expression, + ) + ) ############################################################### null = params.get("null") if null is not None: - await _Table.alter().set_null( - column=alter_column.db_column_name, boolean=null - ).run() + await self._run_query( + _Table.alter().set_null( + column=alter_column.db_column_name, boolean=null + ) + ) length = params.get("length") if length is not None: - await _Table.alter().set_length( - column=alter_column.db_column_name, length=length - ).run() + await self._run_query( + _Table.alter().set_length( + column=alter_column.db_column_name, length=length + ) + ) unique = params.get("unique") if unique is not None: @@ -463,9 +489,11 @@ async def _run_alter_columns(self, backwards=False): column._meta._table = _Table column._meta._name = alter_column.column_name column._meta.db_column_name = alter_column.db_column_name - await _Table.alter().set_unique( - column=column, boolean=unique - ).run() + await self._run_query( + _Table.alter().set_unique( + column=column, boolean=unique + ) + ) index = params.get("index") index_method = params.get("index_method") @@ -480,10 +508,14 @@ async def _run_alter_columns(self, backwards=False): column._meta.db_column_name = ( alter_column.db_column_name ) - await _Table.drop_index([column]).run() - await _Table.create_index( - [column], method=index_method, if_not_exists=True - ).run() + await self._run_query(_Table.drop_index([column])) + await self._run_query( + _Table.create_index( + [column], + method=index_method, + if_not_exists=True, + ) + ) else: # If the index value has changed, then we are either # dropping, or creating an index. @@ -496,11 +528,13 @@ async def _run_alter_columns(self, backwards=False): kwargs = ( {"method": index_method} if index_method else {} ) - await _Table.create_index( - [column], if_not_exists=True, **kwargs - ).run() + await self._run_query( + _Table.create_index( + [column], if_not_exists=True, **kwargs + ) + ) else: - await _Table.drop_index([column]).run() + await self._run_query(_Table.drop_index([column])) # None is a valid value, so retrieve ellipsis if not found. default = params.get("default", ...) @@ -511,39 +545,45 @@ async def _run_alter_columns(self, backwards=False): column._meta.db_column_name = alter_column.db_column_name if default is None: - await _Table.alter().drop_default(column=column).run() + await self._run_query( + _Table.alter().drop_default(column=column) + ) else: column.default = default - await _Table.alter().set_default( - column=column, value=column.get_default_value() - ).run() + await self._run_query( + _Table.alter().set_default( + column=column, value=column.get_default_value() + ) + ) # None is a valid value, so retrieve ellipsis if not found. digits = params.get("digits", ...) if digits is not ...: - await _Table.alter().set_digits( - column=alter_column.db_column_name, - digits=digits, - ).run() + await self._run_query( + _Table.alter().set_digits( + column=alter_column.db_column_name, + digits=digits, + ) + ) async def _run_drop_tables(self, backwards=False): for diffable_table in self.drop_tables: if backwards: - _Table = await self.get_table_from_snaphot( + _Table = await self.get_table_from_snapshot( table_class_name=diffable_table.class_name, app_name=self.app_name, offset=-1, ) - await _Table.create_table().run() + await self._run_query(_Table.create_table()) else: - await ( - diffable_table.to_table_class().alter().drop_table().run() + await self._run_query( + diffable_table.to_table_class().alter().drop_table() ) async def _run_drop_columns(self, backwards=False): if backwards: for drop_column in self.drop_columns.drop_columns: - _Table = await self.get_table_from_snaphot( + _Table = await self.get_table_from_snapshot( table_class_name=drop_column.table_class_name, app_name=self.app_name, offset=-1, @@ -551,9 +591,11 @@ async def _run_drop_columns(self, backwards=False): column_to_restore = _Table._meta.get_column_by_name( drop_column.column_name ) - await _Table.alter().add_column( - name=drop_column.column_name, column=column_to_restore - ).run() + await self._run_query( + _Table.alter().add_column( + name=drop_column.column_name, column=column_to_restore + ) + ) else: for table_class_name in self.drop_columns.table_class_names: columns = self.drop_columns.for_table_class_name( @@ -569,9 +611,9 @@ async def _run_drop_columns(self, backwards=False): ) for column in columns: - await _Table.alter().drop_column( - column=column.column_name - ).run() + await self._run_query( + _Table.alter().drop_column(column=column.column_name) + ) async def _run_rename_tables(self, backwards=False): for rename_table in self.rename_tables: @@ -595,7 +637,9 @@ async def _run_rename_tables(self, backwards=False): class_name=class_name, class_kwargs={"tablename": tablename} ) - await _Table.alter().rename_table(new_name=new_tablename).run() + await self._run_query( + _Table.alter().rename_table(new_name=new_tablename) + ) async def _run_rename_columns(self, backwards=False): for table_class_name in self.rename_columns.table_class_names: @@ -623,10 +667,12 @@ async def _run_rename_columns(self, backwards=False): else rename_column.new_db_column_name ) - await _Table.alter().rename_column( - column=column, - new_name=new_name, - ).run() + await self._run_query( + _Table.alter().rename_column( + column=column, + new_name=new_name, + ) + ) async def _run_add_tables(self, backwards=False): table_classes: t.List[t.Type[Table]] = [] @@ -649,10 +695,10 @@ async def _run_add_tables(self, backwards=False): if backwards: for _Table in reversed(sorted_table_classes): - await _Table.alter().drop_table(cascade=True).run() + await self._run_query(_Table.alter().drop_table(cascade=True)) else: for _Table in sorted_table_classes: - await _Table.create_table().run() + await self._run_query(_Table.create_table()) async def _run_add_columns(self, backwards=False): """ @@ -672,7 +718,9 @@ async def _run_add_columns(self, backwards=False): class_kwargs={"tablename": add_column.tablename}, ) - await _Table.alter().drop_column(add_column.column).run() + await self._run_query( + _Table.alter().drop_column(add_column.column) + ) else: for table_class_name in self.add_columns.table_class_names: if table_class_name in [i.class_name for i in self.add_tables]: @@ -699,38 +747,21 @@ async def _run_add_columns(self, backwards=False): column = _Table._meta.get_column_by_name( add_column.column._meta.name ) - await _Table.alter().add_column( - name=column._meta.name, column=column - ).run() + await self._run_query( + _Table.alter().add_column( + name=column._meta.name, column=column + ) + ) if add_column.column._meta.index: - await _Table.create_index([add_column.column]).run() - - async def run(self): - print(f" - {self.migration_id} [forwards]... ", end="") - - engine = engine_finder() - - if not engine: - raise Exception("Can't find engine") - - async with engine.transaction(): - - for raw in self.raw: - if inspect.iscoroutinefunction(raw): - await raw() - else: - raw() - - await self._run_add_tables() - await self._run_rename_tables() - await self._run_add_columns() - await self._run_drop_columns() - await self._run_drop_tables() - await self._run_rename_columns() - await self._run_alter_columns() + await self._run_query( + _Table.create_index([add_column.column]) + ) - async def run_backwards(self): - print(f" - {self.migration_id} [backwards]... ", end="") + async def run(self, backwards=False): + direction = "backwards" if backwards else "forwards" + if self.preview: + direction = "preview " + direction + print(f" - {self.migration_id} [{direction}]... ", end="") engine = engine_finder() @@ -739,16 +770,17 @@ async def run_backwards(self): async with engine.transaction(): - for raw in self.raw_backwards: - if inspect.iscoroutinefunction(raw): - await raw() - else: - raw() - - await self._run_add_columns(backwards=True) - await self._run_add_tables(backwards=True) - await self._run_drop_tables(backwards=True) - await self._run_rename_tables(backwards=True) - await self._run_drop_columns(backwards=True) - await self._run_rename_columns(backwards=True) - await self._run_alter_columns(backwards=True) + if not self.preview: + for raw in self.raw: + if inspect.iscoroutinefunction(raw): + await raw() + else: + raw() + + await self._run_add_tables(backwards=backwards) + await self._run_rename_tables(backwards=backwards) + await self._run_add_columns(backwards=backwards) + await self._run_drop_columns(backwards=backwards) + await self._run_drop_tables(backwards=backwards) + await self._run_rename_columns(backwards=backwards) + await self._run_alter_columns(backwards=backwards) diff --git a/piccolo/apps/migrations/commands/backwards.py b/piccolo/apps/migrations/commands/backwards.py index 107cb9635..f3dc6fd33 100644 --- a/piccolo/apps/migrations/commands/backwards.py +++ b/piccolo/apps/migrations/commands/backwards.py @@ -18,11 +18,13 @@ def __init__( migration_id: str, auto_agree: bool = False, clean: bool = False, + preview: bool = False, ): self.migration_id = migration_id self.app_name = app_name self.auto_agree = auto_agree self.clean = clean + self.preview = preview super().__init__() async def run(self) -> MigrationResult: @@ -91,14 +93,16 @@ async def run(self) -> MigrationResult: response = await migration_module.forwards() if isinstance(response, MigrationManager): - await response.run_backwards() + if self.preview: + response.preview = True + await response.run(backwards=True) + if not self.preview: + await Migration.delete().where( + Migration.name == migration_id + ).run() - await Migration.delete().where( - Migration.name == migration_id - ).run() - - if self.clean and migration_module.__file__: - os.unlink(migration_module.__file__) + if self.clean and migration_module.__file__: + os.unlink(migration_module.__file__) print("ok! ✔️") return MigrationResult(success=True) @@ -114,6 +118,7 @@ async def run_backwards( migration_id: str = "1", auto_agree: bool = False, clean: bool = False, + preview: bool = False, ) -> MigrationResult: if app_name == "all": sorted_app_names = BaseMigrationManager().get_sorted_app_names() @@ -140,6 +145,7 @@ async def run_backwards( app_name=_app_name, migration_id="all", auto_agree=auto_agree, + preview=preview, ) await manager.run() return MigrationResult(success=True) @@ -149,6 +155,7 @@ async def run_backwards( migration_id=migration_id, auto_agree=auto_agree, clean=clean, + preview=preview, ) return await manager.run() @@ -158,6 +165,7 @@ async def backwards( migration_id: str = "1", auto_agree: bool = False, clean: bool = False, + preview: bool = False, ): """ Undo migrations up to a specific migration. @@ -174,6 +182,8 @@ async def backwards( :param clean: If true, the migration files which have been run backwards are deleted from the disk after completing. + :param preview: + If true, don't actually run the migration, just print the SQL queries. """ response = await run_backwards( @@ -181,6 +191,7 @@ async def backwards( migration_id=migration_id, auto_agree=auto_agree, clean=clean, + preview=preview, ) if not response.success: diff --git a/piccolo/apps/migrations/commands/forwards.py b/piccolo/apps/migrations/commands/forwards.py index 19ee6366e..b284bd43a 100644 --- a/piccolo/apps/migrations/commands/forwards.py +++ b/piccolo/apps/migrations/commands/forwards.py @@ -14,11 +14,16 @@ class ForwardsMigrationManager(BaseMigrationManager): def __init__( - self, app_name: str, migration_id: str = "all", fake: bool = False + self, + app_name: str, + migration_id: str = "all", + fake: bool = False, + preview: bool = False, ): self.app_name = app_name self.migration_id = migration_id self.fake = fake + self.preview = preview super().__init__() async def run_migrations(self, app_config: AppConfig) -> MigrationResult: @@ -71,13 +76,15 @@ async def run_migrations(self, app_config: AppConfig) -> MigrationResult: response = await migration_module.forwards() if isinstance(response, MigrationManager): + if self.preview: + response.preview = True await response.run() print("ok! ✔️") - - await Migration.insert().add( - Migration(name=_id, app_name=app_config.app_name) - ).run() + if not self.preview: + await Migration.insert().add( + Migration(name=_id, app_name=app_config.app_name) + ).run() return MigrationResult(success=True, message="migration succeeded") @@ -90,7 +97,10 @@ async def run(self) -> MigrationResult: async def run_forwards( - app_name: str, migration_id: str = "all", fake: bool = False + app_name: str, + migration_id: str = "all", + fake: bool = False, + preview: bool = False, ) -> MigrationResult: """ Run the migrations. This function can be used to programatically run @@ -102,7 +112,10 @@ async def run_forwards( print(f"\n{_app_name.upper():^64}") print("-" * 64) manager = ForwardsMigrationManager( - app_name=_app_name, migration_id="all", fake=fake + app_name=_app_name, + migration_id="all", + fake=fake, + preview=preview, ) response = await manager.run() if not response.success: @@ -112,13 +125,19 @@ async def run_forwards( else: manager = ForwardsMigrationManager( - app_name=app_name, migration_id=migration_id, fake=fake + app_name=app_name, + migration_id=migration_id, + fake=fake, + preview=preview, ) return await manager.run() async def forwards( - app_name: str, migration_id: str = "all", fake: bool = False + app_name: str, + migration_id: str = "all", + fake: bool = False, + preview: bool = False, ): """ Runs any migrations which haven't been run yet. @@ -133,9 +152,15 @@ async def forwards( :param fake: If set, will record the migrations as being run without actually running them. + :param preview: + If true, don't actually run the migration, just print the SQL queries + """ response = await run_forwards( - app_name=app_name, migration_id=migration_id, fake=fake + app_name=app_name, + migration_id=migration_id, + fake=fake, + preview=preview, ) if not response.success: diff --git a/tests/apps/migrations/auto/test_migration_manager.py b/tests/apps/migrations/auto/test_migration_manager.py index 2f1055271..286241e0c 100644 --- a/tests/apps/migrations/auto/test_migration_manager.py +++ b/tests/apps/migrations/auto/test_migration_manager.py @@ -1,4 +1,5 @@ import asyncio +from io import StringIO from unittest import TestCase from unittest.mock import MagicMock, patch @@ -95,7 +96,19 @@ def test_rename_column(self): self.assertTrue("name" not in response[0].keys()) # Reverse - asyncio.run(manager.run_backwards()) + asyncio.run(manager.run(backwards=True)) + response = self.run_sync("SELECT * FROM band;") + self.assertTrue("title" not in response[0].keys()) + self.assertTrue("name" in response[0].keys()) + + # Preview + manager.preview = True + with patch("sys.stdout", new=StringIO()) as fake_out: + asyncio.run(manager.run()) + self.assertEqual( + fake_out.getvalue(), + """ - [preview forwards]... \n ALTER TABLE band RENAME COLUMN "name" TO "title";\n""", # noqa: E501 + ) response = self.run_sync("SELECT * FROM band;") self.assertTrue("title" not in response[0].keys()) self.assertTrue("name" in response[0].keys()) @@ -120,7 +133,7 @@ def run(): # Reverse with self.assertRaises(HasRun): - asyncio.run(manager.run_backwards()) + asyncio.run(manager.run(backwards=True)) def test_raw_coroutine(self): """ @@ -142,7 +155,7 @@ async def run(): # Reverse with self.assertRaises(HasRun): - asyncio.run(manager.run_backwards()) + asyncio.run(manager.run(backwards=True)) @postgres_only @patch.object(BaseMigrationManager, "get_app_config") @@ -172,10 +185,20 @@ def test_add_table(self, get_app_config: MagicMock): get_app_config.return_value = AppConfig( app_name="music", migrations_folder_path="" ) - asyncio.run(manager.run_backwards()) + asyncio.run(manager.run(backwards=True)) self.assertEqual(self.table_exists("musician"), False) self.run_sync("DROP TABLE IF EXISTS musician;") + # Preview + manager.preview = True + with patch("sys.stdout", new=StringIO()) as fake_out: + asyncio.run(manager.run()) + self.assertEqual( + fake_out.getvalue(), + """ - [preview forwards]... \n CREATE TABLE musician ("id" SERIAL PRIMARY KEY NOT NULL, "name" VARCHAR(255) NOT NULL DEFAULT '');\n""", # noqa: E501 + ) + self.assertEqual(self.table_exists("musician"), False) + @postgres_only def test_add_column(self): """ @@ -210,7 +233,18 @@ def test_add_column(self): ) # Reverse - asyncio.run(manager.run_backwards()) + asyncio.run(manager.run(backwards=True)) + response = self.run_sync("SELECT * FROM manager;") + self.assertEqual(response, [{"id": 1, "name": "Dave"}]) + + # Preview + manager.preview = True + with patch("sys.stdout", new=StringIO()) as fake_out: + asyncio.run(manager.run()) + self.assertEqual( + fake_out.getvalue(), + """ - [preview forwards]... \n ALTER TABLE manager ADD COLUMN "email" VARCHAR(100) UNIQUE DEFAULT '';\n""", # noqa: E501 + ) response = self.run_sync("SELECT * FROM manager;") self.assertEqual(response, [{"id": 1, "name": "Dave"}]) @@ -242,7 +276,20 @@ def test_add_column_with_index(self): self.assertTrue(index_name in Manager.indexes().run_sync()) # Reverse - asyncio.run(manager.run_backwards()) + asyncio.run(manager.run(backwards=True)) + self.assertTrue(index_name not in Manager.indexes().run_sync()) + + # Preview + manager.preview = True + with patch("sys.stdout", new=StringIO()) as fake_out: + asyncio.run(manager.run()) + self.assertEqual( + fake_out.getvalue(), + ( + """ - [preview forwards]... \n ALTER TABLE manager ADD COLUMN "email" VARCHAR(100) UNIQUE DEFAULT '';\n""" # noqa: E501 + """\n CREATE INDEX manager_email ON manager USING btree ("email");\n""" # noqa: E501 + ), + ) self.assertTrue(index_name not in Manager.indexes().run_sync()) @postgres_only @@ -285,7 +332,7 @@ def test_add_foreign_key_self_column(self): ) # Reverse - asyncio.run(manager.run_backwards()) + asyncio.run(manager.run(backwards=True)) response = self.run_sync("SELECT * FROM manager;") self.assertEqual( response, @@ -360,7 +407,7 @@ def test_drop_column( get_migration_managers.return_value = [manager_1] app_config = AppConfig(app_name="music", migrations_folder_path="") get_app_config.return_value = app_config - asyncio.run(manager_2.run_backwards()) + asyncio.run(manager_2.run(backwards=True)) response = self.run_sync("SELECT * FROM musician;") self.assertEqual(response, [{"id": 1, "name": ""}]) @@ -386,7 +433,7 @@ def test_rename_table(self): self.assertEqual(response, [{"id": 1, "name": "Dave"}]) # Reverse - asyncio.run(manager.run_backwards()) + asyncio.run(manager.run(backwards=True)) response = self.run_sync("SELECT * FROM manager;") self.assertEqual(response, [{"id": 1, "name": "Dave"}]) @@ -414,7 +461,7 @@ def test_alter_column_unique(self): ) # Reverse - asyncio.run(manager.run_backwards()) + asyncio.run(manager.run(backwards=True)) self.run_sync( "INSERT INTO manager VALUES (default, 'Dave'), (default, 'Dave');" ) @@ -444,7 +491,7 @@ def test_alter_column_set_null(self): ) # Reverse - asyncio.run(manager.run_backwards()) + asyncio.run(manager.run(backwards=True)) self.assertFalse( self.get_postgres_is_nullable( tablename="manager", column_name="name" @@ -490,7 +537,7 @@ def test_alter_column_digits(self): [{"numeric_precision": 6, "numeric_scale": 2}], ) - asyncio.run(manager.run_backwards()) + asyncio.run(manager.run(backwards=True)) self.assertEqual( self._get_column_precision_and_scale(), [{"numeric_precision": 5, "numeric_scale": 2}], @@ -517,7 +564,7 @@ def test_alter_column_set_default(self): [{"column_default": "'Unknown'::character varying"}], ) - asyncio.run(manager.run_backwards()) + asyncio.run(manager.run(backwards=True)) self.assertEqual( self._get_column_default(), [{"column_default": "''::character varying"}], @@ -567,19 +614,19 @@ def test_alter_column_drop_default(self): ) # Run them all backwards - asyncio.run(manager_3.run_backwards()) + asyncio.run(manager_3.run(backwards=True)) self.assertEqual( self._get_column_default(), [{"column_default": None}], ) - asyncio.run(manager_2.run_backwards()) + asyncio.run(manager_2.run(backwards=True)) self.assertEqual( self._get_column_default(), [{"column_default": "'Mr Manager'::character varying"}], ) - asyncio.run(manager_1.run_backwards()) + asyncio.run(manager_1.run(backwards=True)) self.assertEqual( self._get_column_default(), [{"column_default": None}], @@ -605,7 +652,7 @@ def test_alter_column_add_index(self): Manager._get_index_name(["name"]) in Manager.indexes().run_sync() ) - asyncio.run(manager.run_backwards()) + asyncio.run(manager.run(backwards=True)) self.assertTrue( Manager._get_index_name(["name"]) not in Manager.indexes().run_sync() @@ -634,7 +681,7 @@ def test_alter_column_set_type(self): ) self.assertEqual(column_type_str, "TEXT") - asyncio.run(manager.run_backwards()) + asyncio.run(manager.run(backwards=True)) column_type_str = self.get_postgres_column_type( tablename="manager", column_name="name" ) @@ -665,7 +712,7 @@ def test_alter_column_set_length(self): 500, ) - asyncio.run(manager.run_backwards()) + asyncio.run(manager.run(backwards=True)) self.assertEqual( self.get_postgres_varchar_length( tablename="manager", column_name="name" @@ -702,7 +749,7 @@ def test_drop_table( get_migration_managers.return_value = [manager_1] app_config = AppConfig(app_name="music", migrations_folder_path="") get_app_config.return_value = app_config - asyncio.run(manager_2.run_backwards()) + asyncio.run(manager_2.run(backwards=True)) get_migration_managers.assert_called_with( app_config=app_config, max_migration_id="2", offset=-1 diff --git a/tests/apps/migrations/commands/test_forwards_backwards.py b/tests/apps/migrations/commands/test_forwards_backwards.py index 3bfe7ebb6..427fd1901 100644 --- a/tests/apps/migrations/commands/test_forwards_backwards.py +++ b/tests/apps/migrations/commands/test_forwards_backwards.py @@ -24,7 +24,6 @@ if t.TYPE_CHECKING: # pragma: no cover from piccolo.table import Table - TABLE_CLASSES: t.List[t.Type[Table]] = [ Manager, Band, @@ -53,6 +52,7 @@ def test_forwards_backwards_all_migrations(self): # Check the tables exist for table_class in TABLE_CLASSES: self.assertTrue(table_class.table_exists().run_sync()) + self.assertNotEqual(Migration.count().run_sync(), 0) run_sync( backwards( @@ -63,20 +63,34 @@ def test_forwards_backwards_all_migrations(self): # Check the tables don't exist for table_class in TABLE_CLASSES: self.assertTrue(not table_class.table_exists().run_sync()) + self.assertEqual(Migration.count().run_sync(), 0) + # Preview + run_sync( + forwards(app_name=app_name, migration_id="all", preview=True) + ) + for table_class in TABLE_CLASSES: + self.assertTrue(not table_class.table_exists().run_sync()) + self.assertEqual(Migration.count().run_sync(), 0) def test_forwards_backwards_single_migration(self): """ Test running a single migrations forwards, then backwards. """ + table_classes = [Band, Manager] + for migration_id in ["1", "2020-12-17T18:44:30"]: run_sync(forwards(app_name="music", migration_id=migration_id)) - table_classes = [Band, Manager] - # Check the tables exist for table_class in table_classes: self.assertTrue(table_class.table_exists().run_sync()) + self.assertTrue( + Migration.exists() + .where(Migration.name == "2020-12-17T18:44:30") + .run_sync() + ) + run_sync( backwards( app_name="music", @@ -88,6 +102,25 @@ def test_forwards_backwards_single_migration(self): # Check the tables don't exist for table_class in table_classes: self.assertTrue(not table_class.table_exists().run_sync()) + self.assertFalse( + Migration.exists() + .where(Migration.name == "2020-12-17T18:44:30") + .run_sync() + ) + + # Preview + run_sync( + forwards( + app_name="music", migration_id=migration_id, preview=True + ) + ) + for table_class in table_classes: + self.assertTrue(not table_class.table_exists().run_sync()) + self.assertFalse( + Migration.exists() + .where(Migration.name == "2020-12-17T18:44:30") + .run_sync() + ) @patch("piccolo.apps.migrations.commands.forwards.print") def test_forwards_unknown_migration(self, print_: MagicMock): From 8c1e8dc7919f88673c77fd131ed9bb6a7b62062c Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 12 Aug 2022 13:11:24 +0100 Subject: [PATCH 353/727] bumped version --- CHANGES.rst | 13 +++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index fd2ee9e47..ce04cad7c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,19 @@ Changes ======= +0.84.0 +------ + +You can now preview the DDL statements which will be run by Piccolo migrations. + +.. code-block:: bash + + piccolo migrations forwards my_app --preview + +Thanks to @AliSayyah for adding this feature. + +------------------------------------------------------------------------------- + 0.83.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index b5ddf8235..2a12b5aea 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.83.0" +__VERSION__ = "0.84.0" From d7e1396ea048596940fc54812e42599422dbca32 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 12 Aug 2022 13:51:44 +0100 Subject: [PATCH 354/727] fix `get_table_from_snapshot` typos (#587) --- piccolo/apps/migrations/auto/migration_manager.py | 2 +- piccolo/apps/migrations/commands/base.py | 2 +- tests/apps/migrations/commands/test_base.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/piccolo/apps/migrations/auto/migration_manager.py b/piccolo/apps/migrations/auto/migration_manager.py index b9c585f2b..fa0b7dab9 100644 --- a/piccolo/apps/migrations/auto/migration_manager.py +++ b/piccolo/apps/migrations/auto/migration_manager.py @@ -337,7 +337,7 @@ async def get_table_from_snapshot( if app_name is None: app_name = self.app_name - diffable_table = await BaseMigrationManager().get_table_from_snaphot( + diffable_table = await BaseMigrationManager().get_table_from_snapshot( app_name=app_name, table_class_name=table_class_name, max_migration_id=migration_id, diff --git a/piccolo/apps/migrations/commands/base.py b/piccolo/apps/migrations/commands/base.py index 2a715070b..3b4cee100 100644 --- a/piccolo/apps/migrations/commands/base.py +++ b/piccolo/apps/migrations/commands/base.py @@ -114,7 +114,7 @@ async def get_migration_managers( else: return migration_managers - async def get_table_from_snaphot( + async def get_table_from_snapshot( self, app_name: str, table_class_name: str, diff --git a/tests/apps/migrations/commands/test_base.py b/tests/apps/migrations/commands/test_base.py index fb80f2c28..25d1b9494 100644 --- a/tests/apps/migrations/commands/test_base.py +++ b/tests/apps/migrations/commands/test_base.py @@ -43,9 +43,9 @@ def test_get_migration_modules(self): class TestGetTableFromSnapshot(TestCase): @patch.object(BaseMigrationManager, "get_app_config") - def test_get_table_from_snaphot(self, get_app_config: MagicMock): + def test_get_table_from_snapshot(self, get_app_config: MagicMock): """ - Test the get_table_from_snaphot method. + Test the get_table_from_snapshot method. """ get_app_config.return_value = AppConfig( app_name="music", @@ -55,7 +55,7 @@ def test_get_table_from_snaphot(self, get_app_config: MagicMock): ) table = run_sync( - BaseMigrationManager().get_table_from_snaphot( + BaseMigrationManager().get_table_from_snapshot( app_name="music", table_class_name="Band" ) ) From 9dc34d5752eb5e7c94b3e029cfac2cffc5f16ece Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 18 Aug 2022 10:28:28 +0100 Subject: [PATCH 355/727] added `Array.cat` (#591) * added `Array.cat` * increase code coverage --- docs/src/piccolo/query_types/update.rst | 6 +++ docs/src/piccolo/schema/column_types.rst | 6 +++ piccolo/columns/column_types.py | 34 +++++++++++++++ tests/columns/test_array.py | 53 +++++++++++++++++++++++- 4 files changed, 97 insertions(+), 2 deletions(-) diff --git a/docs/src/piccolo/query_types/update.rst b/docs/src/piccolo/query_types/update.rst index 80c25a62f..acd0e1bb8 100644 --- a/docs/src/piccolo/query_types/update.rst +++ b/docs/src/piccolo/query_types/update.rst @@ -152,6 +152,12 @@ Likewise, we can decrease the values by 1 day: force=True ) +Array columns +~~~~~~~~~~~~~ + +You can append values to an array (Postgres only). See :meth:`cat `. + + What about null values? ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/src/piccolo/schema/column_types.rst b/docs/src/piccolo/schema/column_types.rst index e327a4acf..a19ea4753 100644 --- a/docs/src/piccolo/schema/column_types.rst +++ b/docs/src/piccolo/schema/column_types.rst @@ -331,3 +331,9 @@ all === .. automethod:: Array.all + +=== +cat +=== + +.. automethod:: Array.cat diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 2fc76a722..427f6d3ec 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -2535,6 +2535,40 @@ def all(self, value: t.Any) -> Where: else: raise ValueError("Unrecognised engine type") + def cat(self, value: t.List[t.Any]) -> QueryString: + """ + Used in an ``update`` query to append items to an array. + + .. code-block:: python + + >>> await Ticket.update({ + ... Ticket.seat_numbers: Ticket.seat_numbers.cat([1000]) + ... }).where(Ticket.id == 1) + + You can also use the ``+`` symbol if you prefer: + + .. code-block:: python + + >>> await Ticket.update({ + ... Ticket.seat_numbers: Ticket.seat_numbers + [1000] + ... }).where(Ticket.id == 1) + + """ + engine_type = self._meta.engine_type + if engine_type != "postgres": + raise ValueError( + "Only Postgres supports array appending currently." + ) + + if not isinstance(value, list): + value = [value] + + db_column_name = self._meta.db_column_name + return QueryString(f'array_cat("{db_column_name}", {{}})', value) + + def __add__(self, value: t.List[t.Any]) -> QueryString: + return self.cat(value) + ########################################################################### # Descriptors diff --git a/tests/columns/test_array.py b/tests/columns/test_array.py index e46ad4c6a..78e82d231 100644 --- a/tests/columns/test_array.py +++ b/tests/columns/test_array.py @@ -2,7 +2,7 @@ from piccolo.columns.column_types import Array, Integer from piccolo.table import Table -from tests.base import postgres_only +from tests.base import postgres_only, sqlite_only class MyTable(Table): @@ -20,7 +20,7 @@ def test_array_default(self): self.assertTrue(column.default is list) -class TestArrayPostgres(TestCase): +class TestArray(TestCase): """ Make sure an Array column can be created. """ @@ -97,3 +97,52 @@ def test_any(self): .run_sync(), None, ) + + @postgres_only + def test_cat(self): + """ + Make sure values can be appended to an array. + """ + MyTable(value=[1, 1, 1]).save().run_sync() + + MyTable.update( + {MyTable.value: MyTable.value.cat([2])}, force=True + ).run_sync() + + self.assertEqual( + MyTable.select().run_sync(), [{"id": 1, "value": [1, 1, 1, 2]}] + ) + + # Try plus symbol + + MyTable.update( + {MyTable.value: MyTable.value + [3]}, force=True + ).run_sync() + + self.assertEqual( + MyTable.select().run_sync(), [{"id": 1, "value": [1, 1, 1, 2, 3]}] + ) + + # Make sure non-list values work + + MyTable.update( + {MyTable.value: MyTable.value + 4}, force=True + ).run_sync() + + self.assertEqual( + MyTable.select().run_sync(), + [{"id": 1, "value": [1, 1, 1, 2, 3, 4]}], + ) + + @sqlite_only + def test_cat_sqlite(self): + """ + If using SQLite then an exception should be raised currently. + """ + with self.assertRaises(ValueError) as manager: + MyTable.value.cat([2]) + + self.assertEqual( + str(manager.exception), + "Only Postgres supports array appending currently.", + ) From c0a162abeb1a283d5d2621c2037371435f2d7815 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 18 Aug 2022 10:32:27 +0100 Subject: [PATCH 356/727] bumped version --- CHANGES.rst | 16 ++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index ce04cad7c..f6b4b7912 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,22 @@ Changes ======= +0.85.0 +------ + +You can now append items to an array in an update query: + +.. code-block:: python + + await Ticket.update({ + Ticket.seat_numbers: Ticket.seat_numbers + [1000] + }).where(Ticket.id == 1) + +Currently Postgres only. Thanks to @sumitsharansatsangi for suggesting this +feature. + +------------------------------------------------------------------------------- + 0.84.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 2a12b5aea..1e4a39409 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.84.0" +__VERSION__ = "0.85.0" From beddc28a009c8d87bc94abf2d15ca76fa5443b74 Mon Sep 17 00:00:00 2001 From: "Dr. Juno Woods" Date: Thu, 18 Aug 2022 12:49:22 -0700 Subject: [PATCH 357/727] Piccolo raw backwards migrations were not previously being run; it would just run the forwards raw migrations again instead. This patch forces Piccolo to explicitly run forwards or backwards raw migrations. (#592) --- .../apps/migrations/auto/migration_manager.py | 7 ++++++- .../migrations/auto/test_migration_manager.py | 20 +++++++++++++++---- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/piccolo/apps/migrations/auto/migration_manager.py b/piccolo/apps/migrations/auto/migration_manager.py index fa0b7dab9..77da21717 100644 --- a/piccolo/apps/migrations/auto/migration_manager.py +++ b/piccolo/apps/migrations/auto/migration_manager.py @@ -771,7 +771,12 @@ async def run(self, backwards=False): async with engine.transaction(): if not self.preview: - for raw in self.raw: + if direction == "backwards": + raw_list = self.raw_backwards + else: + raw_list = self.raw + + for raw in raw_list: if inspect.iscoroutinefunction(raw): await raw() else: diff --git a/tests/apps/migrations/auto/test_migration_manager.py b/tests/apps/migrations/auto/test_migration_manager.py index 286241e0c..4c5a08df7 100644 --- a/tests/apps/migrations/auto/test_migration_manager.py +++ b/tests/apps/migrations/auto/test_migration_manager.py @@ -121,18 +121,24 @@ def test_raw_function(self): class HasRun(Exception): pass + class HasRunBackwards(Exception): + pass + def run(): raise HasRun("I was run!") + def run_back(): + raise HasRunBackwards("I was run backwards!") + manager = MigrationManager() manager.add_raw(run) - manager.add_raw_backwards(run) + manager.add_raw_backwards(run_back) with self.assertRaises(HasRun): asyncio.run(manager.run()) # Reverse - with self.assertRaises(HasRun): + with self.assertRaises(HasRunBackwards): asyncio.run(manager.run(backwards=True)) def test_raw_coroutine(self): @@ -143,18 +149,24 @@ def test_raw_coroutine(self): class HasRun(Exception): pass + class HasRunBackwards(Exception): + pass + async def run(): raise HasRun("I was run!") + async def run_back(): + raise HasRunBackwards("I was run backwards!") + manager = MigrationManager() manager.add_raw(run) - manager.add_raw_backwards(run) + manager.add_raw_backwards(run_back) with self.assertRaises(HasRun): asyncio.run(manager.run()) # Reverse - with self.assertRaises(HasRun): + with self.assertRaises(HasRunBackwards): asyncio.run(manager.run(backwards=True)) @postgres_only From 5c2b35b0d5671f41f3d79f5ef7d1c92cbd7a609a Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 18 Aug 2022 20:52:04 +0100 Subject: [PATCH 358/727] bumped version --- CHANGES.rst | 8 ++++++++ piccolo/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index f6b4b7912..538e31bfc 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,14 @@ Changes ======= +0.85.1 +------ + +Fixed a bug with migrations - when run backwards, ``raw`` was being called +instead of ``raw_backwards``. Thanks to @translunar for the fix. + +------------------------------------------------------------------------------- + 0.85.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 1e4a39409..d2c907b22 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.85.0" +__VERSION__ = "0.85.1" From 0f785ab947f99d0e2c2848a55df119ab5aafcd8b Mon Sep 17 00:00:00 2001 From: sinisaos Date: Sat, 20 Aug 2022 23:41:36 +0200 Subject: [PATCH 359/727] add email column (#595) * add email column * fix VSCode format error * dependencies changed * add min version to pydantic * add link to piccolo admin docs * add an extra test to make sure email validation works * update `Email` docstring - add more links Co-authored-by: Daniel Townsend --- docs/src/piccolo/ecosystem/index.rst | 3 ++- docs/src/piccolo/schema/column_types.rst | 8 +++++++- piccolo/columns/__init__.py | 1 + piccolo/columns/column_types.py | 11 +++++++++++ piccolo/utils/pydantic.py | 3 +++ requirements/requirements.txt | 2 +- tests/utils/test_pydantic.py | 20 ++++++++++++++++++++ 7 files changed, 45 insertions(+), 3 deletions(-) diff --git a/docs/src/piccolo/ecosystem/index.rst b/docs/src/piccolo/ecosystem/index.rst index 08efe4512..8bcf5aff4 100644 --- a/docs/src/piccolo/ecosystem/index.rst +++ b/docs/src/piccolo/ecosystem/index.rst @@ -25,7 +25,8 @@ Piccolo Admin ------------- Lets you create a powerful web GUI for your tables in two minutes. View the -project on `Github `_. +project on `Github `_, and the +`docs `_ for more information. .. image:: https://raw.githubusercontent.com/piccolo-orm/piccolo_admin/master/docs/images/screenshot.png diff --git a/docs/src/piccolo/schema/column_types.rst b/docs/src/piccolo/schema/column_types.rst index a19ea4753..7f39f7c60 100644 --- a/docs/src/piccolo/schema/column_types.rst +++ b/docs/src/piccolo/schema/column_types.rst @@ -134,7 +134,13 @@ Varchar .. autoclass:: Varchar -------------------------------------------------------------------------------- +===== +Email +===== + +.. autoclass:: Email + +-------------------------------------------------------------------------- **** Time diff --git a/piccolo/columns/__init__.py b/piccolo/columns/__init__.py index 4f8c968e0..88d05320a 100644 --- a/piccolo/columns/__init__.py +++ b/piccolo/columns/__init__.py @@ -11,6 +11,7 @@ Date, Decimal, DoublePrecision, + Email, Float, ForeignKey, Integer, diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 427f6d3ec..dfadda686 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -360,6 +360,17 @@ def __set__(self, obj, value: t.Union[str, None]): obj.__dict__[self._meta.name] = value +class Email(Varchar): + """ + Used for storing email addresses. It's identical to :class:`Varchar`, + except when using :func:`create_pydantic_model ` - + we add email validation to the Pydantic model. This means that :ref:`PiccoloAdmin` + also validates emails addresses. + """ # noqa: E501 + + pass + + class Secret(Varchar): """ This is just an alias to ``Varchar(secret=True)``. It's here for backwards diff --git a/piccolo/utils/pydantic.py b/piccolo/utils/pydantic.py index 216807539..62489e143 100644 --- a/piccolo/utils/pydantic.py +++ b/piccolo/utils/pydantic.py @@ -14,6 +14,7 @@ JSONB, Array, Decimal, + Email, ForeignKey, Numeric, Secret, @@ -211,6 +212,8 @@ def create_pydantic_model( value_type: t.Type = pydantic.condecimal( max_digits=column.precision, decimal_places=column.scale ) + elif isinstance(column, Email): + value_type = pydantic.EmailStr elif isinstance(column, Varchar): value_type = pydantic.constr(max_length=column.length) elif isinstance(column, Array): diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 320b5f9c4..c38e71fb4 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -4,4 +4,4 @@ Jinja2>=2.11.0 targ>=0.3.7 inflection>=0.5.1 typing-extensions>=3.10.0.0 -pydantic>=1.6 +pydantic[email]>=1.6 diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index cc891be37..dea4f3e67 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -8,6 +8,7 @@ JSON, JSONB, Array, + Email, Integer, Numeric, Secret, @@ -32,6 +33,25 @@ class Director(Table): pydantic_model(name="short name") +class TestEmailColumn(TestCase): + def test_email(self): + class Director(Table): + email = Email() + + pydantic_model = create_pydantic_model(table=Director) + + self.assertEqual( + pydantic_model.schema()["properties"]["email"]["format"], + "email", + ) + + with self.assertRaises(ValidationError): + pydantic_model(email="not a valid email") + + # Shouldn't raise an exception: + pydantic_model(email="test@gmail.com") + + class TestNumericColumn(TestCase): """ Numeric and Decimal are the same - so we'll just test Numeric. From 2c1a172b8bc4a1b0cb73a342581ab340145c9a3e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 20 Aug 2022 22:53:28 +0100 Subject: [PATCH 360/727] bumped version --- CHANGES.rst | 27 +++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 538e31bfc..2455bb28d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,33 @@ Changes ======= +0.86.0 +------ + +Added the ``Email`` column type. It's basically identical to ``Varchar``, +except that when we use ``create_pydantic_model`` we add email validation +to the generated Pydantic model. + +.. code-block:: python + + from piccolo.columns.column_types import Email + from piccolo.table import Table + from piccolo.utils.pydantic import create_pydantic_model + + + class MyTable(Table): + email = Email() + + + model = create_pydantic_model(MyTable) + + model(email="not a valid email") + # ValidationError! + +Thanks to @sinisaos for implementing this feature. + +------------------------------------------------------------------------------- + 0.85.1 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index d2c907b22..c220c8874 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.85.1" +__VERSION__ = "0.86.0" From bd4e053e9e2f5253ed8326c4d55ff537635fadb3 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 25 Aug 2022 11:08:55 +0100 Subject: [PATCH 361/727] make sure `get_or_create` works with `prefetch` when creating a new row (#598) * make sure `get_or_create` works with `prefetch` when creating a new row * check band name in test --- piccolo/query/methods/objects.py | 19 ++++++++++- tests/table/test_objects.py | 57 +++++++++++++++++++++++++++----- 2 files changed, 67 insertions(+), 9 deletions(-) diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index 6750e6b30..1931ca3ce 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -65,8 +65,25 @@ async def run(self): await instance.save().run() - instance._was_created = True + # If the user wants us to prefetch related objects, for example: + # + # await Band.objects(Band.manager).get_or_create( + # (Band.name == 'Pythonistas') & (Band.manager == 1) + # ) + # + # Then we need to fetch the related objects. + # See https://github.com/piccolo-orm/piccolo/issues/597 + prefetch = self.query.prefetch_delegate.fk_columns + if prefetch: + table = instance.__class__ + primary_key = table._meta.primary_key + instance = ( + await table.objects(*prefetch) + .get(primary_key == getattr(instance, primary_key._meta.name)) + .run() + ) + instance._was_created = True return instance def __await__(self): diff --git a/tests/table/test_objects.py b/tests/table/test_objects.py index 4593c0ca3..61091778f 100644 --- a/tests/table/test_objects.py +++ b/tests/table/test_objects.py @@ -2,7 +2,7 @@ from tests.example_apps.music.tables import Band, Manager -class TestObjects(DBTestCase): +class TestGetAll(DBTestCase): def test_get_all(self): self.insert_row() @@ -25,6 +25,8 @@ def test_get_all(self): "Rustaceans", ) + +class TestOffset(DBTestCase): @postgres_only def test_offset_postgres(self): """ @@ -55,6 +57,8 @@ def test_offset_sqlite(self): [i.name for i in response], ["Pythonistas", "Rustaceans"] ) + +class TestGet(DBTestCase): def test_get(self): self.insert_row() @@ -62,7 +66,7 @@ def test_get(self): self.assertEqual(band.name, "Pythonistas") - def test_get__prefetch(self): + def test_get_prefetch(self): self.insert_rows() # With prefetch clause @@ -82,7 +86,9 @@ def test_get__prefetch(self): ) self.assertIsInstance(band.manager, Manager) - def test_get_or_create(self): + +class TestGetOrCreate(DBTestCase): + def test_simple_where_clause(self): """ Make sure `get_or_create` works for simple where clauses. """ @@ -112,7 +118,7 @@ def test_get_or_create(self): self.assertEqual(instance.name, "Pink Floyd") self.assertEqual(instance.popularity, 100) - def test_get_or_create_complex(self): + def test_complex_where_clause(self): """ Make sure `get_or_create` works with complex where clauses. """ @@ -140,7 +146,7 @@ def test_get_or_create_complex(self): self.assertIsInstance(instance, Band) self.assertEqual(instance._was_created, True) - def test_get_or_create_very_complex(self): + def test_very_complex_where_clause(self): """ Make sure `get_or_create` works with very complex where clauses. """ @@ -176,7 +182,7 @@ def test_get_or_create_very_complex(self): # be used for the column. self.assertEqual(instance.popularity, 0) - def test_get_or_create_with_joins(self): + def test_joins(self): """ Make sure that that `get_or_create` creates rows correctly when using joins. @@ -196,9 +202,10 @@ def test_get_or_create_with_joins(self): # mistake. self.assertEqual(Band.name, "My new band") - def test_get_or_create__prefetch(self): + def test_prefetch_existing_object(self): """ - Make sure that that `get_or_create` works with the `prefetch` clause. + Make sure that that `get_or_create` works with the `prefetch` clause, + when it's an existing row in the database. """ self.insert_rows() @@ -210,6 +217,7 @@ def test_get_or_create__prefetch(self): .run_sync() ) self.assertIsInstance(band.manager, Manager) + self.assertEqual(band.manager.name, "Guido") # Just passing it straight into objects band = ( @@ -218,3 +226,36 @@ def test_get_or_create__prefetch(self): .run_sync() ) self.assertIsInstance(band.manager, Manager) + self.assertEqual(band.manager.name, "Guido") + + def test_prefetch_new_object(self): + """ + Make sure that that `get_or_create` works with the `prefetch` clause, + when the row is being created in the database. + """ + manager = Manager({Manager.name: "Guido"}) + manager.save().run_sync() + + # With prefetch clause + band = ( + Band.objects() + .get_or_create( + (Band.name == "New Band") & (Band.manager == manager) + ) + .prefetch(Band.manager) + .run_sync() + ) + self.assertIsInstance(band.manager, Manager) + self.assertEqual(band.name, "New Band") + + # Just passing it straight into objects + band = ( + Band.objects(Band.manager) + .get_or_create( + (Band.name == "New Band 2") & (Band.manager == manager) + ) + .run_sync() + ) + self.assertIsInstance(band.manager, Manager) + self.assertEqual(band.name, "New Band 2") + self.assertEqual(band.manager.name, "Guido") From eea071723add422622ce7e340ad24bb83d8a4a8b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 25 Aug 2022 11:15:54 +0100 Subject: [PATCH 362/727] bumped version --- CHANGES.rst | 22 ++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2455bb28d..80df26225 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,28 @@ Changes ======= +0.87.0 +------ + +When using ``get_or_create`` with ``prefetch`` the behaviour was inconsistent - +it worked as expected when the row already existed, but prefetch wasn't working +if the row was being created. This now works as expected: + +.. code-block:: python + + >>> band = Band.objects(Band.manager).get_or_create( + ... (Band.name == "New Band 2") & (Band.manager == 1) + ... ) + + >>> band.manager + + >>> band.manager.name + "Mr Manager" + +Thanks to @backwardspy for reporting this issue. + +------------------------------------------------------------------------------- + 0.86.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index c220c8874..5836170a8 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.86.0" +__VERSION__ = "0.87.0" From e1e881f35e72ca890e16be57e70c06a16eb61552 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 27 Aug 2022 11:53:46 +0100 Subject: [PATCH 363/727] add link to source code from docs (#601) * update doc requirements * add link to GitHub in nav bar --- docs/src/conf.py | 1 + requirements/doc-requirements.txt | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/src/conf.py b/docs/src/conf.py index ca8a64159..4339368b9 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -64,6 +64,7 @@ html_short_title = "Piccolo" html_show_sphinx = False globaltoc_maxdepth = 3 +html_theme_options = {"source_url": "https://github.com/piccolo-orm/piccolo/"} # -- Options for HTMLHelp output --------------------------------------------- diff --git a/requirements/doc-requirements.txt b/requirements/doc-requirements.txt index b646370b0..3c23f8905 100644 --- a/requirements/doc-requirements.txt +++ b/requirements/doc-requirements.txt @@ -1,3 +1,3 @@ -Sphinx==4.4.0 -piccolo-theme>=0.3.0 +Sphinx==5.1.1 +piccolo-theme>=0.12.0 sphinx-autobuild==2021.3.14 From 305819612bb935cef25804667b223478f6ad71b7 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 27 Aug 2022 15:15:47 +0100 Subject: [PATCH 364/727] fix bug with `db_column_name` not being used in alter statements (#602) --- piccolo/apps/migrations/auto/schema_differ.py | 2 +- .../migrations/auto/test_schema_differ.py | 40 ++++++++++++++++++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/piccolo/apps/migrations/auto/schema_differ.py b/piccolo/apps/migrations/auto/schema_differ.py index 2427c44a5..5e8ca076c 100644 --- a/piccolo/apps/migrations/auto/schema_differ.py +++ b/piccolo/apps/migrations/auto/schema_differ.py @@ -363,7 +363,7 @@ def alter_columns(self) -> AlterStatements: ) response.append( - f"manager.alter_column(table_class_name='{table.class_name}', tablename='{table.tablename}', column_name='{alter_column.column_name}', params={new_params.params}, old_params={old_params.params}, column_class={column_class}, old_column_class={old_column_class})" # noqa: E501 + f"manager.alter_column(table_class_name='{table.class_name}', tablename='{table.tablename}', column_name='{alter_column.column_name}', db_column_name='{alter_column.db_column_name}', params={new_params.params}, old_params={old_params.params}, column_class={column_class}, old_column_class={old_column_class})" # noqa: E501 ) return AlterStatements( diff --git a/tests/apps/migrations/auto/test_schema_differ.py b/tests/apps/migrations/auto/test_schema_differ.py index 6fccff2e5..565aa332d 100644 --- a/tests/apps/migrations/auto/test_schema_differ.py +++ b/tests/apps/migrations/auto/test_schema_differ.py @@ -406,7 +406,45 @@ def test_alter_column_precision(self): self.assertTrue(len(schema_differ.alter_columns.statements) == 1) self.assertEqual( schema_differ.alter_columns.statements[0], - "manager.alter_column(table_class_name='Ticket', tablename='ticket', column_name='price', params={'digits': (4, 2)}, old_params={'digits': (5, 2)}, column_class=Numeric, old_column_class=Numeric)", # noqa + "manager.alter_column(table_class_name='Ticket', tablename='ticket', column_name='price', db_column_name='price', params={'digits': (4, 2)}, old_params={'digits': (5, 2)}, column_class=Numeric, old_column_class=Numeric)", # noqa + ) + + def test_db_column_name(self): + """ + Make sure alter statements use the ``db_column_name`` if provided. + + https://github.com/piccolo-orm/piccolo/issues/513 + + """ + price_1 = Numeric(digits=(4, 2), db_column_name="custom") + price_1._meta.name = "price" + + price_2 = Numeric(digits=(5, 2), db_column_name="custom") + price_2._meta.name = "price" + + schema: t.List[DiffableTable] = [ + DiffableTable( + class_name="Ticket", + tablename="ticket", + columns=[price_1], + ) + ] + schema_snapshot: t.List[DiffableTable] = [ + DiffableTable( + class_name="Ticket", + tablename="ticket", + columns=[price_2], + ) + ] + + schema_differ = SchemaDiffer( + schema=schema, schema_snapshot=schema_snapshot, auto_input="y" + ) + + self.assertTrue(len(schema_differ.alter_columns.statements) == 1) + self.assertEqual( + schema_differ.alter_columns.statements[0], + "manager.alter_column(table_class_name='Ticket', tablename='ticket', column_name='price', db_column_name='custom', params={'digits': (4, 2)}, old_params={'digits': (5, 2)}, column_class=Numeric, old_column_class=Numeric)", # noqa ) def test_alter_default(self): From eedc8688bc32b859be44a870270e230ff8a85f72 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 27 Aug 2022 15:40:49 +0100 Subject: [PATCH 365/727] bumped version --- CHANGES.rst | 16 ++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 80df26225..ac5efc093 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,22 @@ Changes ======= +0.88.0 +------ + +Fixed a bug with migrations - when using ``db_column_name`` it wasn't being +used in some alter statements. Thanks to @theelderbeever for reporting this +issue. + +.. code-block:: python + + class Concert(Table): + # We use `db_column_name`` when the column name is problematic - e.g. if + # it clashes with a Python ketword. + in_ = Varchar(db_column_name='in') + +------------------------------------------------------------------------------- + 0.87.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 5836170a8..c3ddb547c 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.87.0" +__VERSION__ = "0.88.0" From ed93feb85050895cc54e7db9650eddc69ab94fa0 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 27 Aug 2022 15:44:29 +0100 Subject: [PATCH 366/727] fix typos in changelog --- CHANGES.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index ac5efc093..69b206fc8 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -11,8 +11,8 @@ issue. .. code-block:: python class Concert(Table): - # We use `db_column_name`` when the column name is problematic - e.g. if - # it clashes with a Python ketword. + # We use `db_column_name` when the column name is problematic - e.g. if + # it clashes with a Python keyword. in_ = Varchar(db_column_name='in') ------------------------------------------------------------------------------- From a6c8008c90eb9e29f5899a375027c411e8bbfa62 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 1 Sep 2022 19:36:38 +0100 Subject: [PATCH 367/727] add `Table._meta.email_columns` (#610) --- piccolo/table.py | 7 +++++++ tests/table/test_metaclass.py | 19 ++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/piccolo/table.py b/piccolo/table.py index 604f630fd..c0bcac287 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -11,6 +11,7 @@ JSON, JSONB, Array, + Email, ForeignKey, Secret, Serial, @@ -69,6 +70,7 @@ class TableMeta: columns: t.List[Column] = field(default_factory=list) default_columns: t.List[Column] = field(default_factory=list) non_default_columns: t.List[Column] = field(default_factory=list) + email_columns: t.List[Email] = field(default_factory=list) foreign_key_columns: t.List[ForeignKey] = field(default_factory=list) primary_key: Column = field(default_factory=Column) json_columns: t.List[t.Union[JSON, JSONB]] = field(default_factory=list) @@ -216,6 +218,7 @@ def __init_subclass__( foreign_key_columns: t.List[ForeignKey] = [] secret_columns: t.List[Secret] = [] json_columns: t.List[t.Union[JSON, JSONB]] = [] + email_columns: t.List[Email] = [] primary_key: t.Optional[Column] = None m2m_relationships: t.List[M2M] = [] @@ -249,6 +252,9 @@ def __init_subclass__( if isinstance(column, Array): column.base_column._meta._table = cls + if isinstance(column, Email): + email_columns.append(column) + if isinstance(column, Secret): secret_columns.append(column) @@ -275,6 +281,7 @@ def __init_subclass__( columns=columns, default_columns=default_columns, non_default_columns=non_default_columns, + email_columns=email_columns, primary_key=primary_key, foreign_key_columns=foreign_key_columns, json_columns=json_columns, diff --git a/tests/table/test_metaclass.py b/tests/table/test_metaclass.py index 0afff0290..a89bd005d 100644 --- a/tests/table/test_metaclass.py +++ b/tests/table/test_metaclass.py @@ -1,7 +1,13 @@ from unittest import TestCase from piccolo.columns import Secret -from piccolo.columns.column_types import JSON, JSONB, ForeignKey +from piccolo.columns.column_types import ( + JSON, + JSONB, + Email, + ForeignKey, + Varchar, +) from piccolo.table import Table from tests.example_apps.music.tables import Band @@ -76,6 +82,17 @@ class MyTable(Table): MyTable._meta.json_columns, [MyTable.column_a, MyTable.column_b] ) + def test_email_columns(self): + """ + Make sure ``TableMeta.email_columns`` are setup correctly. + """ + + class MyTable(Table): + column_a = Email() + column_b = Varchar() + + self.assertEqual(MyTable._meta.email_columns, [MyTable.column_a]) + def test_id_column(self): """ Makes sure an id column is added. From 69672fc050d6fb11511e0e4f16550216bd8f41de Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 1 Sep 2022 20:08:50 +0100 Subject: [PATCH 368/727] bumped version --- CHANGES.rst | 14 ++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 69b206fc8..b98b15297 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,20 @@ Changes ======= +0.89.0 +------ + +Made it easier to access the ``Email`` columns on table. + +.. code-block:: python + + >>> MyTable._meta.email_columns + [MyTable.email_column_1, MyTable.email_column_2] + +This was added for Piccolo Admin. + +------------------------------------------------------------------------------- + 0.88.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index c3ddb547c..b2406f42b 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.88.0" +__VERSION__ = "0.89.0" From 0b9a86b3c304dccd912115c2d3d77c4bebc132bc Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 1 Sep 2022 20:10:38 +0100 Subject: [PATCH 369/727] fix grammar in change log --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index b98b15297..cd78ef799 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,7 +4,7 @@ Changes 0.89.0 ------ -Made it easier to access the ``Email`` columns on table. +Made it easier to access the ``Email`` columns on a table. .. code-block:: python From 9475b7d7fcaefe66f82ddf360e04ccd8e2dc9e3c Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 7 Sep 2022 15:32:05 +0100 Subject: [PATCH 370/727] fix migrations with long foreign key chains (#617) --- piccolo/table.py | 11 +-- .../auto/integration/test_migrations.py | 80 ++++++++++++++----- .../migrations/auto/test_migration_manager.py | 48 +++++++++-- 3 files changed, 110 insertions(+), 29 deletions(-) diff --git a/piccolo/table.py b/piccolo/table.py index c0bcac287..8aeeee6f4 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -1439,12 +1439,13 @@ def _get_graph( # We also recursively check the related tables to get a fuller # picture of the schema and relationships. - output.update( - _get_graph( - [referenced_table], - iterations=iterations + 1, + if referenced_table._meta.tablename not in output: + output.update( + _get_graph( + [referenced_table], + iterations=iterations + 1, + ) ) - ) output[table_class._meta.tablename] = dependents diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index d7261a074..bb16a4f0a 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -182,10 +182,15 @@ def tearDown(self): ########################################################################### def table(self, column: Column): + """ + A utility for creating Piccolo tables with the given column. + """ return create_table_class( class_name="MyTable", class_members={"my_column": column} ) + ########################################################################### + def test_varchar_column(self): self._test_migrations( table_snapshots=[ @@ -854,25 +859,56 @@ def test_m2m(self): ############################################################################### -class TableA(Table): - name = Varchar(unique=True) +@postgres_only +class TestForeignKeys(MigrationTestCase): + def setUp(self): + class TableA(Table): + pass + + class TableB(Table): + fk = ForeignKey(TableA) + + class TableC(Table): + fk = ForeignKey(TableB) + + class TableD(Table): + fk = ForeignKey(TableC) + class TableE(Table): + fk = ForeignKey(TableD) -class TableB(Table): - table_a = ForeignKey(TableA, target_column="name") + self.table_classes = [TableA, TableB, TableC, TableD, TableE] + def tearDown(self): + drop_db_tables_sync(Migration, *self.table_classes) -class TableC(Table): - table_a = ForeignKey(TableA, target_column=TableA.name) + def test_foreign_keys(self): + """ + Make sure that if we try creating tables with lots of foreign keys + to each other it runs successfully. + + https://github.com/piccolo-orm/piccolo/issues/616 + + """ + + self._test_migrations(table_snapshots=[self.table_classes]) + for table_class in self.table_classes: + self.assertTrue(table_class.table_exists().run_sync()) @postgres_only class TestTargetColumn(MigrationTestCase): def setUp(self): - pass + class TableA(Table): + name = Varchar(unique=True) + + class TableB(Table): + table_a = ForeignKey(TableA, target_column=TableA.name) + + self.table_classes = [TableA, TableB] def tearDown(self): - drop_db_tables_sync(Migration, TableA, TableC) + drop_db_tables_sync(Migration, *self.table_classes) def test_target_column(self): """ @@ -880,14 +916,14 @@ def test_target_column(self): other than the primary key. """ self._test_migrations( - table_snapshots=[[TableA, TableC]], + table_snapshots=[self.table_classes], ) - for table_class in [TableA, TableC]: + for table_class in self.table_classes: self.assertTrue(table_class.table_exists().run_sync()) # Make sure the constraint was created correctly. - response = TableA.raw( + response = self.run_sync( """ SELECT EXISTS( SELECT 1 @@ -895,22 +931,28 @@ def test_target_column(self): JOIN INFORMATION_SCHEMA.TABLE_CONSTRAINTS TC ON CCU.CONSTRAINT_NAME = TC.CONSTRAINT_NAME WHERE CONSTRAINT_TYPE = 'FOREIGN KEY' - AND TC.TABLE_NAME = 'table_c' + AND TC.TABLE_NAME = 'table_b' AND CCU.TABLE_NAME = 'table_a' AND CCU.COLUMN_NAME = 'name' ) """ - ).run_sync() + ) self.assertTrue(response[0]["exists"]) @postgres_only class TestTargetColumnString(MigrationTestCase): def setUp(self): - pass + class TableA(Table): + name = Varchar(unique=True) + + class TableB(Table): + table_a = ForeignKey(TableA, target_column="name") + + self.table_classes = [TableA, TableB] def tearDown(self): - drop_db_tables_sync(Migration, TableA, TableB) + drop_db_tables_sync(Migration, *self.table_classes) def test_target_column(self): """ @@ -918,14 +960,14 @@ def test_target_column(self): other than the primary key. """ self._test_migrations( - table_snapshots=[[TableA, TableB]], + table_snapshots=[self.table_classes], ) - for table_class in [TableA, TableB]: + for table_class in self.table_classes: self.assertTrue(table_class.table_exists().run_sync()) # Make sure the constraint was created correctly. - response = TableA.raw( + response = self.run_sync( """ SELECT EXISTS( SELECT 1 @@ -938,5 +980,5 @@ def test_target_column(self): AND CCU.COLUMN_NAME = 'name' ) """ - ).run_sync() + ) self.assertTrue(response[0]["exists"]) diff --git a/tests/apps/migrations/auto/test_migration_manager.py b/tests/apps/migrations/auto/test_migration_manager.py index 4c5a08df7..c416ae33f 100644 --- a/tests/apps/migrations/auto/test_migration_manager.py +++ b/tests/apps/migrations/auto/test_migration_manager.py @@ -1,4 +1,5 @@ import asyncio +import random from io import StringIO from unittest import TestCase from unittest.mock import MagicMock, patch @@ -19,8 +20,15 @@ class TestSortTableClasses(TestCase): def test_sort_table_classes(self): - self.assertEqual(sort_table_classes([Manager, Band]), [Manager, Band]) - self.assertEqual(sort_table_classes([Band, Manager]), [Manager, Band]) + """ + Make sure simple use cases work correctly. + """ + self.assertListEqual( + sort_table_classes([Manager, Band]), [Manager, Band] + ) + self.assertListEqual( + sort_table_classes([Band, Manager]), [Manager, Band] + ) sorted_tables = sort_table_classes([Manager, Venue, Concert, Band]) self.assertTrue( @@ -45,7 +53,7 @@ class TableA(Table): class TableB(Table): pass - self.assertEqual( + self.assertListEqual( sort_table_classes([TableA, TableB]), [TableA, TableB] ) @@ -54,7 +62,7 @@ def test_single_table(self): Make sure that sorting a list with only a single table in it still works. """ - self.assertEqual(sort_table_classes([Band]), [Band]) + self.assertListEqual(sort_table_classes([Band]), [Band]) def test_recursive_table(self): """ @@ -68,10 +76,40 @@ class TableA(Table): class TableB(Table): table_a = ForeignKey(TableA) - self.assertEqual( + self.assertListEqual( sort_table_classes([TableA, TableB]), [TableA, TableB] ) + def test_long_chain(self): + """ + Make sure sorting works when there are a lot of tables with foreign + keys to each other. + + https://github.com/piccolo-orm/piccolo/issues/616 + + """ + + class TableA(Table): + pass + + class TableB(Table): + fk = ForeignKey(TableA) + + class TableC(Table): + fk = ForeignKey(TableB) + + class TableD(Table): + fk = ForeignKey(TableC) + + class TableE(Table): + fk = ForeignKey(TableD) + + tables = [TableA, TableB, TableC, TableD, TableE] + + shuffled_tables = random.sample(tables, len(tables)) + + self.assertListEqual(sort_table_classes(shuffled_tables), tables) + class TestMigrationManager(DBTestCase): @postgres_only From c9bb7ffce36e076d793f09349d4291d03a981f47 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 7 Sep 2022 15:47:10 +0100 Subject: [PATCH 371/727] bumped version --- CHANGES.rst | 30 ++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index cd78ef799..e85f34bba 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,36 @@ Changes ======= +0.90.0 +------ + +Fixed an edge case, where a migration could fail if: + +* 5 or more tables were being created at once. +* They all contained foreign keys to each other, as shown below. + +.. code-block:: python + + class TableA(Table): + pass + + class TableB(Table): + fk = ForeignKey(TableA) + + class TableC(Table): + fk = ForeignKey(TableB) + + class TableD(Table): + fk = ForeignKey(TableC) + + class TableE(Table): + fk = ForeignKey(TableD) + + +Thanks to @sumitsharansatsangi for reporting this issue. + +------------------------------------------------------------------------------- + 0.89.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index b2406f42b..adad69fdf 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.89.0" +__VERSION__ = "0.90.0" From 7754498705540c81c40ec203a7bb429d59a07da5 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 7 Sep 2022 16:27:54 +0100 Subject: [PATCH 372/727] make foreign key migration test more thorough (#618) Shuffling the tables. --- .../apps/migrations/auto/integration/test_migrations.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index bb16a4f0a..5b6c58a6d 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -3,6 +3,7 @@ import datetime import decimal import os +import random import shutil import tempfile import time @@ -890,9 +891,13 @@ def test_foreign_keys(self): https://github.com/piccolo-orm/piccolo/issues/616 """ + # We'll shuffle them, to make it a more thorough test. + table_classes = random.sample( + self.table_classes, len(self.table_classes) + ) - self._test_migrations(table_snapshots=[self.table_classes]) - for table_class in self.table_classes: + self._test_migrations(table_snapshots=[table_classes]) + for table_class in table_classes: self.assertTrue(table_class.table_exists().run_sync()) From 3e0859a5f09d1d4a6996e8877b6ce603bb335fb1 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Sat, 24 Sep 2022 17:56:38 +0200 Subject: [PATCH 373/727] add Starlite asgi template (#623) --- README.md | 2 +- docs/src/piccolo/asgi/index.rst | 4 +- piccolo/apps/asgi/commands/new.py | 2 +- .../templates/app/_starlite_app.py.jinja | 116 ++++++++++++++++++ .../asgi/commands/templates/app/app.py.jinja | 2 + .../app/home/_starlite_endpoints.py.jinja | 21 ++++ .../templates/app/home/endpoints.py.jinja | 2 + .../app/home/templates/home.html.jinja_raw | 5 + 8 files changed, 150 insertions(+), 4 deletions(-) create mode 100644 piccolo/apps/asgi/commands/templates/app/_starlite_app.py.jinja create mode 100644 piccolo/apps/asgi/commands/templates/app/home/_starlite_endpoints.py.jinja diff --git a/README.md b/README.md index bf53d082a..2d7b1eef1 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ Let Piccolo scaffold you an ASGI web app, using Piccolo as the ORM: piccolo asgi new ``` -[Starlette](https://www.starlette.io/), [FastAPI](https://fastapi.tiangolo.com/), [BlackSheep](https://www.neoteroi.dev/blacksheep/) and [Xpresso](https://xpresso-api.dev/) are currently supported. +[Starlette](https://www.starlette.io/), [FastAPI](https://fastapi.tiangolo.com/), [BlackSheep](https://www.neoteroi.dev/blacksheep/), [Xpresso](https://xpresso-api.dev/) and [Starlite](https://starlite-api.github.io/starlite/) are currently supported. ## Are you a Django user? diff --git a/docs/src/piccolo/asgi/index.rst b/docs/src/piccolo/asgi/index.rst index 1da8d7a64..7f88a64d7 100644 --- a/docs/src/piccolo/asgi/index.rst +++ b/docs/src/piccolo/asgi/index.rst @@ -21,8 +21,8 @@ Routing frameworks ****************** Currently, `Starlette `_, `FastAPI `_, -`BlackSheep `_ and `Xpresso `_ -are supported. +`BlackSheep `_, `Xpresso `_ and +`Starlite `_ are supported. Other great ASGI routing frameworks exist, and may be supported in the future (`Quart `_ , diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index 10217793f..06d48f07b 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -10,7 +10,7 @@ TEMPLATE_DIR = os.path.join(os.path.dirname(__file__), "templates/app/") SERVERS = ["uvicorn", "Hypercorn"] -ROUTERS = ["starlette", "fastapi", "blacksheep", "xpresso"] +ROUTERS = ["starlette", "fastapi", "blacksheep", "xpresso", "starlite"] def print_instruction(message: str): diff --git a/piccolo/apps/asgi/commands/templates/app/_starlite_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_starlite_app.py.jinja new file mode 100644 index 000000000..f73b53d24 --- /dev/null +++ b/piccolo/apps/asgi/commands/templates/app/_starlite_app.py.jinja @@ -0,0 +1,116 @@ +import typing as t + +from piccolo.engine import engine_finder +from piccolo_admin.endpoints import create_admin +from starlette.routing import Mount, Router +from starlite import ( + MediaType, + Response, + Starlite, + StaticFilesConfig, + TemplateConfig, + asgi, + delete, + get, + patch, + post, +) +from starlite.plugins.piccolo_orm import PiccoloORMPlugin +from starlite.template.jinja import JinjaTemplateEngine +from starlite.types import Receive, Scope, Send + +from home.endpoints import home +from home.piccolo_app import APP_CONFIG +from home.tables import Task + + +@asgi(path="/{admin:path}") +async def admin(scope: Scope, receive: Receive, send: Send) -> None: + router = Router( + [ + Mount( + path="/admin", + app=create_admin(tables=APP_CONFIG.table_classes), + ), + ] + ) + await router(scope=scope, receive=receive, send=send) + + +@get("/tasks", tags=["Task"]) +async def tasks() -> t.List[Task]: + tasks = await Task.select().order_by(Task.id, ascending=False) + return tasks + + +@post("/tasks", tags=["Task"]) +async def create_task(data: Task) -> Task: + task = Task(**data.to_dict()) + await task.save() + return task + + +@patch("/tasks/{task_id:int}", tags=["Task"]) +async def update_task(task_id: int, data: Task) -> Task: + task = await Task.objects().get(Task.id == task_id) + if not task: + return Response( + content={}, + media_type=MediaType.JSON, + status_code=404, + ) + for key, value in data.to_dict().items(): + task.id = task_id + setattr(task, key, value) + + await task.save() + return task + + +@delete("/tasks/{task_id:int}", tags=["Task"]) +async def delete_task(task_id: int) -> None: + task = await Task.objects().get(Task.id == task_id) + if not task: + return Response( + content={}, + media_type=MediaType.JSON, + status_code=404, + ) + await task.remove() + + +async def open_database_connection_pool(): + try: + engine = engine_finder() + await engine.start_connection_pool() + except Exception: + print("Unable to connect to the database") + + +async def close_database_connection_pool(): + try: + engine = engine_finder() + await engine.close_connection_pool() + except Exception: + print("Unable to connect to the database") + + +app = Starlite( + route_handlers=[ + admin, + home, + tasks, + create_task, + update_task, + delete_task, + ], + plugins=[PiccoloORMPlugin()], + template_config=TemplateConfig( + directory="home/templates", engine=JinjaTemplateEngine + ), + static_files_config=[ + StaticFilesConfig(directories=["static"], path="/static/"), + ], + on_startup=[open_database_connection_pool], + on_shutdown=[close_database_connection_pool], +) diff --git a/piccolo/apps/asgi/commands/templates/app/app.py.jinja b/piccolo/apps/asgi/commands/templates/app/app.py.jinja index 18e45a23b..47d673aad 100644 --- a/piccolo/apps/asgi/commands/templates/app/app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/app.py.jinja @@ -6,4 +6,6 @@ {% include '_blacksheep_app.py.jinja' %} {% elif router == 'xpresso' %} {% include '_xpresso_app.py.jinja' %} +{% elif router == 'starlite' %} + {% include '_starlite_app.py.jinja' %} {% endif %} diff --git a/piccolo/apps/asgi/commands/templates/app/home/_starlite_endpoints.py.jinja b/piccolo/apps/asgi/commands/templates/app/home/_starlite_endpoints.py.jinja new file mode 100644 index 000000000..9ced8951c --- /dev/null +++ b/piccolo/apps/asgi/commands/templates/app/home/_starlite_endpoints.py.jinja @@ -0,0 +1,21 @@ +import os + +import jinja2 +from starlite import MediaType, Request, Response, get + +ENVIRONMENT = jinja2.Environment( + loader=jinja2.FileSystemLoader( + searchpath=os.path.join(os.path.dirname(__file__), "templates") + ) +) + + +@get(path="/", include_in_schema=False) +def home(request: Request) -> Response: + template = ENVIRONMENT.get_template("home.html.jinja") + content = template.render(title="Piccolo + ASGI") + return Response( + content, + media_type=MediaType.HTML, + status_code=200, + ) diff --git a/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja b/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja index 36e4eaf49..c864994a9 100644 --- a/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja @@ -4,4 +4,6 @@ {% include '_blacksheep_endpoints.py.jinja' %} {% elif router == 'xpresso' %} {% include '_xpresso_endpoints.py.jinja' %} +{% elif router == 'starlite' %} + {% include '_starlite_endpoints.py.jinja' %} {% endif %} diff --git a/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw b/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw index 614f75ba6..ed5934f9f 100644 --- a/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw +++ b/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw @@ -56,6 +56,11 @@
  • Admin
  • Swagger API
    • +

    Starlite

    +
    {% endblock content %} From 4f3c1f30a75222452cce3a7b80ded98c1b78924d Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 24 Sep 2022 17:01:57 +0100 Subject: [PATCH 374/727] bumped version --- CHANGES.rst | 11 +++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index e85f34bba..3977ebbb3 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,17 @@ Changes ======= +0.91.0 +------ + +Added support for Starlite. If you use ``piccolo asgi new`` you'll see it as +an option for a router. + +Thanks to @sinisaos for adding this, and @peterschutt for helping debug ASGI +mounting. + +------------------------------------------------------------------------------- + 0.90.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index adad69fdf..716c5a75a 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.90.0" +__VERSION__ = "0.91.0" From ce970b8de66e4a6a6cff9d653f5bad5f47dfdf72 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 3 Oct 2022 20:16:42 +0100 Subject: [PATCH 375/727] version pin Starlite to an older version so it works with FastAPI (#631) --- piccolo/apps/asgi/commands/new.py | 6 +++++- .../apps/asgi/commands/templates/app/requirements.txt.jinja | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index 06d48f07b..4d5b09db6 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -11,6 +11,7 @@ TEMPLATE_DIR = os.path.join(os.path.dirname(__file__), "templates/app/") SERVERS = ["uvicorn", "Hypercorn"] ROUTERS = ["starlette", "fastapi", "blacksheep", "xpresso", "starlite"] +ROUTER_VERSIONS = {"starlite": "==1.23.0"} def print_instruction(message: str): @@ -47,8 +48,11 @@ def new(root: str = ".", name: str = "piccolo_project"): """ tree = os.walk(TEMPLATE_DIR) + router = get_routing_framework() + template_context = { - "router": get_routing_framework(), + "router": router, + "router_version": ROUTER_VERSIONS.get(router), "server": get_server(), "project_identifier": name.replace(" ", "_").lower(), } diff --git a/piccolo/apps/asgi/commands/templates/app/requirements.txt.jinja b/piccolo/apps/asgi/commands/templates/app/requirements.txt.jinja index 4b4a52171..ff7f895d2 100644 --- a/piccolo/apps/asgi/commands/templates/app/requirements.txt.jinja +++ b/piccolo/apps/asgi/commands/templates/app/requirements.txt.jinja @@ -1,5 +1,6 @@ -{{ router }} +{{ router }}{% if router_version %}{{ router_version }}{% endif %} {{ server }} jinja2 piccolo[postgres] piccolo_admin +requests From 45dbb8b3053313ea91adf160bd6300309a54def9 Mon Sep 17 00:00:00 2001 From: Miguel Guthridge Date: Tue, 4 Oct 2022 05:42:38 +1000 Subject: [PATCH 376/727] Add named exports for column module (#629) * Add named exports for column module * fix linter Co-authored-by: Daniel Townsend --- piccolo/columns/__init__.py | 39 +++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/piccolo/columns/__init__.py b/piccolo/columns/__init__.py index 88d05320a..e84d4c3ad 100644 --- a/piccolo/columns/__init__.py +++ b/piccolo/columns/__init__.py @@ -30,3 +30,42 @@ from .combination import And, Or, Where from .m2m import M2M from .reference import LazyTableReference + +__all__ = [ + "Column", + "ForeignKeyMeta", + "OnDelete", + "OnUpdate", + "Selectable", + "JSON", + "JSONB", + "UUID", + "Array", + "BigInt", + "BigSerial", + "Boolean", + "Bytea", + "Date", + "Decimal", + "DoublePrecision", + "Email", + "Float", + "ForeignKey", + "Integer", + "Interval", + "Numeric", + "PrimaryKey", + "Real", + "Secret", + "Serial", + "SmallInt", + "Text", + "Timestamp", + "Timestamptz", + "Varchar", + "And", + "Or", + "Where", + "M2M", + "LazyTableReference", +] From 721dc2bf9297fdedbf74ca96ce8dd19aa0e93481 Mon Sep 17 00:00:00 2001 From: Nathaniel Sabanski Date: Tue, 11 Oct 2022 03:42:41 -0700 Subject: [PATCH 377/727] CockroachDB Support (#608) * Cockroach config for tests. * Added feedback from #607 * New CockroachDB engine. * No more extension=[] ! * Running tests as postgres. * Initial build for CockroachDB * Running tests. * Run workflow. * Naming consistency. Making roughly equivalent to Postgres. * Timestamp bug. * DDL for Cockroach. for_engines decorator. * Progress. * Added database prep for cluster settings. * Tests passing. * Passing migration type checks. * Cockroach ALTER COLUMN TYPE moved outside of transaction until #49351 is solved. * New test helpers. * Fixtures. * Cockroach specific stuff. * Array tests. * Column tests. * JSON tests. * M2M tables. as_list is effected by https://github.com/cockroachdb/cockroach/issues/71908 * Skip test for numeric. * Pool test. * Transaction tests. * Related readable tests. * Save tests. * To dict tests. * JOIN tests. * Output tests. * Raw tests. * Repr tests. * Select tests. * "As string" tests. * Cockroach having trouble with one specific operator. * Skipping until CRDB issue is resolved. Should work though! * Minor updates. * Test dump load. Other CRDB things. * Migration manager tests. * ALTER tests. * SQL Generator tests. * Serialization tests. * Test helpers. * Cockroach Specific stuff. * Numerics all the same for CRDB. * Cockroach specific stuff. * Cleanup. * Skipping timestamp tests. * DDL resolution stuff for Cockroach. * WIP. * Cohesion and special test tables for Cockroach. * WIP * Special table for Cockroach. * Removed hack. * Should pass now. * Removed some debug stuff. * LGTM stuff. * Feedback. * Feedback. * Feedback. * More readable "postgres" or "cockroach. * Added Cockroach overrides for COLUMN_TYPE_MAP, COLUMN_DEFAULT_PARSER * Restore SmallInt functionality for Cockroach. * Restored type() comparison for reflection / generators. * Updated to latest Cockroach release. * Cleanup. * Cleanup. * Build stuff. * Cockroach Serial now uses unique_rowid() as default value always instead of DEFAULT. * Removed first_id() test helper. * Time Travel queries using AS OF SYSTEM TIME * Added documentation. * Small docs fix. * Test update. * Refactored out unnecessary code because we use unique_rowid() as default for Cockroach Serial and BigSerial. * BIGINT instead of BIGSERIAL for Cockroach. * LGTM stuff * LGTM... * Made compatible with older Python versions. * Get Cockroach passing without Python 3.9 stuff. * Removed unused variable. * Fixed test for SQLite. * Re-add Python 3.7 for CRDB + new CRDB version. * Consolidated cockroach_skip() into engines_skip() everywhere. * Re-add SQLite DDL. * Moved back to original engine DDL selector. * Reverted mistake in test suite. * Remove migration that should not be available. * Moving back to postgres_only for specific test. Probably cannot detect engine type at this stage. * postgres_only() to engine_only() * Set sane default for JSONB because we override column_type in JSON for some engines. * Ran isort. * Black being obnoxious, part 1. * Black being obnoxious Part 2. * Flake8 * Black * Flake8 * Flake8 * Flake8 * isort * Flake8 * Flake8 * Added alternate test names for duplicates to remove F811 * Added alternative name. * mypy * mypy * mypy * mypy * mypy * mypy * mypy * mypy * mypy * Cockroach: Now testing latest stable and upcoming release. * Cockroach: Testing only v22.2 * Cockroach version documentation. * Engine detection now based on type because CockroachEngine inherits PostgresEngine. * postgres_only is now engines_only("postgres", "cockroach") because CockroachEngine was always tested as a child class. * Updated to Cockroach 22.2 beta. * isort. * assertAlmostEqual for Decimal test. Docs update. * Linter * Cockroach 22.2 Beta 2. * Added notes about special Column types for Cockroach * version pin xpresso, so integration tests pass * add small section about running cockroach in contributing docs * fix typo Co-authored-by: Daniel Townsend --- .github/workflows/tests.yaml | 36 +- docs/src/piccolo/contributing/index.rst | 20 ++ docs/src/piccolo/engines/cockroach_engine.rst | 87 +++++ docs/src/piccolo/engines/index.rst | 1 + .../piccolo/features/supported_databases.rst | 6 + .../getting_started/database_support.rst | 2 + docs/src/piccolo/getting_started/index.rst | 1 + .../getting_started/installing_piccolo.rst | 2 +- .../getting_started/setup_cockroach.rst | 68 ++++ piccolo/apps/asgi/commands/new.py | 2 +- .../apps/migrations/auto/migration_manager.py | 7 + piccolo/apps/schema/commands/generate.py | 56 ++- piccolo/columns/base.py | 9 +- piccolo/columns/column_types.py | 49 ++- piccolo/columns/defaults/date.py | 12 + piccolo/columns/defaults/interval.py | 4 + piccolo/columns/defaults/time.py | 15 + piccolo/columns/defaults/timestamp.py | 17 + piccolo/columns/defaults/timestamptz.py | 15 + piccolo/columns/defaults/uuid.py | 4 + piccolo/columns/m2m.py | 2 +- piccolo/columns/readable.py | 4 + piccolo/engine/__init__.py | 9 +- piccolo/engine/cockroach.py | 117 +++++++ piccolo/query/base.py | 20 +- piccolo/query/methods/create_index.py | 14 + piccolo/query/methods/indexes.py | 10 + piccolo/query/methods/insert.py | 2 +- piccolo/query/methods/objects.py | 8 + piccolo/query/methods/select.py | 13 +- piccolo/query/methods/table_exists.py | 9 + piccolo/query/mixins.py | 31 ++ piccolo/querystring.py | 2 +- piccolo_conf.py | 1 - scripts/test-cockroach.sh | 14 + .../apps/fixtures/commands/test_dump_load.py | 85 +++++ .../auto/integration/test_migrations.py | 86 +++-- .../migrations/auto/test_migration_manager.py | 329 +++++++++++++++--- .../migrations/auto/test_serialisation.py | 32 +- .../commands/test_forwards_backwards.py | 5 +- tests/apps/migrations/commands/test_new.py | 4 +- tests/apps/schema/commands/test_generate.py | 13 +- tests/base.py | 256 +++++++++++--- tests/cockroach_conf.py | 22 ++ .../foreign_key/test_on_delete_on_update.py | 4 +- tests/columns/test_array.py | 27 +- tests/columns/test_bigint.py | 4 +- tests/columns/test_db_column_name.py | 130 +++++-- tests/columns/test_jsonb.py | 27 +- tests/columns/test_m2m.py | 169 ++++++--- tests/columns/test_numeric.py | 2 +- tests/columns/test_smallint.py | 4 +- tests/columns/test_time.py | 3 + tests/columns/test_varchar.py | 4 +- tests/engine/test_extra_nodes.py | 4 +- tests/engine/test_pool.py | 13 +- tests/engine/test_transaction.py | 7 +- tests/engine/test_version_parsing.py | 4 +- tests/example_apps/mega/tables.py | 88 +++-- tests/example_apps/music/tables.py | 35 +- tests/query/test_mixins.py | 38 +- .../instance/test_get_related_readable.py | 40 ++- tests/table/instance/test_instantiate.py | 11 +- tests/table/instance/test_save.py | 62 +++- tests/table/instance/test_to_dict.py | 71 +++- tests/table/test_alter.py | 22 +- tests/table/test_batch.py | 4 +- tests/table/test_join.py | 170 ++++++--- tests/table/test_objects.py | 4 +- tests/table/test_output.py | 28 +- tests/table/test_raw.py | 51 ++- tests/table/test_repr.py | 2 +- tests/table/test_select.py | 55 ++- tests/table/test_str.py | 27 +- tests/table/test_update.py | 2 + tests/testing/test_model_builder.py | 3 + tests/utils/test_lazy_loader.py | 4 +- tests/utils/test_table_reflection.py | 4 +- 78 files changed, 2179 insertions(+), 445 deletions(-) create mode 100644 docs/src/piccolo/engines/cockroach_engine.rst create mode 100644 docs/src/piccolo/getting_started/setup_cockroach.rst create mode 100644 piccolo/engine/cockroach.py create mode 100755 scripts/test-cockroach.sh create mode 100644 tests/cockroach_conf.py diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 0bb8b58d1..285d2fcb7 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -2,7 +2,7 @@ name: Test Suite on: push: - branches: ["master"] + branches: ["master", "cockroachdb"] pull_request: branches: ["master"] @@ -132,6 +132,40 @@ jobs: uses: codecov/codecov-action@v1 if: matrix.python-version == '3.7' + cockroach: + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + matrix: + python-version: ["3.7", "3.8", "3.9", "3.10"] + cockroachdb-version: ["v22.2.0-beta.2"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements/requirements.txt + pip install -r requirements/test-requirements.txt + pip install -r requirements/extras/postgres.txt + - name: Setup CockroachDB + run: | + wget -qO- https://binaries.cockroachdb.com/cockroach-${{ matrix.cockroachdb-version }}.linux-amd64.tgz | tar xz + ./cockroach-${{ matrix.cockroachdb-version }}.linux-amd64/cockroach start-single-node --insecure --background + ./cockroach-${{ matrix.cockroachdb-version }}.linux-amd64/cockroach sql --insecure -e 'create database piccolo;' + + - name: Test with pytest, CockroachDB + run: ./scripts/test-cockroach.sh + env: + PG_HOST: localhost + PG_DATABASE: piccolo + - name: Upload coverage + uses: codecov/codecov-action@v1 + if: matrix.python-version == '3.7' + sqlite: runs-on: ubuntu-latest timeout-minutes: 30 diff --git a/docs/src/piccolo/contributing/index.rst b/docs/src/piccolo/contributing/index.rst index 7c7f0af28..52b37d72f 100644 --- a/docs/src/piccolo/contributing/index.rst +++ b/docs/src/piccolo/contributing/index.rst @@ -8,6 +8,25 @@ instructions. ------------------------------------------------------------------------------- +Running Cockroach +----------------- + +To get a local Cockroach instance running, you can use: + +.. code-block:: console + + cockroach start-single-node --insecure --store=type=mem,size=2GiB + +Make sure the test database exists: + +.. code-block:: console + + cockroach sql --insecure + >>> create database piccolo + >>> use piccolo + +------------------------------------------------------------------------------- + Get the tests running --------------------- @@ -20,6 +39,7 @@ Get the tests running * Setup Postgres, and make sure a database called ``piccolo`` exists (see ``tests/postgres_conf.py``). * Run the automated code linting/formatting tools: ``./scripts/lint.sh`` * Run the test suite with Postgres: ``./scripts/test-postgres.sh`` +* Run the test suite with Cockroach: ``./scripts/test-cockroach.sh`` * Run the test suite with Sqlite: ``./scripts/test-sqlite.sh`` ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/engines/cockroach_engine.rst b/docs/src/piccolo/engines/cockroach_engine.rst new file mode 100644 index 000000000..5dbfced80 --- /dev/null +++ b/docs/src/piccolo/engines/cockroach_engine.rst @@ -0,0 +1,87 @@ +CockroachEngine +=============== + +Configuration +------------- + +.. code-block:: python + + # piccolo_conf.py + from piccolo.engine.cockroach import CockroachEngine + + + DB = CockroachEngine(config={ + 'host': 'localhost', + 'database': 'piccolo', + 'user': 'root', + 'password': '', + 'port': '26257', + }) + +config +~~~~~~ + +The config dictionary is passed directly to the underlying database adapter, +asyncpg. See the `asyncpg docs `_ +to learn more. + +------------------------------------------------------------------------------- + +Connection pool +--------------- + +To use a connection pool, you need to first initialise it. The best place to do +this is in the startup event handler of whichever web framework you are using. + +Here's an example using Starlette. Notice that we also close the connection +pool in the shutdown event handler. + +.. code-block:: python + + from piccolo.engine import engine_finder + from starlette.applications import Starlette + + + app = Starlette() + + + @app.on_event('startup') + async def open_database_connection_pool(): + engine = engine_finder() + await engine.start_connection_pool() + + + @app.on_event('shutdown') + async def close_database_connection_pool(): + engine = engine_finder() + await engine.close_connection_pool() + +.. hint:: Using a connection pool helps with performance, since connections + are reused instead of being created for each query. + +Once a connection pool has been started, the engine will use it for making +queries. + +.. hint:: If you're running several instances of an app on the same server, + you may prefer an external connection pooler - like pgbouncer. + +Configuration +~~~~~~~~~~~~~ + +The connection pool uses the same configuration as your engine. You can also +pass in additional parameters, which are passed to the underlying database +adapter. Here's an example: + +.. code-block:: python + + # To increase the number of connections available: + await engine.start_connection_pool(max_size=20) + +------------------------------------------------------------------------------- + +Source +------ + +.. currentmodule:: piccolo.engine.cockroach + +.. autoclass:: CockroachEngine diff --git a/docs/src/piccolo/engines/index.rst b/docs/src/piccolo/engines/index.rst index c39777da7..fad00546c 100644 --- a/docs/src/piccolo/engines/index.rst +++ b/docs/src/piccolo/engines/index.rst @@ -126,3 +126,4 @@ Engine types ./sqlite_engine ./postgres_engine + ./cockroach_engine diff --git a/docs/src/piccolo/features/supported_databases.rst b/docs/src/piccolo/features/supported_databases.rst index 2b2ef74a4..4ad199ec8 100644 --- a/docs/src/piccolo/features/supported_databases.rst +++ b/docs/src/piccolo/features/supported_databases.rst @@ -8,6 +8,12 @@ will be using in production. ------------------------------------------------------------------------------- +Cockroach DB +------------ +Cockroach support is in experimental beta. + +------------------------------------------------------------------------------- + SQLite ------ SQLite support is not as complete as Postgres, but it is available - mostly diff --git a/docs/src/piccolo/getting_started/database_support.rst b/docs/src/piccolo/getting_started/database_support.rst index 0ce384ce0..3dc192888 100644 --- a/docs/src/piccolo/getting_started/database_support.rst +++ b/docs/src/piccolo/getting_started/database_support.rst @@ -6,6 +6,8 @@ Database Support `Postgres `_ is the primary database which Piccolo was designed for. +`CockroachDB `_ is in experimental beta. + Limited `SQLite `_ support is available, mostly to enable tooling like the :ref:`playground `. Postgres is the only database we recommend for use in production with Piccolo. diff --git a/docs/src/piccolo/getting_started/index.rst b/docs/src/piccolo/getting_started/index.rst index e31ce6619..373cad786 100644 --- a/docs/src/piccolo/getting_started/index.rst +++ b/docs/src/piccolo/getting_started/index.rst @@ -10,6 +10,7 @@ Getting Started ./installing_piccolo ./playground ./setup_postgres + ./setup_cockroach ./setup_sqlite ./example_schema ./sync_and_async diff --git a/docs/src/piccolo/getting_started/installing_piccolo.rst b/docs/src/piccolo/getting_started/installing_piccolo.rst index cdc695771..ce60967c2 100644 --- a/docs/src/piccolo/getting_started/installing_piccolo.rst +++ b/docs/src/piccolo/getting_started/installing_piccolo.rst @@ -23,7 +23,7 @@ Now install Piccolo, ideally inside a `virtualenv `_. + + +Versions +-------- + +We support the latest stable version. + +.. note:: + Features using ``format()`` will be available in v22.2 or higher, but we recommend using the stable version so you can upgrade automatically when it becomes generally available. + + Cockroach is designed to be a "rolling database": Upgrades are as simple as switching out to the next version of a binary (or changing a number in a ``docker-compose.yml``). This has one caveat: You cannot upgrade an "alpha" release. It is best to stay on the latest stable. + + +------------------------------------------------------------------------------- + +Creating a database +******************* + +cockroach sql +------------- + +CockroachDB comes with its own management tooling. + +.. code-block:: bash + + cd ~/wherever/you/installed/cockroachdb + cockroach sql --insecure + +Enter the following: + +.. code-block:: bash + + create database piccolo; + use piccolo; + +Management GUI +-------------- + +CockroachDB comes with its own web-based management GUI available on localhost: http://127.0.0.1:8080/ + +Beekeeper Studio +---------------- + +If you prefer a GUI, Beekeeper Studio is recommended and has an `installer available `_. + + +------------------------------------------------------------------------------- + + +Column Types +************ + +As of this writing, CockroachDB will always convert ``JSON`` to ``JSONB`` and will always report ``INTEGER`` as ``BIGINT``. + +Piccolo will automatically handle these special cases for you, but we recommend being explicit about this to prevent complications in future versions of Piccolo. + +* Use ``JSONB()`` instead of ``JSON()`` +* Use ``BigInt()`` instead of ``Integer()`` diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index 4d5b09db6..a95bc3eef 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -11,7 +11,7 @@ TEMPLATE_DIR = os.path.join(os.path.dirname(__file__), "templates/app/") SERVERS = ["uvicorn", "Hypercorn"] ROUTERS = ["starlette", "fastapi", "blacksheep", "xpresso", "starlite"] -ROUTER_VERSIONS = {"starlite": "==1.23.0"} +ROUTER_VERSIONS = {"starlite": "==1.23.0", "xpresso": "==0.43.0"} def print_instruction(message: str): diff --git a/piccolo/apps/migrations/auto/migration_manager.py b/piccolo/apps/migrations/auto/migration_manager.py index 77da21717..ed5cf0905 100644 --- a/piccolo/apps/migrations/auto/migration_manager.py +++ b/piccolo/apps/migrations/auto/migration_manager.py @@ -788,4 +788,11 @@ async def run(self, backwards=False): await self._run_drop_columns(backwards=backwards) await self._run_drop_tables(backwards=backwards) await self._run_rename_columns(backwards=backwards) + # We can remove this for cockroach when resolved. + # https://github.com/cockroachdb/cockroach/issues/49351 + # "ALTER COLUMN TYPE is not supported inside a transaction" + if engine.engine_type != "cockroach": + await self._run_alter_columns(backwards=backwards) + + if engine.engine_type == "cockroach": await self._run_alter_columns(backwards=backwards) diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index 566f5a547..1eb30f390 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -307,6 +307,12 @@ def __add__(self, value: OutputSchema) -> OutputSchema: "uuid": UUID, } +# Re-map for Cockroach compatibility. +COLUMN_TYPE_MAP_COCKROACH = { + **COLUMN_TYPE_MAP, + **{"integer": BigInt, "json": JSONB}, +} + COLUMN_DEFAULT_PARSER = { BigInt: re.compile(r"^'?(?P-?[0-9]\d*)'?(?:::bigint)?$"), Boolean: re.compile(r"^(?Ptrue|false)$"), @@ -366,11 +372,24 @@ def __add__(self, value: OutputSchema) -> OutputSchema: ForeignKey: None, } +# Re-map for Cockroach compatibility. +COLUMN_DEFAULT_PARSER_COCKROACH = { + **COLUMN_DEFAULT_PARSER, + **{BigInt: re.compile(r"^(?P-?\d+)$")}, +} + def get_column_default( - column_type: t.Type[Column], column_default: str + column_type: t.Type[Column], column_default: str, engine_type: str ) -> t.Any: - pat = COLUMN_DEFAULT_PARSER.get(column_type) + + if engine_type == "cockroach": + pat = COLUMN_DEFAULT_PARSER_COCKROACH.get(column_type) + else: + pat = COLUMN_DEFAULT_PARSER.get(column_type) + + # Strip extra, incorrect typing stuff from Cockroach. + column_default = column_default.split(":::", 1)[0] if pat is None: return None @@ -444,9 +463,8 @@ def get_column_default( "gin": IndexMethod.gin, } - # 'Indices' seems old-fashioned and obscure in this context. -async def get_indexes( +async def get_indexes( # noqa: E302 table_class: t.Type[Table], tablename: str, schema_name: str = "public" ) -> TableIndexes: """ @@ -653,7 +671,10 @@ def get_table_name(name: str, schema: str) -> str: async def create_table_class_from_db( - table_class: t.Type[Table], tablename: str, schema_name: str + table_class: t.Type[Table], + tablename: str, + schema_name: str, + engine_type: str, ) -> OutputSchema: output_schema = OutputSchema() @@ -676,7 +697,12 @@ async def create_table_class_from_db( for pg_row_meta in table_schema: data_type = pg_row_meta.data_type - column_type = COLUMN_TYPE_MAP.get(data_type, None) + + if engine_type == "cockroach": + column_type = COLUMN_TYPE_MAP_COCKROACH.get(data_type, None) + else: + column_type = COLUMN_TYPE_MAP.get(data_type, None) + column_name = pg_row_meta.column_name column_default = pg_row_meta.column_default if not column_type: @@ -699,6 +725,9 @@ async def create_table_class_from_db( kwargs["primary_key"] = True if column_type == Integer: column_type = Serial + if column_type == BigInt: + column_type = Serial + # column_type = BigSerial if constraints.is_foreign_key(column_name=column_name): fk_constraint_table = constraints.get_foreign_key_constraint_name( @@ -722,6 +751,7 @@ async def create_table_class_from_db( table_class=table_class, tablename=constraint_table.name, schema_name=constraint_table.schema, + engine_type=engine_type, ) ) referenced_table = ( @@ -753,7 +783,8 @@ async def create_table_class_from_db( kwargs["references"] = ForeignKeyPlaceholder output_schema.imports.append( - "from piccolo.columns.column_types import " + column_type.__name__ + "from piccolo.columns.column_types import " + + column_type.__name__ # type: ignore ) if column_type is Varchar: @@ -765,11 +796,13 @@ async def create_table_class_from_db( kwargs["digits"] = (precision, scale) if column_default: - default_value = get_column_default(column_type, column_default) + default_value = get_column_default( + column_type, column_default, engine_type + ) if default_value: kwargs["default"] = default_value - column = column_type(**kwargs) + column = column_type(**kwargs) # type: ignore serialised_params = serialise_params(column._meta.params) for extra_import in serialised_params.extra_imports: @@ -838,7 +871,10 @@ class Schema(Table, db=engine): ] table_coroutines = ( create_table_class_from_db( - table_class=Schema, tablename=tablename, schema_name=schema_name + table_class=Schema, + tablename=tablename, + schema_name=schema_name, + engine_type=engine.engine_type, ) for tablename in tablenames ) diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 0cd435d3a..14ac5e003 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -594,7 +594,7 @@ def ilike(self, value: str) -> Where: For SQLite, it's just proxied to a LIKE query instead. """ - if self._meta.engine_type == "postgres": + if self._meta.engine_type in ("postgres", "cockroach"): operator: t.Type[ComparisonOperator] = ILike else: colored_warning( @@ -843,7 +843,12 @@ def ddl(self) -> str: f" ON UPDATE {on_update}" ) - if self.__class__.__name__ not in ("Serial", "BigSerial"): + # Always ran for Cockroach because unique_rowid() is directly + # defined for Cockroach Serial and BigSerial. + # Postgres and SQLite will not run this for Serial and BigSerial. + if self._meta.engine_type in ( + "cockroach" + ) or self.__class__.__name__ not in ("Serial", "BigSerial"): default = self.get_default_value() sql_value = self.get_sql_value(value=default) query += f" DEFAULT {sql_value}" diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index dfadda686..49e86a92a 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -236,7 +236,7 @@ def get_querystring( if not isinstance(value, timedelta): raise ValueError("Only timedelta values can be added.") - if engine_type == "postgres": + if engine_type in ("postgres", "cockroach"): value_string = self.get_postgres_interval_string(interval=value) return QueryString( f'"{column_name}" {operator} INTERVAL {value_string}', @@ -683,6 +683,8 @@ def column_type(self): engine_type = self._meta.engine_type if engine_type == "postgres": return "BIGINT" + elif engine_type == "cockroach": + return "BIGINT" elif engine_type == "sqlite": return "INTEGER" raise Exception("Unrecognized engine type") @@ -731,6 +733,8 @@ def column_type(self): engine_type = self._meta.engine_type if engine_type == "postgres": return "SMALLINT" + elif engine_type == "cockroach": + return "SMALLINT" elif engine_type == "sqlite": return "INTEGER" raise Exception("Unrecognized engine type") @@ -770,14 +774,19 @@ def column_type(self): engine_type = self._meta.engine_type if engine_type == "postgres": return "SERIAL" + elif engine_type == "cockroach": + return "INTEGER" elif engine_type == "sqlite": return "INTEGER" raise Exception("Unrecognized engine type") def default(self): engine_type = self._meta.engine_type + if engine_type == "postgres": return DEFAULT + elif engine_type == "cockroach": + return Unquoted("unique_rowid()") elif engine_type == "sqlite": return NULL raise Exception("Unrecognized engine type") @@ -810,6 +819,8 @@ def column_type(self): engine_type = self._meta.engine_type if engine_type == "postgres": return "BIGSERIAL" + elif engine_type == "cockroach": + return "BIGINT" elif engine_type == "sqlite": return "INTEGER" raise Exception("Unrecognized engine type") @@ -1248,7 +1259,7 @@ def __init__( @property def column_type(self): engine_type = self._meta.engine_type - if engine_type == "postgres": + if engine_type in ("postgres", "cockroach"): return "INTERVAL" elif engine_type == "sqlite": # We can't use 'INTERVAL' because the type affinity in SQLite would @@ -1429,6 +1440,9 @@ class Ticket(Table): @property def column_type(self): + engine_type = self._meta.engine_type + if engine_type == "cockroach": + return "NUMERIC" # All Numeric is the same for Cockroach. if self.digits: return f"NUMERIC({self.precision}, {self.scale})" else: @@ -1818,6 +1832,9 @@ def column_type(self): column of the table being referenced. """ target_column = self._foreign_key_meta.resolved_target_column + engine_type = self._meta.engine_type + if engine_type == "cockroach": + return target_column.column_type if isinstance(target_column, Serial): return Integer().column_type else: @@ -2198,6 +2215,14 @@ def __init__( self.json_operator: t.Optional[str] = None + @property + def column_type(self): + engine_type = self._meta.engine_type + if engine_type == "cockroach": + return "JSONB" # Cockroach is always JSONB. + else: + return "JSON" + ########################################################################### # Descriptors @@ -2229,6 +2254,10 @@ class JSONB(JSON): """ + @property + def column_type(self): + return "JSONB" # Must be defined, we override column_type() in JSON() + def arrow(self, key: str) -> JSONB: """ Allows part of the JSON structure to be returned - for example, @@ -2310,7 +2339,7 @@ class Token(Table): @property def column_type(self): engine_type = self._meta.engine_type - if engine_type == "postgres": + if engine_type in ("postgres", "cockroach"): return "BYTEA" elif engine_type == "sqlite": return "BLOB" @@ -2456,7 +2485,7 @@ def __init__( @property def column_type(self): engine_type = self._meta.engine_type - if engine_type == "postgres": + if engine_type in ("postgres", "cockroach"): return f"{self.base_column.column_type}[]" elif engine_type == "sqlite": return "ARRAY" @@ -2480,9 +2509,9 @@ def __getitem__(self, value: int) -> Array: """ # noqa: E501 engine_type = self._meta.engine_type - if engine_type != "postgres": + if engine_type != "postgres" and engine_type != "cockroach": raise ValueError( - "Only Postgres supports array indexing currently." + "Only Postgres and Cockroach support array indexing." ) if isinstance(value, int): @@ -2521,7 +2550,7 @@ def any(self, value: t.Any) -> Where: """ engine_type = self._meta.engine_type - if engine_type == "postgres": + if engine_type in ("postgres", "cockroach"): return Where(column=self, value=value, operator=ArrayAny) elif engine_type == "sqlite": return self.like(f"%{value}%") @@ -2539,7 +2568,7 @@ def all(self, value: t.Any) -> Where: """ engine_type = self._meta.engine_type - if engine_type == "postgres": + if engine_type in ("postgres", "cockroach"): return Where(column=self, value=value, operator=ArrayAll) elif engine_type == "sqlite": raise ValueError("Unsupported by SQLite") @@ -2566,9 +2595,9 @@ def cat(self, value: t.List[t.Any]) -> QueryString: """ engine_type = self._meta.engine_type - if engine_type != "postgres": + if engine_type != "postgres" and engine_type != "cockroach": raise ValueError( - "Only Postgres supports array appending currently." + "Only Postgres and Cockroach support array appending." ) if not isinstance(value, list): diff --git a/piccolo/columns/defaults/date.py b/piccolo/columns/defaults/date.py index 27fa3327b..87e431390 100644 --- a/piccolo/columns/defaults/date.py +++ b/piccolo/columns/defaults/date.py @@ -35,6 +35,10 @@ def postgres(self): interval_string = self.get_postgres_interval_string(["days"]) return f"CURRENT_DATE + INTERVAL '{interval_string}'" + @property + def cockroach(self): + return self.postgres + @property def sqlite(self): interval_string = self.get_sqlite_interval_string(["days"]) @@ -51,6 +55,10 @@ class DateNow(Default): def postgres(self): return "CURRENT_DATE" + @property + def cockroach(self): + return self.postgres + @property def sqlite(self): return "CURRENT_DATE" @@ -75,6 +83,10 @@ def __init__( def postgres(self): return f"'{self.date.isoformat()}'" + @property + def cockroach(self): + return self.postgres + @property def sqlite(self): return f"'{self.date.isoformat()}'" diff --git a/piccolo/columns/defaults/interval.py b/piccolo/columns/defaults/interval.py index b133a0e8b..5d12fdda3 100644 --- a/piccolo/columns/defaults/interval.py +++ b/piccolo/columns/defaults/interval.py @@ -53,6 +53,10 @@ def postgres(self): ) return f"'{value}'" + @property + def cockroach(self): + return self.postgres + @property def sqlite(self): return self.timedelta.total_seconds() diff --git a/piccolo/columns/defaults/time.py b/piccolo/columns/defaults/time.py index fc22567d9..25535cb5d 100644 --- a/piccolo/columns/defaults/time.py +++ b/piccolo/columns/defaults/time.py @@ -20,6 +20,13 @@ def postgres(self): ) return f"CURRENT_TIME + INTERVAL '{interval_string}'" + @property + def cockroach(self): + interval_string = self.get_postgres_interval_string( + ["hours", "minutes", "seconds"] + ) + return f"CURRENT_TIME::TIMESTAMP + INTERVAL '{interval_string}'" + @property def sqlite(self): interval_string = self.get_sqlite_interval_string( @@ -41,6 +48,10 @@ class TimeNow(Default): def postgres(self): return "CURRENT_TIME" + @property + def cockroach(self): + return "CURRENT_TIME::TIMESTAMP" + @property def sqlite(self): return "CURRENT_TIME" @@ -60,6 +71,10 @@ def __init__(self, hour: int, minute: int, second: int): def postgres(self): return f"'{self.time.isoformat()}'" + @property + def cockroach(self): + return f"'{self.time.isoformat()}'::TIMESTAMP" + @property def sqlite(self): return f"'{self.time.isoformat()}'" diff --git a/piccolo/columns/defaults/timestamp.py b/piccolo/columns/defaults/timestamp.py index c1264a284..9558f4100 100644 --- a/piccolo/columns/defaults/timestamp.py +++ b/piccolo/columns/defaults/timestamp.py @@ -23,6 +23,13 @@ def postgres(self): ) return f"CURRENT_TIMESTAMP + INTERVAL '{interval_string}'" + @property + def cockroach(self): + interval_string = self.get_postgres_interval_string( + ["days", "hours", "minutes", "seconds"] + ) + return f"CURRENT_TIMESTAMP::TIMESTAMP + INTERVAL '{interval_string}'" + @property def sqlite(self): interval_string = self.get_sqlite_interval_string( @@ -44,6 +51,10 @@ class TimestampNow(Default): def postgres(self): return "current_timestamp" + @property + def cockroach(self): + return "current_timestamp::TIMESTAMP" + @property def sqlite(self): return "current_timestamp" @@ -84,6 +95,12 @@ def datetime(self): def postgres(self): return "'{}'".format(self.datetime.isoformat().replace("T", " ")) + @property + def cockroach(self): + return "'{}'::TIMESTAMP".format( + self.datetime.isoformat().replace("T", " ") + ) + @property def sqlite(self): return "'{}'".format(self.datetime.isoformat().replace("T", " ")) diff --git a/piccolo/columns/defaults/timestamptz.py b/piccolo/columns/defaults/timestamptz.py index e52c2e62a..5db6ebd54 100644 --- a/piccolo/columns/defaults/timestamptz.py +++ b/piccolo/columns/defaults/timestamptz.py @@ -8,6 +8,13 @@ class TimestamptzOffset(TimestampOffset): + @property + def cockroach(self): + interval_string = self.get_postgres_interval_string( + ["days", "hours", "minutes", "seconds"] + ) + return f"CURRENT_TIMESTAMP + INTERVAL '{interval_string}'" + def python(self): return datetime.datetime.now( tz=datetime.timezone.utc @@ -20,11 +27,19 @@ def python(self): class TimestamptzNow(TimestampNow): + @property + def cockroach(self): + return "current_timestamp" + def python(self): return datetime.datetime.now(tz=datetime.timezone.utc) class TimestamptzCustom(TimestampCustom): + @property + def cockroach(self): + return "'{}'".format(self.datetime.isoformat().replace("T", " ")) + @property def datetime(self): return datetime.datetime( diff --git a/piccolo/columns/defaults/uuid.py b/piccolo/columns/defaults/uuid.py index 44a5c76c0..176d282ec 100644 --- a/piccolo/columns/defaults/uuid.py +++ b/piccolo/columns/defaults/uuid.py @@ -10,6 +10,10 @@ class UUID4(Default): def postgres(self): return "uuid_generate_v4()" + @property + def cockroach(self): + return self.postgres + @property def sqlite(self): return "''" diff --git a/piccolo/columns/m2m.py b/piccolo/columns/m2m.py index db0d61d77..6cd5a752c 100644 --- a/piccolo/columns/m2m.py +++ b/piccolo/columns/m2m.py @@ -82,7 +82,7 @@ def get_select_string(self, engine_type: str, with_alias=True) -> str: WHERE "{m2m_table_name}"."{fk_1_name}" = "{table_1_name}"."{table_1_pk_name}" """ # noqa: E501 - if engine_type == "postgres": + if engine_type in ("postgres", "cockroach"): if self.as_list: column_name = self.columns[0]._meta.db_column_name return f""" diff --git a/piccolo/columns/readable.py b/piccolo/columns/readable.py index 630299f95..2748648d8 100644 --- a/piccolo/columns/readable.py +++ b/piccolo/columns/readable.py @@ -41,6 +41,10 @@ def sqlite_string(self) -> str: def postgres_string(self) -> str: return self._get_string(operator="FORMAT") + @property + def cockroach_string(self) -> str: + return self._get_string(operator="FORMAT") + def get_select_string(self, engine_type: str, with_alias=True) -> str: try: return getattr(self, f"{engine_type}_string") diff --git a/piccolo/engine/__init__.py b/piccolo/engine/__init__.py index 2cae3a4db..eb050f5e6 100644 --- a/piccolo/engine/__init__.py +++ b/piccolo/engine/__init__.py @@ -1,6 +1,13 @@ from .base import Engine +from .cockroach import CockroachEngine from .finder import engine_finder from .postgres import PostgresEngine from .sqlite import SQLiteEngine -__all__ = ["Engine", "PostgresEngine", "SQLiteEngine", "engine_finder"] +__all__ = [ + "Engine", + "PostgresEngine", + "SQLiteEngine", + "CockroachEngine", + "engine_finder", +] diff --git a/piccolo/engine/cockroach.py b/piccolo/engine/cockroach.py new file mode 100644 index 000000000..59386335f --- /dev/null +++ b/piccolo/engine/cockroach.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +import contextvars +import typing as t + +from piccolo.engine.exceptions import TransactionError +from piccolo.query.base import Query +from piccolo.utils.lazy_loader import LazyLoader +from piccolo.utils.warnings import Level, colored_warning + +from .postgres import Atomic as PostgresAtomic +from .postgres import PostgresEngine +from .postgres import Transaction as PostgresTransaction + +asyncpg = LazyLoader("asyncpg", globals(), "asyncpg") + +if t.TYPE_CHECKING: # pragma: no cover + from asyncpg.pool import Pool + + +############################################################################### + + +class Atomic(PostgresAtomic): + """ + This is useful if you want to build up a transaction programatically, by + adding queries to it. + + Usage:: + + transaction = engine.atomic() + transaction.add(Foo.create_table()) + + # Either: + transaction.run_sync() + await transaction.run() + + """ + + def __init__(self, engine: CockroachEngine): + self.engine = engine + self.queries: t.List[Query] = [] + super(Atomic, self).__init__(engine) + + +############################################################################### + + +class Transaction(PostgresTransaction): + """ + Used for wrapping queries in a transaction, using a context manager. + Currently it's async only. + + Usage:: + + async with engine.transaction(): + # Run some queries: + await Band.select().run() + + """ + + def __init__(self, engine: CockroachEngine): + self.engine = engine + if self.engine.transaction_connection.get(): + raise TransactionError( + "A transaction is already active - nested transactions aren't " + "currently supported." + ) + super(Transaction, self).__init__(engine) + + +############################################################################### + + +class CockroachEngine(PostgresEngine): + """ + An extension of the cockroach backend. + """ + + engine_type = "cockroach" + min_version_number = 0 # Doesn't seem to work with cockroach versioning. + + def __init__( + self, + config: t.Dict[str, t.Any], + extensions: t.Sequence[str] = (), + log_queries: bool = False, + extra_nodes: t.Dict[str, PostgresEngine] = None, + ) -> None: + if extra_nodes is None: + extra_nodes = {} + + self.config = config + self.extensions = extensions + self.log_queries = log_queries + self.extra_nodes = extra_nodes + self.pool: t.Optional[Pool] = None + database_name = config.get("database", "Unknown") + self.transaction_connection = contextvars.ContextVar( + f"pg_transaction_connection_{database_name}", default=None + ) + super( + PostgresEngine, self + ).__init__() # lgtm[py/super-not-enclosing-class] + + async def prep_database(self): + try: + await self._run_in_new_connection( + "SET CLUSTER SETTING sql.defaults.experimental_alter_column_type.enabled = true;" # noqa: E501 + ) + except asyncpg.exceptions.InsufficientPrivilegeError: + colored_warning( + "=> Unable to set up Cockroach DB " + "functionality may not behave as expected. Make sure " + "your database user has permission to set cluster options.", + level=Level.medium, + ) diff --git a/piccolo/query/base.py b/piccolo/query/base.py index dd3ae9fac..3beae8017 100644 --- a/piccolo/query/base.py +++ b/piccolo/query/base.py @@ -49,7 +49,7 @@ async def _process_results(self, results): # noqa: C901 if results: keys = results[0].keys() keys = [i.replace("$", ".") for i in keys] - if self.engine_type == "postgres": + if self.engine_type in ("postgres", "cockroach"): # asyncpg returns a special Record object. We can pass it # directly into zip without calling `values` on it. This can # save us hundreds of microseconds, depending on the number of @@ -265,6 +265,10 @@ def sqlite_querystrings(self) -> t.Sequence[QueryString]: def postgres_querystrings(self) -> t.Sequence[QueryString]: raise NotImplementedError + @property + def cockroach_querystrings(self) -> t.Sequence[QueryString]: + raise NotImplementedError + @property def default_querystrings(self) -> t.Sequence[QueryString]: raise NotImplementedError @@ -288,6 +292,11 @@ def querystrings(self) -> t.Sequence[QueryString]: return self.sqlite_querystrings except NotImplementedError: return self.default_querystrings + elif engine_type == "cockroach": + try: + return self.cockroach_querystrings + except NotImplementedError: + return self.default_querystrings else: raise Exception( f"No querystring found for the {engine_type} engine." @@ -403,6 +412,10 @@ def sqlite_ddl(self) -> t.Sequence[str]: def postgres_ddl(self) -> t.Sequence[str]: raise NotImplementedError + @property + def cockroach_ddl(self) -> t.Sequence[str]: + raise NotImplementedError + @property def default_ddl(self) -> t.Sequence[str]: raise NotImplementedError @@ -423,6 +436,11 @@ def ddl(self) -> t.Sequence[str]: return self.sqlite_ddl except NotImplementedError: return self.default_ddl + elif engine_type == "cockroach": + try: + return self.cockroach_ddl + except NotImplementedError: + return self.default_ddl else: raise Exception( f"No querystring found for the {engine_type} engine." diff --git a/piccolo/query/methods/create_index.py b/piccolo/query/methods/create_index.py index 594fd63b6..197a6dbda 100644 --- a/piccolo/query/methods/create_index.py +++ b/piccolo/query/methods/create_index.py @@ -52,6 +52,20 @@ def postgres_ddl(self) -> t.Sequence[str]: ) ] + @property + def cockroach_ddl(self) -> t.Sequence[str]: + column_names = self.column_names + index_name = self.table._get_index_name(column_names) + tablename = self.table._meta.tablename + method_name = self.method.value + column_names_str = ", ".join([f'"{i}"' for i in self.column_names]) + return [ + ( + f"{self.prefix} {index_name} ON {tablename} USING " + f"{method_name} ({column_names_str})" + ) + ] + @property def sqlite_ddl(self) -> t.Sequence[str]: column_names = self.column_names diff --git a/piccolo/query/methods/indexes.py b/piccolo/query/methods/indexes.py index 6ab4dbeae..1edbdf00b 100644 --- a/piccolo/query/methods/indexes.py +++ b/piccolo/query/methods/indexes.py @@ -21,6 +21,16 @@ def postgres_querystrings(self) -> t.Sequence[QueryString]: ) ] + @property + def cockroach_querystrings(self) -> t.Sequence[QueryString]: + return [ + QueryString( + "SELECT indexname AS name FROM pg_indexes " + "WHERE tablename = {}", + self.table._meta.tablename, + ) + ] + @property def sqlite_querystrings(self) -> t.Sequence[QueryString]: tablename = self.table._meta.tablename diff --git a/piccolo/query/methods/insert.py b/piccolo/query/methods/insert.py index 31b9cd134..b1b439626 100644 --- a/piccolo/query/methods/insert.py +++ b/piccolo/query/methods/insert.py @@ -65,7 +65,7 @@ def default_querystrings(self) -> t.Sequence[QueryString]: engine_type = self.engine_type - if engine_type == "postgres" or ( + if engine_type in ("postgres", "cockroach") or ( engine_type == "sqlite" and self.table._meta.db.get_version_sync() >= 3.35 ): diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index 1931ca3ce..82507be87 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -9,6 +9,7 @@ from piccolo.engine.base import Batch from piccolo.query.base import Query from piccolo.query.mixins import ( + AsOfDelegate, CallbackDelegate, CallbackType, LimitDelegate, @@ -139,6 +140,7 @@ class Objects(Query): __slots__ = ( "nested", + "as_of_delegate", "limit_delegate", "offset_delegate", "order_by_delegate", @@ -155,6 +157,7 @@ def __init__( **kwargs, ): super().__init__(table, **kwargs) + self.as_of_delegate = AsOfDelegate() self.limit_delegate = LimitDelegate() self.offset_delegate = OffsetDelegate() self.order_by_delegate = OrderByDelegate() @@ -180,6 +183,10 @@ def callback( self.callback_delegate.callback(callbacks, on=on) return self + def as_of(self, interval: str = "-1s") -> Objects: + self.as_of_delegate.as_of(interval) + return self + def limit(self, number: int) -> Objects: self.limit_delegate.limit(number) return self @@ -253,6 +260,7 @@ def default_querystrings(self) -> t.Sequence[QueryString]: select = Select(table=self.table) for attr in ( + "as_of_delegate", "limit_delegate", "where_delegate", "offset_delegate", diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 623230ada..08cdd46c3 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -12,6 +12,7 @@ from piccolo.engine.base import Batch from piccolo.query.base import Query from piccolo.query.mixins import ( + AsOfDelegate, CallbackDelegate, CallbackType, ColumnsDelegate, @@ -216,6 +217,7 @@ class Select(Query): __slots__ = ( "columns_list", "exclude_secrets", + "as_of_delegate", "columns_delegate", "distinct_delegate", "group_by_delegate", @@ -239,6 +241,7 @@ def __init__( super().__init__(table, **kwargs) self.exclude_secrets = exclude_secrets + self.as_of_delegate = AsOfDelegate() self.columns_delegate = ColumnsDelegate() self.distinct_delegate = DistinctDelegate() self.group_by_delegate = GroupByDelegate() @@ -269,6 +272,10 @@ def group_by(self, *columns: Column) -> Select: self.group_by_delegate.group_by(*_columns) return self + def as_of(self, interval: str = "-1s") -> Select: + self.as_of_delegate.as_of(interval) + return self + def limit(self, number: int) -> Select: self.limit_delegate.limit(number) return self @@ -392,7 +399,7 @@ async def response_handler(self, response): m2m_select, ) - elif self.engine_type == "postgres": + elif self.engine_type in ("postgres", "cockroach"): if m2m_select.as_list: # We get the data back as an array, and can just return it # unless it's JSON. @@ -608,6 +615,10 @@ def default_querystrings(self) -> t.Sequence[QueryString]: args: t.List[t.Any] = [] + if self.as_of_delegate._as_of: + query += " {}" + args.append(self.as_of_delegate._as_of.querystring) + if self.where_delegate._where: query += " WHERE {}" args.append(self.where_delegate._where.querystring) diff --git a/piccolo/query/methods/table_exists.py b/piccolo/query/methods/table_exists.py index 836b6e54a..bf9afb08a 100644 --- a/piccolo/query/methods/table_exists.py +++ b/piccolo/query/methods/table_exists.py @@ -30,3 +30,12 @@ def postgres_querystrings(self) -> t.Sequence[QueryString]: f"table_name = '{self.table._meta.tablename}')" ) ] + + @property + def cockroach_querystrings(self) -> t.Sequence[QueryString]: + return [ + QueryString( + "SELECT EXISTS(SELECT * FROM information_schema.tables WHERE " + f"table_name = '{self.table._meta.tablename}')" + ) + ] diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index e65ac8630..ee36f1f80 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -37,6 +37,24 @@ def copy(self) -> Limit: return self.__class__(number=self.number) +@dataclass +class AsOf: + __slots__ = ("interval",) + + interval: str + + def __post_init__(self): + if type(self.interval) != str: + raise TypeError("As Of must be a string. Example: '-1s'") + + @property + def querystring(self) -> QueryString: + return QueryString(f" AS OF SYSTEM TIME '{self.interval}'") + + def __str__(self) -> str: + return self.querystring.__str__() + + @dataclass class Offset: __slots__ = ("number",) @@ -193,6 +211,19 @@ def copy(self) -> LimitDelegate: return self.__class__(_limit=_limit, _first=self._first) +@dataclass +class AsOfDelegate: + """ + Time travel queries using "As Of" syntax. + Currently supports Cockroach using AS OF SYSTEM TIME. + """ + + _as_of: t.Optional[AsOf] = None + + def as_of(self, interval: str = "-1s"): + self._as_of = AsOf(interval) + + @dataclass class DistinctDelegate: diff --git a/piccolo/querystring.py b/piccolo/querystring.py index 1bc8ee26b..7470deb15 100644 --- a/piccolo/querystring.py +++ b/piccolo/querystring.py @@ -173,7 +173,7 @@ def compile_string( _, bundled, combined_args = self.bundle( start_index=1, bundled=[], combined_args=[] ) - if engine_type == "postgres": + if engine_type in ("postgres", "cockroach"): string = "".join( fragment.prefix + ("" if fragment.no_arg else f"${fragment.index}") diff --git a/piccolo_conf.py b/piccolo_conf.py index a020d97b4..6ece2f685 100644 --- a/piccolo_conf.py +++ b/piccolo_conf.py @@ -11,7 +11,6 @@ from piccolo.conf.apps import AppRegistry from piccolo.engine.postgres import PostgresEngine - DB = PostgresEngine(config={}) diff --git a/scripts/test-cockroach.sh b/scripts/test-cockroach.sh new file mode 100755 index 000000000..1c944aeae --- /dev/null +++ b/scripts/test-cockroach.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# To run all in a folder tests/ +# To run all in a file tests/test_foo.py +# To run all in a class tests/test_foo.py::TestFoo +# To run a single test tests/test_foo.py::TestFoo::test_foo + +export PICCOLO_CONF="tests.cockroach_conf" +python3 -m pytest \ + --cov=piccolo \ + --cov-report=xml \ + --cov-report=html \ + --cov-fail-under=80 \ + -m "not integration" \ + -s $@ diff --git a/tests/apps/fixtures/commands/test_dump_load.py b/tests/apps/fixtures/commands/test_dump_load.py index 0c27bdcda..253958a59 100644 --- a/tests/apps/fixtures/commands/test_dump_load.py +++ b/tests/apps/fixtures/commands/test_dump_load.py @@ -9,6 +9,7 @@ ) from piccolo.apps.fixtures.commands.load import load_json_string from piccolo.utils.sync import run_sync +from tests.base import engines_only from tests.example_apps.mega.tables import MegaTable, SmallTable @@ -59,6 +60,7 @@ def insert_row(self): ) mega_table.save().run_sync() + @engines_only("postgres", "sqlite") def test_dump_load(self): """ Make sure we can dump some rows into a JSON fixture, then load them @@ -135,3 +137,86 @@ def test_dump_load(self): "not_null_col": "hello", }, ) + + @engines_only("cockroach") + def test_dump_load_alt(self): + """ + Make sure we can dump some rows into a JSON fixture, then load them + back into the database. + """ + self.insert_row() + + json_string = run_sync( + dump_to_json_string( + fixture_configs=[ + FixtureConfig( + app_name="mega", + table_class_names=["SmallTable", "MegaTable"], + ) + ] + ) + ) + + # We need to clear the data out now, otherwise when loading the data + # back in, there will be a constraint errors over clashing primary + # keys. + SmallTable.delete(force=True).run_sync() + MegaTable.delete(force=True).run_sync() + + run_sync(load_json_string(json_string)) + + result = SmallTable.select().run_sync()[0] + result.pop("id") + + self.assertDictEqual( + result, + {"varchar_col": "Test"}, + ) + + mega_table_data = MegaTable.select().run_sync() + + # Real numbers don't have perfect precision when coming back from the + # database, so we need to round them to be able to compare them. + mega_table_data[0]["real_col"] = round( + mega_table_data[0]["real_col"], 1 + ) + + # Remove white space from the JSON values + for col_name in ("json_col", "jsonb_col"): + mega_table_data[0][col_name] = mega_table_data[0][ + col_name + ].replace(" ", "") + + self.assertTrue(len(mega_table_data) == 1) + + mega_table_data = mega_table_data[0] + mega_table_data.pop("id") + mega_table_data.pop("foreignkey_col") + + self.assertDictEqual( + mega_table_data, + { + "bigint_col": 1, + "boolean_col": True, + "bytea_col": b"hello", + "date_col": datetime.date(2021, 1, 1), + "integer_col": 1, + "interval_col": datetime.timedelta(seconds=10), + "json_col": '{"a":1}', + "jsonb_col": '{"a":1}', + "numeric_col": decimal.Decimal("1.1"), + "real_col": 1.1, + "double_precision_col": 1.344, + "smallint_col": 1, + "text_col": "hello", + "timestamp_col": datetime.datetime(2021, 1, 1, 0, 0), + "timestamptz_col": datetime.datetime( + 2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc + ), + "uuid_col": uuid.UUID("12783854-c012-4c15-8183-8eecb46f2c4e"), + "varchar_col": "hello", + "unique_col": "hello", + "null_col": None, + "not_null_col": "hello", + }, + ) diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index 5b6c58a6d..23387c192 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -48,7 +48,7 @@ from piccolo.conf.apps import AppConfig from piccolo.table import Table, create_table_class, drop_db_tables_sync from piccolo.utils.sync import run_sync -from tests.base import DBTestCase, postgres_only +from tests.base import DBTestCase, engines_only, engines_skip if t.TYPE_CHECKING: from piccolo.columns.base import Column @@ -169,7 +169,7 @@ def _test_migrations( ) -@postgres_only +@engines_only("postgres", "cockroach") class TestMigrations(MigrationTestCase): def setUp(self): pass @@ -190,8 +190,7 @@ def table(self, column: Column): class_name="MyTable", class_members={"my_column": column} ) - ########################################################################### - + @engines_skip("cockroach") def test_varchar_column(self): self._test_migrations( table_snapshots=[ @@ -211,7 +210,8 @@ def test_varchar_column(self): [ x.data_type == "character varying", x.is_nullable == "NO", - x.column_default == "''::character varying", + x.column_default + in ("''::character varying", "'':::STRING"), ] ), ) @@ -234,7 +234,7 @@ def test_text_column(self): [ x.data_type == "text", x.is_nullable == "NO", - x.column_default == "''::text", + x.column_default in ("''::text", "'':::STRING"), ] ), ) @@ -255,9 +255,9 @@ def test_integer_column(self): ], test_function=lambda x: all( [ - x.data_type == "integer", + x.data_type in ("integer", "bigint"), # Cockroach DB. x.is_nullable == "NO", - x.column_default == "0", + x.column_default in ("0", "0:::INT8"), # Cockroach DB. ] ), ) @@ -279,7 +279,7 @@ def test_real_column(self): [ x.data_type == "real", x.is_nullable == "NO", - x.column_default == "0.0", + x.column_default in ("0.0", "0.0:::FLOAT8"), ] ), ) @@ -301,7 +301,7 @@ def test_double_precision_column(self): [ x.data_type == "double precision", x.is_nullable == "NO", - x.column_default == "0.0", + x.column_default in ("0.0", "0.0:::FLOAT8"), ] ), ) @@ -324,7 +324,7 @@ def test_smallint_column(self): [ x.data_type == "smallint", x.is_nullable == "NO", - x.column_default == "0", + x.column_default in ("0", "0:::INT8"), # Cockroach DB. ] ), ) @@ -347,7 +347,7 @@ def test_bigint_column(self): [ x.data_type == "bigint", x.is_nullable == "NO", - x.column_default == "0", + x.column_default in ("0", "0:::INT8"), # Cockroach DB. ] ), ) @@ -403,11 +403,17 @@ def test_timestamp_column(self): [ x.data_type == "timestamp without time zone", x.is_nullable == "NO", - x.column_default in ("now()", "CURRENT_TIMESTAMP"), + x.column_default + in ( + "now()", + "CURRENT_TIMESTAMP", + "current_timestamp():::TIMESTAMPTZ::TIMESTAMP", + ), ] ), ) + @engines_skip("cockroach") def test_time_column(self): self._test_migrations( table_snapshots=[ @@ -453,7 +459,11 @@ def test_date_column(self): x.data_type == "date", x.is_nullable == "NO", x.column_default - in ("('now'::text)::date", "CURRENT_DATE"), + in ( + "('now'::text)::date", + "CURRENT_DATE", + "current_date()", + ), ] ), ) @@ -476,7 +486,8 @@ def test_interval_column(self): [ x.data_type == "interval", x.is_nullable == "NO", - x.column_default == "'00:00:00'::interval", + x.column_default + in ("'00:00:00'::interval", "'00:00:00':::INTERVAL"), ] ), ) @@ -504,6 +515,7 @@ def test_boolean_column(self): ), ) + @engines_skip("cockroach") def test_numeric_column(self): self._test_migrations( table_snapshots=[ @@ -529,6 +541,7 @@ def test_numeric_column(self): ), ) + @engines_skip("cockroach") def test_decimal_column(self): self._test_migrations( table_snapshots=[ @@ -554,7 +567,11 @@ def test_decimal_column(self): ), ) + @engines_skip("cockroach") def test_array_column_integer(self): + """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/35730 "column my_column is of type int[] and thus is not indexable" + """ # noqa: E501 self._test_migrations( table_snapshots=[ [self.table(column)] @@ -579,7 +596,11 @@ def test_array_column_integer(self): ), ) + @engines_skip("cockroach") def test_array_column_varchar(self): + """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/35730 "column my_column is of type varchar[] and thus is not indexable" + """ # noqa: E501 self._test_migrations( table_snapshots=[ [self.table(column)] @@ -599,7 +620,8 @@ def test_array_column_varchar(self): [ x.data_type == "ARRAY", x.is_nullable == "NO", - x.column_default == "'{}'::character varying[]", + x.column_default + in ("'{}'::character varying[]", "'':::STRING"), ] ), ) @@ -624,7 +646,11 @@ def test_array_column_bigint(self): # We deliberately don't test setting JSON or JSONB columns as indexes, as # we know it'll fail. + @engines_skip("cockroach") def test_json_column(self): + """ + Cockroach sees all json as jsonb, so we can skip this. + """ self._test_migrations( table_snapshots=[ [self.table(column)] @@ -663,7 +689,7 @@ def test_jsonb_column(self): [ x.data_type == "jsonb", x.is_nullable == "NO", - x.column_default == "'{}'::jsonb", + x.column_default in ("'{}'::jsonb", "'{}':::JSONB"), ] ), ) @@ -685,7 +711,8 @@ def test_db_column_name(self): [ x.data_type == "character varying", x.is_nullable == "NO", - x.column_default == "''::character varying", + x.column_default + in ("''::character varying", "'':::STRING"), ] ), ) @@ -706,7 +733,8 @@ def test_db_column_name_initial(self): [ x.data_type == "character varying", x.is_nullable == "NO", - x.column_default == "''::character varying", + x.column_default + in ("''::character varying", "'':::STRING"), ] ), ) @@ -731,7 +759,11 @@ def test_column_type_conversion_string(self): ] ) + @engines_skip("cockroach") def test_column_type_conversion_integer(self): + """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/49351 "ALTER COLUMN TYPE is not supported inside a transaction" + """ # noqa: E501 self._test_migrations( table_snapshots=[ [self.table(column)] @@ -745,7 +777,11 @@ def test_column_type_conversion_integer(self): ] ) + @engines_skip("cockroach") def test_column_type_conversion_string_to_integer(self): + """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/49351 "ALTER COLUMN TYPE is not supported inside a transaction" + """ # noqa: E501 self._test_migrations( table_snapshots=[ [self.table(column)] @@ -757,7 +793,11 @@ def test_column_type_conversion_string_to_integer(self): ] ) + @engines_skip("cockroach") def test_column_type_conversion_float_decimal(self): + """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/49351 "ALTER COLUMN TYPE is not supported inside a transaction" + """ # noqa: E501 self._test_migrations( table_snapshots=[ [self.table(column)] @@ -836,7 +876,7 @@ class GenreToBand(Table): genre = ForeignKey(Genre) -@postgres_only +@engines_only("postgres", "cockroach") class TestM2MMigrations(MigrationTestCase): def setUp(self): pass @@ -860,7 +900,7 @@ def test_m2m(self): ############################################################################### -@postgres_only +@engines_only("postgres", "cockroach") class TestForeignKeys(MigrationTestCase): def setUp(self): class TableA(Table): @@ -901,7 +941,7 @@ def test_foreign_keys(self): self.assertTrue(table_class.table_exists().run_sync()) -@postgres_only +@engines_only("postgres", "cockroach") class TestTargetColumn(MigrationTestCase): def setUp(self): class TableA(Table): @@ -945,7 +985,7 @@ def test_target_column(self): self.assertTrue(response[0]["exists"]) -@postgres_only +@engines_only("postgres", "cockroach") class TestTargetColumnString(MigrationTestCase): def setUp(self): class TableA(Table): diff --git a/tests/apps/migrations/auto/test_migration_manager.py b/tests/apps/migrations/auto/test_migration_manager.py index c416ae33f..bfb771452 100644 --- a/tests/apps/migrations/auto/test_migration_manager.py +++ b/tests/apps/migrations/auto/test_migration_manager.py @@ -12,7 +12,7 @@ from piccolo.conf.apps import AppConfig from piccolo.table import Table, sort_table_classes from piccolo.utils.lazy_loader import LazyLoader -from tests.base import AsyncMock, DBTestCase, postgres_only +from tests.base import AsyncMock, DBTestCase, engine_is, engines_only from tests.example_apps.music.tables import Band, Concert, Manager, Venue asyncpg = LazyLoader("asyncpg", globals(), "asyncpg") @@ -112,7 +112,7 @@ class TableE(Table): class TestMigrationManager(DBTestCase): - @postgres_only + @engines_only("postgres", "cockroach") def test_rename_column(self): """ Test running a MigrationManager which contains a column rename @@ -207,7 +207,7 @@ async def run_back(): with self.assertRaises(HasRunBackwards): asyncio.run(manager.run(backwards=True)) - @postgres_only + @engines_only("postgres", "cockroach") @patch.object(BaseMigrationManager, "get_app_config") def test_add_table(self, get_app_config: MagicMock): """ @@ -225,11 +225,21 @@ def test_add_table(self, get_app_config: MagicMock): ) asyncio.run(manager.run()) - self.run_sync("INSERT INTO musician VALUES (default, 'Bob Jones');") - response = self.run_sync("SELECT * FROM musician;") - - self.assertEqual(response, [{"id": 1, "name": "Bob Jones"}]) + if engine_is("postgres"): + self.run_sync( + "INSERT INTO musician VALUES (default, 'Bob Jones');" + ) + response = self.run_sync("SELECT * FROM musician;") + self.assertEqual(response, [{"id": 1, "name": "Bob Jones"}]) + if engine_is("cockroach"): + id = self.run_sync( + "INSERT INTO musician VALUES (default, 'Bob Jones') RETURNING id;" # noqa: E501 + ) + response = self.run_sync("SELECT * FROM musician;") + self.assertEqual( + response, [{"id": id[0]["id"], "name": "Bob Jones"}] + ) # Reverse get_app_config.return_value = AppConfig( @@ -243,13 +253,20 @@ def test_add_table(self, get_app_config: MagicMock): manager.preview = True with patch("sys.stdout", new=StringIO()) as fake_out: asyncio.run(manager.run()) - self.assertEqual( - fake_out.getvalue(), - """ - [preview forwards]... \n CREATE TABLE musician ("id" SERIAL PRIMARY KEY NOT NULL, "name" VARCHAR(255) NOT NULL DEFAULT '');\n""", # noqa: E501 - ) + + if engine_is("postgres"): + self.assertEqual( + fake_out.getvalue(), + """ - [preview forwards]... \n CREATE TABLE musician ("id" SERIAL PRIMARY KEY NOT NULL, "name" VARCHAR(255) NOT NULL DEFAULT '');\n""", # noqa: E501 + ) + if engine_is("cockroach"): + self.assertEqual( + fake_out.getvalue(), + """ - [preview forwards]... \n CREATE TABLE musician ("id" INTEGER PRIMARY KEY NOT NULL DEFAULT unique_rowid(), "name" VARCHAR(255) NOT NULL DEFAULT '');\n""", # noqa: E501 + ) self.assertEqual(self.table_exists("musician"), False) - @postgres_only + @engines_only("postgres", "cockroach") def test_add_column(self): """ Test adding a column to a MigrationManager. @@ -273,19 +290,35 @@ def test_add_column(self): ) asyncio.run(manager.run()) - self.run_sync( - "INSERT INTO manager VALUES (default, 'Dave', 'dave@me.com');" - ) + if engine_is("postgres"): + self.run_sync( + "INSERT INTO manager VALUES (default, 'Dave', 'dave@me.com');" + ) + response = self.run_sync("SELECT * FROM manager;") + self.assertEqual( + response, [{"id": 1, "name": "Dave", "email": "dave@me.com"}] + ) - response = self.run_sync("SELECT * FROM manager;") - self.assertEqual( - response, [{"id": 1, "name": "Dave", "email": "dave@me.com"}] - ) + # Reverse + asyncio.run(manager.run(backwards=True)) + response = self.run_sync("SELECT * FROM manager;") + self.assertEqual(response, [{"id": 1, "name": "Dave"}]) - # Reverse - asyncio.run(manager.run(backwards=True)) - response = self.run_sync("SELECT * FROM manager;") - self.assertEqual(response, [{"id": 1, "name": "Dave"}]) + id = 0 + if engine_is("cockroach"): + id = self.run_sync( + "INSERT INTO manager VALUES (default, 'Dave', 'dave@me.com') RETURNING id;" # noqa: E501 + ) + response = self.run_sync("SELECT * FROM manager;") + self.assertEqual( + response, + [{"id": id[0]["id"], "name": "Dave", "email": "dave@me.com"}], + ) + + # Reverse + asyncio.run(manager.run(backwards=True)) + response = self.run_sync("SELECT * FROM manager;") + self.assertEqual(response, [{"id": id[0]["id"], "name": "Dave"}]) # Preview manager.preview = True @@ -295,10 +328,14 @@ def test_add_column(self): fake_out.getvalue(), """ - [preview forwards]... \n ALTER TABLE manager ADD COLUMN "email" VARCHAR(100) UNIQUE DEFAULT '';\n""", # noqa: E501 ) + response = self.run_sync("SELECT * FROM manager;") - self.assertEqual(response, [{"id": 1, "name": "Dave"}]) + if engine_is("postgres"): + self.assertEqual(response, [{"id": 1, "name": "Dave"}]) + if engine_is("cockroach"): + self.assertEqual(response, [{"id": id[0]["id"], "name": "Dave"}]) - @postgres_only + @engines_only("postgres", "cockroach") def test_add_column_with_index(self): """ Test adding a column with an index to a MigrationManager. @@ -342,7 +379,7 @@ def test_add_column_with_index(self): ) self.assertTrue(index_name not in Manager.indexes().run_sync()) - @postgres_only + @engines_only("postgres") def test_add_foreign_key_self_column(self): """ Test adding a ForeignKey column to a MigrationManager, with a @@ -389,7 +426,62 @@ def test_add_foreign_key_self_column(self): [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Dave"}], ) - @postgres_only + @engines_only("cockroach") + def test_add_foreign_key_self_column_alt(self): + """ + Test adding a ForeignKey column to a MigrationManager, with a + references argument of 'self'. + """ + manager = MigrationManager() + manager.add_column( + table_class_name="Manager", + tablename="manager", + column_name="advisor", + column_class=ForeignKey, + column_class_name="ForeignKey", + params={ + "references": "self", + "on_delete": OnDelete.cascade, + "on_update": OnUpdate.cascade, + "default": None, + "null": True, + "primary": False, + "key": False, + "unique": False, + "index": False, + }, + ) + asyncio.run(manager.run()) + + id = self.run_sync( + "INSERT INTO manager VALUES (default, 'Alice', null) RETURNING id;" + ) + id2 = Manager.raw( + "INSERT INTO manager VALUES (default, 'Dave', {}) RETURNING id;", + id[0]["id"], + ).run_sync() + + response = self.run_sync("SELECT * FROM manager;") + self.assertEqual( + response, + [ + {"id": id[0]["id"], "name": "Alice", "advisor": None}, + {"id": id2[0]["id"], "name": "Dave", "advisor": id[0]["id"]}, + ], + ) + + # Reverse + asyncio.run(manager.run(backwards=True)) + response = self.run_sync("SELECT * FROM manager;") + self.assertEqual( + response, + [ + {"id": id[0]["id"], "name": "Alice"}, + {"id": id2[0]["id"], "name": "Dave"}, + ], + ) + + @engines_only("postgres", "cockroach") def test_add_non_nullable_column(self): """ Test adding a non nullable column to a MigrationManager. @@ -417,7 +509,7 @@ def test_add_non_nullable_column(self): ) asyncio.run(manager.run()) - @postgres_only + @engines_only("postgres", "cockroach") @patch.object( BaseMigrationManager, "get_migration_managers", new_callable=AsyncMock ) @@ -438,9 +530,20 @@ def test_drop_column( ) asyncio.run(manager_1.run()) - self.run_sync("INSERT INTO musician VALUES (default, 'Dave');") - response = self.run_sync("SELECT * FROM musician;") - self.assertEqual(response, [{"id": 1, "name": "Dave"}]) + if engine_is("postgres"): + self.run_sync("INSERT INTO musician VALUES (default, 'Dave');") + response = self.run_sync("SELECT * FROM musician;") + self.assertEqual(response, [{"id": 1, "name": "Dave"}]) + + id = 0 + if engine_is("cockroach"): + id = self.run_sync( + "INSERT INTO musician VALUES (default, 'Dave') RETURNING id;" + ) + response = self.run_sync("SELECT * FROM musician;") + self.assertEqual( + response, [{"id": id[0]["id"], "name": "Dave"}] # type: ignore + ) manager_2 = MigrationManager() manager_2.drop_column( @@ -450,8 +553,13 @@ def test_drop_column( ) asyncio.run(manager_2.run()) - response = self.run_sync("SELECT * FROM musician;") - self.assertEqual(response, [{"id": 1}]) + if engine_is("postgres"): + response = self.run_sync("SELECT * FROM musician;") + self.assertEqual(response, [{"id": 1}]) + + if engine_is("cockroach"): + response = self.run_sync("SELECT * FROM musician;") + self.assertEqual(response, [{"id": id[0]["id"]}]) # type: ignore # Reverse get_migration_managers.return_value = [manager_1] @@ -459,9 +567,15 @@ def test_drop_column( get_app_config.return_value = app_config asyncio.run(manager_2.run(backwards=True)) response = self.run_sync("SELECT * FROM musician;") - self.assertEqual(response, [{"id": 1, "name": ""}]) + if engine_is("postgres"): + self.assertEqual(response, [{"id": 1, "name": ""}]) + + if engine_is("cockroach"): + self.assertEqual( + response, [{"id": id[0]["id"], "name": ""}] # type: ignore + ) - @postgres_only + @engines_only("postgres", "cockroach") def test_rename_table(self): """ Test renaming a table with MigrationManager. @@ -477,21 +591,37 @@ def test_rename_table(self): asyncio.run(manager.run()) - self.run_sync("INSERT INTO director VALUES (default, 'Dave');") + if engine_is("postgres"): + self.run_sync("INSERT INTO director VALUES (default, 'Dave');") - response = self.run_sync("SELECT * FROM director;") - self.assertEqual(response, [{"id": 1, "name": "Dave"}]) + response = self.run_sync("SELECT * FROM director;") + self.assertEqual(response, [{"id": 1, "name": "Dave"}]) - # Reverse - asyncio.run(manager.run(backwards=True)) - response = self.run_sync("SELECT * FROM manager;") - self.assertEqual(response, [{"id": 1, "name": "Dave"}]) + # Reverse + asyncio.run(manager.run(backwards=True)) + response = self.run_sync("SELECT * FROM manager;") + self.assertEqual(response, [{"id": 1, "name": "Dave"}]) - @postgres_only + if engine_is("cockroach"): + id = 0 + id = self.run_sync( + "INSERT INTO director VALUES (default, 'Dave') RETURNING id;" + ) + + response = self.run_sync("SELECT * FROM director;") + self.assertEqual(response, [{"id": id[0]["id"], "name": "Dave"}]) + + # Reverse + asyncio.run(manager.run(backwards=True)) + response = self.run_sync("SELECT * FROM manager;") + self.assertEqual(response, [{"id": id[0]["id"], "name": "Dave"}]) + + @engines_only("postgres") def test_alter_column_unique(self): """ Test altering a column uniqueness with MigrationManager. - """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/42840 "unimplemented: cannot drop UNIQUE constraint "manager_name_key" using ALTER TABLE DROP CONSTRAINT, use DROP INDEX CASCADE instead" + """ # noqa: E501 manager = MigrationManager() manager.alter_column( @@ -518,7 +648,7 @@ def test_alter_column_unique(self): response = self.run_sync("SELECT name FROM manager;") self.assertEqual(response, [{"name": "Dave"}, {"name": "Dave"}]) - @postgres_only + @engines_only("postgres", "cockroach") def test_alter_column_set_null(self): """ Test altering whether a column is nullable with MigrationManager. @@ -566,11 +696,12 @@ def _get_column_default(self, tablename="manager", column_name="name"): f"AND column_name = '{column_name}';" ) - @postgres_only + @engines_only("postgres") def test_alter_column_digits(self): """ Test altering a column digits with MigrationManager. - """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/49351 "ALTER COLUMN TYPE is not supported inside a transaction" + """ # noqa: E501 manager = MigrationManager() manager.alter_column( @@ -593,7 +724,7 @@ def test_alter_column_digits(self): [{"numeric_precision": 5, "numeric_scale": 2}], ) - @postgres_only + @engines_only("postgres") def test_alter_column_set_default(self): """ Test altering a column default with MigrationManager. @@ -620,7 +751,34 @@ def test_alter_column_set_default(self): [{"column_default": "''::character varying"}], ) - @postgres_only + @engines_only("cockroach") + def test_alter_column_set_default_alt(self): + """ + Test altering a column default with MigrationManager. + """ + manager = MigrationManager() + + manager.alter_column( + table_class_name="Manager", + tablename="manager", + column_name="name", + params={"default": "Unknown"}, + old_params={"default": ""}, + ) + + asyncio.run(manager.run()) + self.assertEqual( + self._get_column_default(), + [{"column_default": "'Unknown':::STRING"}], + ) + + asyncio.run(manager.run(backwards=True)) + self.assertEqual( + self._get_column_default(), + [{"column_default": "'':::STRING"}], + ) + + @engines_only("postgres") def test_alter_column_drop_default(self): """ Test setting a column default to None with MigrationManager. @@ -682,7 +840,69 @@ def test_alter_column_drop_default(self): [{"column_default": None}], ) - @postgres_only + @engines_only("cockroach") + def test_alter_column_drop_default_alt(self): + """ + Test setting a column default to None with MigrationManager. + """ + # Make sure it has a non-null default to start with. + manager_1 = MigrationManager() + manager_1.alter_column( + table_class_name="Manager", + tablename="manager", + column_name="name", + params={"default": "Mr Manager"}, + old_params={"default": None}, + ) + asyncio.run(manager_1.run()) + self.assertEqual( + self._get_column_default(), + [{"column_default": "'Mr Manager':::STRING"}], + ) + + # Drop the default. + manager_2 = MigrationManager() + manager_2.alter_column( + table_class_name="Manager", + tablename="manager", + column_name="name", + params={"default": None}, + old_params={"default": "Mr Manager"}, + ) + asyncio.run(manager_2.run()) + self.assertEqual( + self._get_column_default(), + [{"column_default": None}], + ) + + # And add it back once more to be sure. + manager_3 = manager_1 + asyncio.run(manager_3.run()) + self.assertEqual( + self._get_column_default(), + [{"column_default": "'Mr Manager':::STRING"}], + ) + + # Run them all backwards + asyncio.run(manager_3.run(backwards=True)) + self.assertEqual( + self._get_column_default(), + [{"column_default": None}], + ) + + asyncio.run(manager_2.run(backwards=True)) + self.assertEqual( + self._get_column_default(), + [{"column_default": "'Mr Manager':::STRING"}], + ) + + asyncio.run(manager_1.run(backwards=True)) + self.assertEqual( + self._get_column_default(), + [{"column_default": None}], + ) + + @engines_only("postgres", "cockroach") def test_alter_column_add_index(self): """ Test altering a column to add an index with MigrationManager. @@ -708,7 +928,7 @@ def test_alter_column_add_index(self): not in Manager.indexes().run_sync() ) - @postgres_only + @engines_only("postgres", "cockroach") def test_alter_column_set_type(self): """ Test altering a column to change it's type with MigrationManager. @@ -737,11 +957,12 @@ def test_alter_column_set_type(self): ) self.assertEqual(column_type_str, "CHARACTER VARYING") - @postgres_only + @engines_only("postgres") def test_alter_column_set_length(self): """ Test altering a Varchar column's length with MigrationManager. - """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/49351 "ALTER COLUMN TYPE is not supported inside a transaction" + """ # noqa: E501 manager = MigrationManager() manager.alter_column( @@ -770,7 +991,7 @@ def test_alter_column_set_length(self): 200, ) - @postgres_only + @engines_only("postgres", "cockroach") @patch.object( BaseMigrationManager, "get_migration_managers", new_callable=AsyncMock ) diff --git a/tests/apps/migrations/auto/test_serialisation.py b/tests/apps/migrations/auto/test_serialisation.py index 1b813ba66..688bdf09e 100644 --- a/tests/apps/migrations/auto/test_serialisation.py +++ b/tests/apps/migrations/auto/test_serialisation.py @@ -19,6 +19,7 @@ from piccolo.columns.column_types import Varchar from piccolo.columns.defaults import UUID4, DateNow, TimeNow, TimestampNow from piccolo.columns.reference import LazyTableReference +from tests.base import engine_is class TestUniqueGlobalNamesMeta: @@ -241,15 +242,28 @@ def test_lazy_table_reference(self): ) self.assertTrue(len(serialised.extra_definitions) == 1) - self.assertEqual( - serialised.extra_definitions[0].__str__(), - ( - 'class Manager(Table, tablename="manager"): ' - "id = Serial(null=False, primary_key=True, unique=False, " - "index=False, index_method=IndexMethod.btree, " - "choices=None, db_column_name='id', secret=False)" - ), - ) + + if engine_is("postgres"): + self.assertEqual( + serialised.extra_definitions[0].__str__(), + ( + 'class Manager(Table, tablename="manager"): ' + "id = Serial(null=False, primary_key=True, unique=False, " # noqa: E501 + "index=False, index_method=IndexMethod.btree, " + "choices=None, db_column_name='id', secret=False)" + ), + ) + + if engine_is("cockroach"): + self.assertEqual( + serialised.extra_definitions[0].__str__(), + ( + 'class Manager(Table, tablename="manager"): ' + "id = Serial(null=False, primary_key=True, unique=False, " # noqa: E501 + "index=False, index_method=IndexMethod.btree, " + "choices=None, db_column_name='id', secret=False)" + ), + ) def test_function(self): serialised = serialise_params(params={"default": example_function}) diff --git a/tests/apps/migrations/commands/test_forwards_backwards.py b/tests/apps/migrations/commands/test_forwards_backwards.py index 427fd1901..d70d7d506 100644 --- a/tests/apps/migrations/commands/test_forwards_backwards.py +++ b/tests/apps/migrations/commands/test_forwards_backwards.py @@ -9,7 +9,7 @@ from piccolo.apps.migrations.commands.forwards import forwards from piccolo.apps.migrations.tables import Migration from piccolo.utils.sync import run_sync -from tests.base import postgres_only +from tests.base import engines_only from tests.example_apps.music.tables import ( Band, Concert, @@ -36,7 +36,7 @@ ] -@postgres_only +@engines_only("postgres", "cockroach") class TestForwardsBackwards(TestCase): """ Test the forwards and backwards migration commands. @@ -187,6 +187,7 @@ def test_forwards_no_migrations(self, print_: MagicMock): print_.mock_calls[-1] == call("🏁 No migrations need to be run") ) + @engines_only("postgres") def test_forwards_fake(self): """ Test running the migrations if they've already run. diff --git a/tests/apps/migrations/commands/test_new.py b/tests/apps/migrations/commands/test_new.py index 9917eda22..bb802b1df 100644 --- a/tests/apps/migrations/commands/test_new.py +++ b/tests/apps/migrations/commands/test_new.py @@ -11,7 +11,7 @@ ) from piccolo.conf.apps import AppConfig from piccolo.utils.sync import run_sync -from tests.base import postgres_only +from tests.base import engines_only from tests.example_apps.music.tables import Manager @@ -42,7 +42,7 @@ def test_create_new_migration(self): self.assertTrue(len(migration_modules.keys()) == 1) - @postgres_only + @engines_only("postgres") @patch("piccolo.apps.migrations.commands.new.print") def test_new_command(self, print_: MagicMock): """ diff --git a/tests/apps/schema/commands/test_generate.py b/tests/apps/schema/commands/test_generate.py index 920436ce9..fc350405c 100644 --- a/tests/apps/schema/commands/test_generate.py +++ b/tests/apps/schema/commands/test_generate.py @@ -23,11 +23,11 @@ from piccolo.engine import Engine, engine_finder from piccolo.table import Table from piccolo.utils.sync import run_sync -from tests.base import AsyncMock, postgres_only +from tests.base import AsyncMock, engines_only, engines_skip from tests.example_apps.mega.tables import MegaTable, SmallTable -@postgres_only +@engines_only("postgres", "cockroach") class TestGenerate(TestCase): def setUp(self): for table_class in (SmallTable, MegaTable): @@ -92,6 +92,8 @@ def test_generate_command(self, print_: MagicMock): # exception otherwise). ast.parse(file_contents) + # Cockroach throws FeatureNotSupportedError, which does not pass this test. + @engines_skip("cockroach") def test_unknown_column_type(self): """ Make sure unknown column types are handled gracefully. @@ -143,6 +145,7 @@ def test_exclude_table(self): SmallTable_ = output_schema.get_table_with_name("SmallTable") self._compare_table_columns(SmallTable, SmallTable_) + @engines_skip("cockroach") def test_self_referencing_fk(self): """ Make sure self-referencing foreign keys are handled correctly. @@ -179,7 +182,7 @@ class Concert(Table): capacity = Integer(index=False) -@postgres_only +@engines_only("postgres") class TestGenerateWithIndexes(TestCase): def setUp(self): Concert.create_table().run_sync() @@ -224,7 +227,7 @@ class Book(Table): popularity = Integer(default=0) -@postgres_only +@engines_only("postgres") class TestGenerateWithSchema(TestCase): def setUp(self) -> None: engine: t.Optional[Engine] = engine_finder() @@ -264,7 +267,7 @@ def test_reference_to_another_schema(self): self.assertEqual(book.writer, writer) -@postgres_only +@engines_only("postgres", "cockroach") class TestGenerateWithException(TestCase): def setUp(self): for table_class in (SmallTable, MegaTable): diff --git a/tests/base.py b/tests/base.py index b6612a124..f08312fc8 100644 --- a/tests/base.py +++ b/tests/base.py @@ -9,6 +9,7 @@ import pytest from piccolo.apps.schema.commands.generate import RowMeta +from piccolo.engine.cockroach import CockroachEngine from piccolo.engine.finder import engine_finder from piccolo.engine.postgres import PostgresEngine from piccolo.engine.sqlite import SQLiteEngine @@ -23,11 +24,15 @@ def engine_version_lt(version: float): def is_running_postgres(): - return isinstance(ENGINE, PostgresEngine) + return type(ENGINE) is PostgresEngine def is_running_sqlite(): - return isinstance(ENGINE, SQLiteEngine) + return type(ENGINE) is SQLiteEngine + + +def is_running_cockroach(): + return type(ENGINE) is CockroachEngine postgres_only = pytest.mark.skipif( @@ -38,11 +43,94 @@ def is_running_sqlite(): not is_running_sqlite(), reason="Only running for SQLite" ) +cockroach_only = pytest.mark.skipif( + not is_running_cockroach(), reason="Only running for Cockroach" +) + unix_only = pytest.mark.skipif( sys.platform.startswith("win"), reason="Only running on a Unix system" ) +def engines_only(*engine_names: str): + """ + Test decorator. Choose what engines can run a test. + + Example + @engines_only('cockroach', 'postgres') + def test_unknown_column_type(...): + self.assertTrue(...) + """ + if ENGINE: + current_engine_name = ENGINE.engine_type + if current_engine_name not in engine_names: + + def wrapper(func): + return pytest.mark.skip( + f"Not running for {current_engine_name}" + )(func) + + return wrapper + else: + + def wrapper(func): + return func + + return wrapper + else: + raise ValueError("Engine not found") + + +def engines_skip(*engine_names: str): + """ + Test decorator. Choose what engines can run a test. + + Example + @engines_skip('cockroach', 'postgres') + def test_unknown_column_type(...): + self.assertTrue(...) + """ + if ENGINE: + current_engine_name = ENGINE.engine_type + if current_engine_name in engine_names: + + def wrapper(func): + return pytest.mark.skip( + f"Not yet available for {current_engine_name}" + )(func) + + return wrapper + else: + + def wrapper(func): + return func + + return wrapper + else: + raise ValueError("Engine not found") + + +def engine_is(*engine_names: str): + """ + Assert branching. Choose what engines can run an assert. + If branching becomes too complex, make a new test with + @engines_only() or engines_skip() + + Example + def test_unknown_column_type(...): + if engine_is('cockroach', 'sqlite'): + self.assertTrue(...) + """ + if ENGINE: + current_engine_name = ENGINE.engine_type + if current_engine_name not in engine_names: + return False + else: + return True + else: + raise ValueError("Engine not found") + + class AsyncMock(MagicMock): """ Async MagicMock for python 3.7+. @@ -134,7 +222,7 @@ def get_postgres_varchar_length( ########################################################################### def create_tables(self): - if ENGINE.engine_type == "postgres": + if ENGINE.engine_type in ("postgres", "cockroach"): self.run_sync( """ CREATE TABLE manager ( @@ -214,60 +302,116 @@ def create_tables(self): raise Exception("Unrecognised engine") def insert_row(self): - self.run_sync( - """ - INSERT INTO manager ( - name - ) VALUES ( - 'Guido' - );""" - ) - self.run_sync( - """ - INSERT INTO band ( - name, - manager, - popularity - ) VALUES ( - 'Pythonistas', - 1, - 1000 - );""" - ) + if ENGINE.engine_type == "cockroach": + id = self.run_sync( + """ + INSERT INTO manager ( + name + ) VALUES ( + 'Guido' + ) RETURNING id;""" + ) + self.run_sync( + f""" + INSERT INTO band ( + name, + manager, + popularity + ) VALUES ( + 'Pythonistas', + {id[0]["id"]}, + 1000 + );""" + ) + else: + self.run_sync( + """ + INSERT INTO manager ( + name + ) VALUES ( + 'Guido' + );""" + ) + self.run_sync( + """ + INSERT INTO band ( + name, + manager, + popularity + ) VALUES ( + 'Pythonistas', + 1, + 1000 + );""" + ) def insert_rows(self): - self.run_sync( - """ - INSERT INTO manager ( - name - ) VALUES ( - 'Guido' - ),( - 'Graydon' - ),( - 'Mads' - );""" - ) - self.run_sync( - """ - INSERT INTO band ( - name, - manager, - popularity - ) VALUES ( - 'Pythonistas', - 1, - 1000 - ),( - 'Rustaceans', - 2, - 2000 - ),( - 'CSharps', - 3, - 10 - );""" - ) + if ENGINE.engine_type == "cockroach": + id = self.run_sync( + """ + INSERT INTO manager ( + name + ) VALUES ( + 'Guido' + ),( + 'Graydon' + ),( + 'Mads' + ) RETURNING id;""" + ) + self.run_sync( + f""" + INSERT INTO band ( + name, + manager, + popularity + ) VALUES ( + 'Pythonistas', + {id[0]["id"]}, + 1000 + ),( + 'Rustaceans', + {id[1]["id"]}, + 2000 + ),( + 'CSharps', + {id[2]["id"]}, + 10 + );""" + ) + else: + self.run_sync( + """ + INSERT INTO manager ( + name + ) VALUES ( + 'Guido' + ),( + 'Graydon' + ),( + 'Mads' + );""" + ) + self.run_sync( + """ + INSERT INTO band ( + name, + manager, + popularity + ) VALUES ( + 'Pythonistas', + 1, + 1000 + ),( + 'Rustaceans', + 2, + 2000 + ),( + 'CSharps', + 3, + 10 + );""" + ) def insert_many_rows(self, row_count=10000): """ @@ -278,7 +422,7 @@ def insert_many_rows(self, row_count=10000): self.run_sync(f"INSERT INTO manager (name) VALUES {values_string};") def drop_tables(self): - if ENGINE.engine_type == "postgres": + if ENGINE.engine_type in ("postgres", "cockroach"): self.run_sync("DROP TABLE IF EXISTS band CASCADE;") self.run_sync("DROP TABLE IF EXISTS manager CASCADE;") self.run_sync("DROP TABLE IF EXISTS ticket CASCADE;") diff --git a/tests/cockroach_conf.py b/tests/cockroach_conf.py new file mode 100644 index 000000000..11b9bf651 --- /dev/null +++ b/tests/cockroach_conf.py @@ -0,0 +1,22 @@ +import os + +from piccolo.conf.apps import AppRegistry +from piccolo.engine.cockroach import CockroachEngine + +DB = CockroachEngine( + config={ + "host": os.environ.get("PG_HOST", "localhost"), + "port": os.environ.get("PG_PORT", "26257"), + "user": os.environ.get("PG_USER", "root"), + "password": os.environ.get("PG_PASSWORD", ""), + "database": os.environ.get("PG_DATABASE", "piccolo"), + } +) + + +APP_REGISTRY = AppRegistry( + apps=[ + "tests.example_apps.music.piccolo_app", + "tests.example_apps.mega.piccolo_app", + ] +) diff --git a/tests/columns/foreign_key/test_on_delete_on_update.py b/tests/columns/foreign_key/test_on_delete_on_update.py index f82461303..aa5239942 100644 --- a/tests/columns/foreign_key/test_on_delete_on_update.py +++ b/tests/columns/foreign_key/test_on_delete_on_update.py @@ -3,7 +3,7 @@ from piccolo.columns import ForeignKey, Varchar from piccolo.columns.base import OnDelete, OnUpdate from piccolo.table import Table -from tests.base import postgres_only +from tests.base import engines_only class Manager(Table): @@ -22,7 +22,7 @@ class Band(Table): ) -@postgres_only +@engines_only("postgres", "cockroach") class TestOnDeleteOnUpdate(TestCase): """ Make sure that on_delete, and on_update are correctly applied in the diff --git a/tests/columns/test_array.py b/tests/columns/test_array.py index 78e82d231..0e3b24a6b 100644 --- a/tests/columns/test_array.py +++ b/tests/columns/test_array.py @@ -2,7 +2,7 @@ from piccolo.columns.column_types import Array, Integer from piccolo.table import Table -from tests.base import postgres_only, sqlite_only +from tests.base import engines_only, sqlite_only class MyTable(Table): @@ -31,32 +31,42 @@ def setUp(self): def tearDown(self): MyTable.alter().drop_table().run_sync() + @engines_only("postgres", "sqlite") def test_storage(self): """ Make sure data can be stored and retrieved. """ + """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + """ # noqa: E501 MyTable(value=[1, 2, 3]).save().run_sync() row = MyTable.objects().first().run_sync() self.assertEqual(row.value, [1, 2, 3]) - @postgres_only + @engines_only("postgres") def test_index(self): """ Indexes should allow individual array elements to be queried. """ + """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + """ # noqa: E501 MyTable(value=[1, 2, 3]).save().run_sync() self.assertEqual( MyTable.select(MyTable.value[0]).first().run_sync(), {"value": 1} ) - @postgres_only + @engines_only("postgres") def test_all(self): """ Make sure rows can be retrieved where all items in an array match a given value. """ + """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + """ # noqa: E501 MyTable(value=[1, 1, 1]).save().run_sync() self.assertEqual( @@ -75,11 +85,15 @@ def test_all(self): None, ) + @engines_only("postgres") def test_any(self): """ Make sure rows can be retrieved where any items in an array match a given value. """ + """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + """ # noqa: E501 MyTable(value=[1, 2, 3]).save().run_sync() self.assertEqual( @@ -98,11 +112,14 @@ def test_any(self): None, ) - @postgres_only + @engines_only("postgres") def test_cat(self): """ Make sure values can be appended to an array. """ + """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + """ # noqa: E501 MyTable(value=[1, 1, 1]).save().run_sync() MyTable.update( @@ -144,5 +161,5 @@ def test_cat_sqlite(self): self.assertEqual( str(manager.exception), - "Only Postgres supports array appending currently.", + "Only Postgres and Cockroach support array appending.", ) diff --git a/tests/columns/test_bigint.py b/tests/columns/test_bigint.py index e7a54eb72..4ebd37de5 100644 --- a/tests/columns/test_bigint.py +++ b/tests/columns/test_bigint.py @@ -4,14 +4,14 @@ from piccolo.columns.column_types import BigInt from piccolo.table import Table -from ..base import postgres_only +from ..base import engines_only class MyTable(Table): value = BigInt() -@postgres_only +@engines_only("postgres", "cockroach") class TestBigIntPostgres(TestCase): """ Make sure a BigInt column in Postgres can store a large number. diff --git a/tests/columns/test_db_column_name.py b/tests/columns/test_db_column_name.py index e18d391a3..24d9bd9eb 100644 --- a/tests/columns/test_db_column_name.py +++ b/tests/columns/test_db_column_name.py @@ -1,6 +1,6 @@ from piccolo.columns.column_types import Integer, Varchar from piccolo.table import Table -from tests.base import DBTestCase, postgres_only +from tests.base import DBTestCase, engine_is, engines_only, engines_skip class Band(Table): @@ -26,7 +26,7 @@ def setUp(self): def tearDown(self): Band.alter().drop_table().run_sync() - @postgres_only + @engines_only("postgres", "cockroach") def test_column_name_correct(self): """ Make sure the column has the correct name in the database. @@ -75,16 +75,28 @@ def test_select(self): # Make sure we can select all columns bands = Band.select().run_sync() - self.assertEqual( - bands, - [ - { - "id": 1, - "regrettable_column_name": "Pythonistas", - "popularity": 1000, - } - ], - ) + if engine_is("cockroach"): + self.assertEqual( + bands, + [ + { + "id": bands[0]["id"], + "regrettable_column_name": "Pythonistas", + "popularity": 1000, + } + ], + ) + else: + self.assertEqual( + bands, + [ + { + "id": 1, + "regrettable_column_name": "Pythonistas", + "popularity": 1000, + } + ], + ) # Make sure we can select a single column bands = Band.select(Band.name).run_sync() @@ -116,19 +128,84 @@ def test_update(self): Band.update({Band.name: "Pythonistas 2"}, force=True).run_sync() + bands = Band.select().run_sync() + if engine_is("cockroach"): + self.assertEqual( + bands, + [ + { + "id": bands[0]["id"], + "regrettable_column_name": "Pythonistas 2", + "popularity": 1000, + } + ], + ) + else: + self.assertEqual( + bands, + [ + { + "id": 1, + "regrettable_column_name": "Pythonistas 2", + "popularity": 1000, + } + ], + ) + + Band.update({"name": "Pythonistas 3"}, force=True).run_sync() + + bands = Band.select().run_sync() + if engine_is("cockroach"): + self.assertEqual( + bands, + [ + { + "id": bands[0]["id"], + "regrettable_column_name": "Pythonistas 3", + "popularity": 1000, + } + ], + ) + else: + self.assertEqual( + bands, + [ + { + "id": 1, + "regrettable_column_name": "Pythonistas 3", + "popularity": 1000, + } + ], + ) + + @engines_skip("cockroach") + def test_delete(self): + """ + Make sure delete queries work correctly. + """ + Band.insert( + Band(name="Pythonistas", popularity=1000), + Band(name="Rustaceans", popularity=500), + ).run_sync() + bands = Band.select().run_sync() self.assertEqual( bands, [ { "id": 1, - "regrettable_column_name": "Pythonistas 2", + "regrettable_column_name": "Pythonistas", "popularity": 1000, - } + }, + { + "id": 2, + "regrettable_column_name": "Rustaceans", + "popularity": 500, + }, ], ) - Band.update({"name": "Pythonistas 3"}, force=True).run_sync() + Band.delete().where(Band.name == "Rustaceans").run_sync() bands = Band.select().run_sync() self.assertEqual( @@ -136,32 +213,37 @@ def test_update(self): [ { "id": 1, - "regrettable_column_name": "Pythonistas 3", + "regrettable_column_name": "Pythonistas", "popularity": 1000, } ], ) - def test_delete(self): + @engines_only("cockroach") + def test_delete_alt(self): """ Make sure delete queries work correctly. """ - Band.insert( - Band(name="Pythonistas", popularity=1000), - Band(name="Rustaceans", popularity=500), - ).run_sync() + result = ( + Band.insert( + Band(name="Pythonistas", popularity=1000), + Band(name="Rustaceans", popularity=500), + ) + .returning(Band.id) + .run_sync() + ) bands = Band.select().run_sync() self.assertEqual( bands, [ { - "id": 1, + "id": result[0]["id"], "regrettable_column_name": "Pythonistas", "popularity": 1000, }, { - "id": 2, + "id": result[1]["id"], "regrettable_column_name": "Rustaceans", "popularity": 500, }, @@ -175,7 +257,7 @@ def test_delete(self): bands, [ { - "id": 1, + "id": result[0]["id"], "regrettable_column_name": "Pythonistas", "popularity": 1000, } diff --git a/tests/columns/test_jsonb.py b/tests/columns/test_jsonb.py index fd4191792..4a2ed1395 100644 --- a/tests/columns/test_jsonb.py +++ b/tests/columns/test_jsonb.py @@ -2,7 +2,7 @@ from piccolo.columns.column_types import JSONB, ForeignKey, Varchar from piccolo.table import Table -from tests.base import postgres_only +from tests.base import engines_only, engines_skip class RecordingStudio(Table): @@ -15,7 +15,7 @@ class Instrument(Table): studio = ForeignKey(RecordingStudio) -@postgres_only +@engines_only("postgres", "cockroach") class TestJSONB(TestCase): def setUp(self): RecordingStudio.create_table().run_sync() @@ -35,6 +35,7 @@ def test_json(self): row.save().run_sync() self.assertEqual(row.facilities, '{"mixing_desk": true}') + @engines_skip("cockroach") def test_raw(self): """ Make sure raw queries convert the Python value into a JSON string. @@ -56,6 +57,28 @@ def test_raw(self): ], ) + @engines_only("cockroach") + def test_raw_alt(self): + """ + Make sure raw queries convert the Python value into a JSON string. + """ + result = RecordingStudio.raw( + "INSERT INTO recording_studio (name, facilities) VALUES ({}, {}) returning id", # noqa: E501 + "Abbey Road", + '{"mixing_desk": true}', + ).run_sync() + + self.assertEqual( + RecordingStudio.select().run_sync(), + [ + { + "id": result[0]["id"], + "name": "Abbey Road", + "facilities": '{"mixing_desk": true}', + } + ], + ) + def test_where(self): """ Test using the where clause to match a subset of rows. diff --git a/tests/columns/test_m2m.py b/tests/columns/test_m2m.py index d3374cfff..e682ba10c 100644 --- a/tests/columns/test_m2m.py +++ b/tests/columns/test_m2m.py @@ -3,6 +3,8 @@ import uuid from unittest import TestCase +from tests.base import engine_is, engines_skip + try: from asyncpg.pgproto.pgproto import UUID as asyncpgUUID except ImportError: @@ -33,8 +35,11 @@ Varchar, ) from piccolo.columns.m2m import M2M +from piccolo.engine.finder import engine_finder from piccolo.table import Table, create_db_tables_sync, drop_db_tables_sync +engine = engine_finder() + class Band(Table): name = Varchar() @@ -59,30 +64,63 @@ class TestM2M(TestCase): def setUp(self): create_db_tables_sync(*SIMPLE_SCHEMA, if_not_exists=True) - Band.insert( - Band(name="Pythonistas"), - Band(name="Rustaceans"), - Band(name="C-Sharps"), - ).run_sync() + if engine_is("cockroach"): + bands = ( + Band.insert( + Band(name="Pythonistas"), + Band(name="Rustaceans"), + Band(name="C-Sharps"), + ) + .returning(Band.id) + .run_sync() + ) - Genre.insert( - Genre(name="Rock"), - Genre(name="Folk"), - Genre(name="Classical"), - ).run_sync() + genres = ( + Genre.insert( + Genre(name="Rock"), + Genre(name="Folk"), + Genre(name="Classical"), + ) + .returning(Genre.id) + .run_sync() + ) - GenreToBand.insert( - GenreToBand(band=1, genre=1), - GenreToBand(band=1, genre=2), - GenreToBand(band=2, genre=2), - GenreToBand(band=3, genre=1), - GenreToBand(band=3, genre=3), - ).run_sync() + GenreToBand.insert( + GenreToBand(band=bands[0]["id"], genre=genres[0]["id"]), + GenreToBand(band=bands[0]["id"], genre=genres[1]["id"]), + GenreToBand(band=bands[1]["id"], genre=genres[1]["id"]), + GenreToBand(band=bands[2]["id"], genre=genres[0]["id"]), + GenreToBand(band=bands[2]["id"], genre=genres[2]["id"]), + ).run_sync() + else: + Band.insert( + Band(name="Pythonistas"), + Band(name="Rustaceans"), + Band(name="C-Sharps"), + ).run_sync() + + Genre.insert( + Genre(name="Rock"), + Genre(name="Folk"), + Genre(name="Classical"), + ).run_sync() + + GenreToBand.insert( + GenreToBand(band=1, genre=1), + GenreToBand(band=1, genre=2), + GenreToBand(band=2, genre=2), + GenreToBand(band=3, genre=1), + GenreToBand(band=3, genre=3), + ).run_sync() def tearDown(self): drop_db_tables_sync(*SIMPLE_SCHEMA) + @engines_skip("cockroach") def test_select_name(self): + """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + """ # noqa: E501 response = Band.select( Band.name, Band.genres(Genre.name, as_list=True) ).run_sync() @@ -108,10 +146,14 @@ def test_select_name(self): ], ) + @engines_skip("cockroach") def test_no_related(self): """ Make sure it still works correctly if there are no related values. """ + """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + """ # noqa: E501 GenreToBand.delete(force=True).run_sync() # Try it with a list response @@ -141,7 +183,11 @@ def test_no_related(self): ], ) + @engines_skip("cockroach") def test_select_multiple(self): + """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + """ # noqa: E501 response = Band.select( Band.name, Band.genres(Genre.id, Genre.name) ).run_sync() @@ -196,7 +242,11 @@ def test_select_multiple(self): ], ) + @engines_skip("cockroach") def test_select_id(self): + """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + """ # noqa: E501 response = Band.select( Band.name, Band.genres(Genre.id, as_list=True) ).run_sync() @@ -435,7 +485,11 @@ def setUp(self): def tearDown(self): drop_db_tables_sync(*CUSTOM_PK_SCHEMA) + @engines_skip("cockroach") def test_select(self): + """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + """ # noqa: E501 response = Customer.select( Customer.name, Customer.concerts(Concert.name, as_list=True) ).run_sync() @@ -515,29 +569,58 @@ class SmallTable(Table): mega_rows = M2M(LazyTableReference("SmallToMega", module_path=__name__)) -class MegaTable(Table): - """ - A table containing all of the column types, and different column kwargs. - """ +if engine.engine_type != "cockroach": # type: ignore - array_col = Array(Varchar()) - bigint_col = BigInt() - boolean_col = Boolean() - bytea_col = Bytea() - date_col = Date() - double_precision_col = DoublePrecision() - integer_col = Integer() - interval_col = Interval() - json_col = JSON() - jsonb_col = JSONB() - numeric_col = Numeric(digits=(5, 2)) - real_col = Real() - smallint_col = SmallInt() - text_col = Text() - timestamp_col = Timestamp() - timestamptz_col = Timestamptz() - uuid_col = UUID() - varchar_col = Varchar() + class MegaTable(Table): # type: ignore + """ + A table containing all of the column types and different column kwargs + """ + + array_col = Array(Varchar()) + bigint_col = BigInt() + boolean_col = Boolean() + bytea_col = Bytea() + date_col = Date() + double_precision_col = DoublePrecision() + integer_col = Integer() + interval_col = Interval() + json_col = JSON() + jsonb_col = JSONB() + numeric_col = Numeric(digits=(5, 2)) + real_col = Real() + smallint_col = SmallInt() + text_col = Text() + timestamp_col = Timestamp() + timestamptz_col = Timestamptz() + uuid_col = UUID() + varchar_col = Varchar() + +else: + + class MegaTable(Table): # type: ignore + """ + Special version for Cockroach. + A table containing all of the column types and different column kwargs + """ + + array_col = Array(Varchar()) + bigint_col = BigInt() + boolean_col = Boolean() + bytea_col = Bytea() + date_col = Date() + double_precision_col = DoublePrecision() + integer_col = BigInt() + interval_col = Interval() + json_col = JSONB() + jsonb_col = JSONB() + numeric_col = Numeric(digits=(5, 2)) + real_col = Real() + smallint_col = SmallInt() + text_col = Text() + timestamp_col = Timestamp() + timestamptz_col = Timestamptz() + uuid_col = UUID() + varchar_col = Varchar() class SmallToMega(Table): @@ -591,11 +674,15 @@ def setUp(self): def tearDown(self): drop_db_tables_sync(*COMPLEX_SCHEMA) + @engines_skip("cockroach") def test_select_all(self): """ Fetch all of the columns from the related table to make sure they're returned correctly. """ + """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + """ # noqa: E501 response = SmallTable.select( SmallTable.varchar_col, SmallTable.mega_rows(load_json=True) ).run_sync() @@ -614,10 +701,14 @@ def test_select_all(self): msg=f"{key} doesn't match", ) + @engines_skip("cockroach") def test_select_single(self): """ Make sure each column can be selected one at a time. """ + """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + """ # noqa: E501 for column in MegaTable._meta.columns: response = SmallTable.select( SmallTable.varchar_col, diff --git a/tests/columns/test_numeric.py b/tests/columns/test_numeric.py index 0c35e50ec..129db5946 100644 --- a/tests/columns/test_numeric.py +++ b/tests/columns/test_numeric.py @@ -27,4 +27,4 @@ def test_creation(self): self.assertEqual(type(_row.column_b), Decimal) self.assertAlmostEqual(_row.column_a, Decimal(1.23)) - self.assertEqual(_row.column_b, Decimal("1.23")) + self.assertAlmostEqual(_row.column_b, Decimal("1.23")) diff --git a/tests/columns/test_smallint.py b/tests/columns/test_smallint.py index 229d66571..4fc6c1a65 100644 --- a/tests/columns/test_smallint.py +++ b/tests/columns/test_smallint.py @@ -4,14 +4,14 @@ from piccolo.columns.column_types import SmallInt from piccolo.table import Table -from ..base import postgres_only +from ..base import engines_only class MyTable(Table): value = SmallInt() -@postgres_only +@engines_only("postgres", "cockroach") class TestSmallIntPostgres(TestCase): """ Make sure a SmallInt column in Postgres can only store small numbers. diff --git a/tests/columns/test_time.py b/tests/columns/test_time.py index d018bbcce..b0be9768e 100644 --- a/tests/columns/test_time.py +++ b/tests/columns/test_time.py @@ -5,6 +5,7 @@ from piccolo.columns.column_types import Time from piccolo.columns.defaults.time import TimeNow from piccolo.table import Table +from tests.base import engines_skip class MyTable(Table): @@ -22,6 +23,7 @@ def setUp(self): def tearDown(self): MyTable.alter().drop_table().run_sync() + @engines_skip("cockroach") def test_timestamp(self): created_on = datetime.datetime.now().time() row = MyTable(created_on=created_on) @@ -38,6 +40,7 @@ def setUp(self): def tearDown(self): MyTableDefault.alter().drop_table().run_sync() + @engines_skip("cockroach") def test_timestamp(self): created_on = datetime.datetime.now().time() row = MyTableDefault() diff --git a/tests/columns/test_varchar.py b/tests/columns/test_varchar.py index bfd24f9e9..89850d801 100644 --- a/tests/columns/test_varchar.py +++ b/tests/columns/test_varchar.py @@ -3,14 +3,14 @@ from piccolo.columns.column_types import Varchar from piccolo.table import Table -from ..base import postgres_only +from ..base import engines_only class MyTable(Table): name = Varchar(length=10) -@postgres_only +@engines_only("postgres", "cockroach") class TestVarchar(TestCase): """ SQLite doesn't enforce any constraints on max character length. diff --git a/tests/engine/test_extra_nodes.py b/tests/engine/test_extra_nodes.py index bbe10b940..2e078c9ee 100644 --- a/tests/engine/test_extra_nodes.py +++ b/tests/engine/test_extra_nodes.py @@ -5,10 +5,10 @@ from piccolo.engine import engine_finder from piccolo.engine.postgres import PostgresEngine from piccolo.table import Table -from tests.base import AsyncMock, postgres_only +from tests.base import AsyncMock, engines_only -@postgres_only +@engines_only("postgres", "cockroach") class TestExtraNodes(TestCase): def test_extra_nodes(self): """ diff --git a/tests/engine/test_pool.py b/tests/engine/test_pool.py index 1c3e80b07..adddc6bda 100644 --- a/tests/engine/test_pool.py +++ b/tests/engine/test_pool.py @@ -6,11 +6,11 @@ from piccolo.engine.postgres import PostgresEngine from piccolo.engine.sqlite import SQLiteEngine -from tests.base import DBTestCase, postgres_only, sqlite_only +from tests.base import DBTestCase, engine_is, engines_only, sqlite_only from tests.example_apps.music.tables import Manager -@postgres_only +@engines_only("postgres", "cockroach") class TestPool(DBTestCase): async def _create_pool(self): engine: PostgresEngine = Manager._meta.db @@ -37,7 +37,12 @@ async def _make_many_queries(self): async def get_data(): response = await Manager.select().run() - self.assertEqual(response, [{"id": 1, "name": "Bob"}]) + if engine_is("cockroach"): + self.assertEqual( + response, [{"id": response[0]["id"], "name": "Bob"}] + ) + else: + self.assertEqual(response, [{"id": 1, "name": "Bob"}]) await asyncio.gather(*[get_data() for _ in range(500)]) @@ -63,7 +68,7 @@ def test_many_queries(self): asyncio.run(self._make_many_queries()) -@postgres_only +@engines_only("postgres", "cockroach") class TestPoolProxyMethods(DBTestCase): async def _create_pool(self): engine: PostgresEngine = Manager._meta.db diff --git a/tests/engine/test_transaction.py b/tests/engine/test_transaction.py index 1cf4538fb..5efe79cb7 100644 --- a/tests/engine/test_transaction.py +++ b/tests/engine/test_transaction.py @@ -4,10 +4,9 @@ from piccolo.engine.postgres import Atomic from piccolo.table import drop_db_tables_sync from piccolo.utils.sync import run_sync +from tests.base import engines_only from tests.example_apps.music.tables import Band, Manager -from ..base import postgres_only - class TestAtomic(TestCase): def test_error(self): @@ -41,7 +40,7 @@ def test_succeeds(self): drop_db_tables_sync(Band, Manager) - @postgres_only + @engines_only("postgres", "cockroach") def test_pool(self): """ Make sure atomic works correctly when a connection pool is active. @@ -108,7 +107,7 @@ async def run_transaction(): self.assertTrue(Band.table_exists().run_sync()) self.assertTrue(Manager.table_exists().run_sync()) - @postgres_only + @engines_only("postgres") def test_transaction_id(self): """ An extra sanity check, that the transaction id is the same for each diff --git a/tests/engine/test_version_parsing.py b/tests/engine/test_version_parsing.py index d49ce65dd..08cd7a7c2 100644 --- a/tests/engine/test_version_parsing.py +++ b/tests/engine/test_version_parsing.py @@ -2,10 +2,10 @@ from piccolo.engine.postgres import PostgresEngine -from ..base import postgres_only +from ..base import engines_only -@postgres_only +@engines_only("postgres", "cockroach") class TestVersionParsing(TestCase): def test_version_parsing(self): """ diff --git a/tests/example_apps/mega/tables.py b/tests/example_apps/mega/tables.py index cf35853be..8947bb3e2 100644 --- a/tests/example_apps/mega/tables.py +++ b/tests/example_apps/mega/tables.py @@ -22,37 +22,73 @@ Timestamptz, Varchar, ) +from piccolo.engine.finder import engine_finder from piccolo.table import Table +engine = engine_finder() + class SmallTable(Table): varchar_col = Varchar() -class MegaTable(Table): - """ - A table containing all of the column types, and different column kwargs. - """ - - bigint_col = BigInt() - boolean_col = Boolean() - bytea_col = Bytea() - date_col = Date() - foreignkey_col = ForeignKey(SmallTable) - integer_col = Integer() - interval_col = Interval() - json_col = JSON() - jsonb_col = JSONB() - numeric_col = Numeric(digits=(5, 2)) - real_col = Real() - double_precision_col = DoublePrecision() - smallint_col = SmallInt() - text_col = Text() - timestamp_col = Timestamp() - timestamptz_col = Timestamptz() - uuid_col = UUID() - varchar_col = Varchar() +if engine.engine_type != "cockroach": # type: ignore + + class MegaTable(Table): # type: ignore + """ + A table containing all of the column types, different column kwargs. + """ + + bigint_col = BigInt() + boolean_col = Boolean() + bytea_col = Bytea() + date_col = Date() + foreignkey_col = ForeignKey(SmallTable) + integer_col = Integer() + interval_col = Interval() + json_col = JSON() + jsonb_col = JSONB() + numeric_col = Numeric(digits=(5, 2)) + real_col = Real() + double_precision_col = DoublePrecision() + smallint_col = SmallInt() + text_col = Text() + timestamp_col = Timestamp() + timestamptz_col = Timestamptz() + uuid_col = UUID() + varchar_col = Varchar() + + unique_col = Varchar(unique=True) + null_col = Varchar(null=True) + not_null_col = Varchar(null=False) + +else: + + class MegaTable(Table): # type: ignore + """ + Special version for Cockroach. + A table containing all of the column types, different column kwargs. + """ + + bigint_col = BigInt() + boolean_col = Boolean() + bytea_col = Bytea() + date_col = Date() + foreignkey_col = ForeignKey(SmallTable) + integer_col = BigInt() + interval_col = Interval() + json_col = JSONB() + jsonb_col = JSONB() + numeric_col = Numeric(digits=(5, 2)) + real_col = Real() + double_precision_col = DoublePrecision() + smallint_col = SmallInt() + text_col = Text() + timestamp_col = Timestamp() + timestamptz_col = Timestamptz() + uuid_col = UUID() + varchar_col = Varchar() - unique_col = Varchar(unique=True) - null_col = Varchar(null=True) - not_null_col = Varchar(null=False) + unique_col = Varchar(unique=True) + null_col = Varchar(null=True) + not_null_col = Varchar(null=False) diff --git a/tests/example_apps/music/tables.py b/tests/example_apps/music/tables.py index 05ad20252..f09f3d8ca 100644 --- a/tests/example_apps/music/tables.py +++ b/tests/example_apps/music/tables.py @@ -3,6 +3,7 @@ from piccolo.columns import ( JSON, JSONB, + BigInt, ForeignKey, Integer, Numeric, @@ -10,8 +11,11 @@ Varchar, ) from piccolo.columns.readable import Readable +from piccolo.engine.finder import engine_finder from piccolo.table import Table +engine = engine_finder() + ############################################################################### # Simple example @@ -24,14 +28,31 @@ def get_readable(cls) -> Readable: return Readable(template="%s", columns=[cls.name]) -class Band(Table): - name = Varchar(length=50) - manager = ForeignKey(Manager, null=True) - popularity = Integer(default=0) +if engine.engine_type != "cockroach": # type: ignore - @classmethod - def get_readable(cls) -> Readable: - return Readable(template="%s", columns=[cls.name]) + class Band(Table): # type: ignore + name = Varchar(length=50) + manager = ForeignKey(Manager, null=True) + popularity = Integer(default=0) + + @classmethod + def get_readable(cls) -> Readable: + return Readable(template="%s", columns=[cls.name]) + +else: + + class Band(Table): # type: ignore + """ + Special version for Cockroach. + """ + + name = Varchar(length=50) + manager = ForeignKey(Manager, null=True) + popularity = BigInt(default=0) + + @classmethod + def get_readable(cls) -> Readable: + return Readable(template="%s", columns=[cls.name]) ############################################################################### diff --git a/tests/query/test_mixins.py b/tests/query/test_mixins.py index bd358fadf..dd9110431 100644 --- a/tests/query/test_mixins.py +++ b/tests/query/test_mixins.py @@ -1,10 +1,11 @@ -from unittest import TestCase +import time # For time travel queries. from piccolo.query.mixins import ColumnsDelegate +from tests.base import DBTestCase, engines_only from tests.example_apps.music.tables import Band -class TestColumnsDelegate(TestCase): +class TestColumnsDelegate(DBTestCase): def test_list_unpacking(self): """ The ``ColumnsDelegate`` should unpack a list of columns if passed in by @@ -26,3 +27,36 @@ def test_list_unpacking(self): self.assertEqual( columns_delegate.selected_columns, [Band.name, Band.id] ) + + @engines_only("cockroach") + def test_as_of(self): + """ + Time travel queries using "As Of" syntax. + Currently supports Cockroach using AS OF SYSTEM TIME. + """ + self.insert_rows() + time.sleep(1) # Ensure time travel queries have some history to use! + + result = ( + Band.select() + .where(Band.name == "Pythonistas") + .as_of("-500ms") + .limit(1) + ) + self.assertTrue("AS OF SYSTEM TIME '-500ms'" in str(result)) + result = result.run_sync() + + self.assertTrue(result[0]["name"] == "Pythonistas") + + result = Band.select().as_of() + self.assertTrue("AS OF SYSTEM TIME '-1s'" in str(result)) + result = result.run_sync() + + self.assertTrue(result[0]["name"] == "Pythonistas") + + # Alternative syntax. + result = Band.objects().get(Band.name == "Pythonistas").as_of("-1s") + self.assertTrue("AS OF SYSTEM TIME '-1s'" in str(result)) + result = result.run_sync() + + self.assertTrue(result.name == "Pythonistas") diff --git a/tests/table/instance/test_get_related_readable.py b/tests/table/instance/test_get_related_readable.py index 1e182a679..2088c867f 100644 --- a/tests/table/instance/test_get_related_readable.py +++ b/tests/table/instance/test_get_related_readable.py @@ -4,6 +4,7 @@ from piccolo.columns import ForeignKey, Varchar from piccolo.columns.readable import Readable from piccolo.table import Table, create_db_tables_sync, drop_db_tables_sync +from tests.base import engine_is from tests.example_apps.music.tables import ( Band, Concert, @@ -123,18 +124,33 @@ def test_get_related_readable(self): Ticket.id, Ticket._get_related_readable(Ticket.concert), ).run_sync() - self.assertEqual( - response, - [ - { - "id": 1, - "concert_readable": ( - "Pythonistas and Rustaceans at Royal Albert Hall, " - "capacity 5900" - ), - } - ], - ) + + if engine_is("cockroach"): + self.assertEqual( + response, + [ + { + "id": response[0]["id"], + "concert_readable": ( + "Pythonistas and Rustaceans at Royal Albert Hall, " + "capacity 5900" + ), + } + ], + ) + else: + self.assertEqual( + response, + [ + { + "id": 1, + "concert_readable": ( + "Pythonistas and Rustaceans at Royal Albert Hall, " + "capacity 5900" + ), + } + ], + ) # A really complex references chain from Piccolo Admin issue #170 response = ThingFour.select( diff --git a/tests/table/instance/test_instantiate.py b/tests/table/instance/test_instantiate.py index 43b84f49d..6fceaa2be 100644 --- a/tests/table/instance/test_instantiate.py +++ b/tests/table/instance/test_instantiate.py @@ -1,4 +1,4 @@ -from tests.base import DBTestCase, postgres_only, sqlite_only +from tests.base import DBTestCase, engines_only, sqlite_only from tests.example_apps.music.tables import Band @@ -7,13 +7,20 @@ class TestInstance(DBTestCase): Test instantiating Table instances """ - @postgres_only + @engines_only("postgres") def test_insert_postgres(self): Pythonistas = Band(name="Pythonistas") self.assertEqual( Pythonistas.__str__(), "(DEFAULT,'Pythonistas',null,0)" ) + @engines_only("cockroach") + def test_insert_postgres_alt(self): + Pythonistas = Band(name="Pythonistas") + self.assertEqual( + Pythonistas.__str__(), "(unique_rowid(),'Pythonistas',null,0)" + ) + @sqlite_only def test_insert_sqlite(self): Pythonistas = Band(name="Pythonistas") diff --git a/tests/table/instance/test_save.py b/tests/table/instance/test_save.py index e97df0d54..67a27ee85 100644 --- a/tests/table/instance/test_save.py +++ b/tests/table/instance/test_save.py @@ -1,6 +1,7 @@ from unittest import TestCase from piccolo.table import create_db_tables_sync, drop_db_tables_sync +from tests.base import engines_only, engines_skip from tests.example_apps.music.tables import Band, Manager @@ -36,6 +37,7 @@ def test_save_new(self): self.assertTrue("Maz2" in names) self.assertTrue("Maz" not in names) + @engines_skip("cockroach") def test_save_specific_columns(self): """ Make sure that we can save a subset of columns. @@ -75,8 +77,6 @@ def test_save_specific_columns(self): ], ) - ####################################################################### - # Also test it using strings to identify columns band.name = "Pythonistas 3" band.popularity = 3000 @@ -94,3 +94,61 @@ def test_save_specific_columns(self): } ], ) + + @engines_only("cockroach") + def test_save_specific_columns_alt(self): + """ + Make sure that we can save a subset of columns. + """ + manager = Manager(name="Guido") + manager.save().run_sync() + + band = Band(name="Pythonistas", popularity=1000, manager=manager) + band.save().run_sync() + + self.assertEqual( + Band.select().run_sync(), + [ + { + "id": band.id, + "name": "Pythonistas", + "manager": band.manager.id, + "popularity": 1000, + } + ], + ) + + band.name = "Pythonistas 2" + band.popularity = 2000 + band.save(columns=[Band.name]).run_sync() + + # Only the name should update, and not the popularity: + self.assertEqual( + Band.select().run_sync(), + [ + { + "id": band.id, + "name": "Pythonistas 2", + "manager": band.manager.id, + "popularity": 1000, + } + ], + ) + + # Also test it using strings to identify columns + band.name = "Pythonistas 3" + band.popularity = 3000 + band.save(columns=["popularity"]).run_sync() + + # Only the popularity should update, and not the name: + self.assertEqual( + Band.select().run_sync(), + [ + { + "id": band.id, + "name": "Pythonistas 2", + "manager": band.manager.id, + "popularity": 3000, + } + ], + ) diff --git a/tests/table/instance/test_to_dict.py b/tests/table/instance/test_to_dict.py index 4142974b7..b40bad790 100644 --- a/tests/table/instance/test_to_dict.py +++ b/tests/table/instance/test_to_dict.py @@ -1,4 +1,4 @@ -from tests.base import DBTestCase +from tests.base import DBTestCase, engine_is from tests.example_apps.music.tables import Band, Manager @@ -11,7 +11,12 @@ def test_to_dict(self): instance = Manager.objects().first().run_sync() dictionary = instance.to_dict() - self.assertDictEqual(dictionary, {"id": 1, "name": "Guido"}) + if engine_is("cockroach"): + self.assertDictEqual( + dictionary, {"id": dictionary["id"], "name": "Guido"} + ) + else: + self.assertDictEqual(dictionary, {"id": 1, "name": "Guido"}) def test_nested(self): """ @@ -22,15 +27,29 @@ def test_nested(self): instance = Band.objects(Band.manager).first().run_sync() dictionary = instance.to_dict() - self.assertDictEqual( - dictionary, - { - "id": 1, - "name": "Pythonistas", - "manager": {"id": 1, "name": "Guido"}, - "popularity": 1000, - }, - ) + if engine_is("cockroach"): + self.assertDictEqual( + dictionary, + { + "id": dictionary["id"], + "name": "Pythonistas", + "manager": { + "id": instance["manager"]["id"], + "name": "Guido", + }, + "popularity": 1000, + }, + ) + else: + self.assertDictEqual( + dictionary, + { + "id": 1, + "name": "Pythonistas", + "manager": {"id": 1, "name": "Guido"}, + "popularity": 1000, + }, + ) def test_filter_rows(self): """ @@ -51,13 +70,22 @@ def test_nested_filter(self): instance = Band.objects(Band.manager).first().run_sync() dictionary = instance.to_dict(Band.name, Band.manager.id) - self.assertDictEqual( - dictionary, - { - "name": "Pythonistas", - "manager": {"id": 1}, - }, - ) + if engine_is("cockroach"): + self.assertDictEqual( + dictionary, + { + "name": "Pythonistas", + "manager": {"id": dictionary["manager"]["id"]}, + }, + ) + else: + self.assertDictEqual( + dictionary, + { + "name": "Pythonistas", + "manager": {"id": 1}, + }, + ) def test_aliases(self): """ @@ -69,4 +97,9 @@ def test_aliases(self): dictionary = instance.to_dict( Manager.id, Manager.name.as_alias("title") ) - self.assertDictEqual(dictionary, {"id": 1, "title": "Guido"}) + if engine_is("cockroach"): + self.assertDictEqual( + dictionary, {"id": dictionary["id"], "title": "Guido"} + ) + else: + self.assertDictEqual(dictionary, {"id": 1, "title": "Guido"}) diff --git a/tests/table/test_alter.py b/tests/table/test_alter.py index cfc637258..6a1e55eb0 100644 --- a/tests/table/test_alter.py +++ b/tests/table/test_alter.py @@ -12,8 +12,8 @@ from tests.base import ( DBTestCase, engine_version_lt, + engines_only, is_running_sqlite, - postgres_only, ) from tests.example_apps.music.tables import Band, Manager @@ -81,7 +81,7 @@ def tearDown(self): self.run_sync("DROP TABLE IF EXISTS act") -@postgres_only +@engines_only("postgres", "cockroach") class TestDropColumn(DBTestCase): """ Unfortunately this only works with Postgres at the moment. @@ -153,9 +153,13 @@ def test_problematic_name(self): ) -@postgres_only class TestUnique(DBTestCase): + @engines_only("postgres") def test_unique(self): + """ + Test altering a column uniqueness with MigrationManager. + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/42840 "unimplemented: cannot drop UNIQUE constraint "manager_name_key" using ALTER TABLE DROP CONSTRAINT, use DROP INDEX CASCADE instead" + """ # noqa: E501 unique_query = Manager.alter().set_unique(Manager.name, True) unique_query.run_sync() @@ -180,7 +184,7 @@ def test_unique(self): self.assertTrue(len(response), 2) -@postgres_only +@engines_only("postgres", "cockroach") class TestMultiple(DBTestCase): """ Make sure multiple alter statements work correctly. @@ -204,7 +208,7 @@ def test_multiple(self): # TODO - test more conversions. -@postgres_only +@engines_only("postgres", "cockroach") class TestSetColumnType(DBTestCase): def test_integer_to_bigint(self): """ @@ -270,7 +274,7 @@ def test_using_expression(self): self.assertEqual(popularity, 1) -@postgres_only +@engines_only("postgres", "cockroach") class TestSetNull(DBTestCase): def test_set_null(self): query = """ @@ -289,7 +293,7 @@ def test_set_null(self): self.assertEqual(response[0]["is_nullable"], "NO") -@postgres_only +@engines_only("postgres", "cockroach") class TestSetLength(DBTestCase): def test_set_length(self): query = """ @@ -305,7 +309,7 @@ def test_set_length(self): self.assertEqual(response[0]["character_maximum_length"], length) -@postgres_only +@engines_only("postgres", "cockroach") class TestSetDefault(DBTestCase): def test_set_default(self): Manager.alter().set_default(Manager.name, "Pending").run_sync() @@ -326,7 +330,6 @@ class Ticket(Table): price = Numeric(digits=(5, 2)) -@postgres_only class TestSetDigits(TestCase): def setUp(self): Ticket.create_table().run_sync() @@ -334,6 +337,7 @@ def setUp(self): def tearDown(self): Ticket.alter().drop_table().run_sync() + @engines_only("postgres") def test_set_digits(self): query = """ SELECT numeric_precision, numeric_scale diff --git a/tests/table/test_batch.py b/tests/table/test_batch.py index 69f7a027c..8fca2944c 100644 --- a/tests/table/test_batch.py +++ b/tests/table/test_batch.py @@ -7,7 +7,7 @@ from piccolo.engine.postgres import AsyncBatch, PostgresEngine from piccolo.table import Table from piccolo.utils.sync import run_sync -from tests.base import AsyncMock, DBTestCase, postgres_only +from tests.base import AsyncMock, DBTestCase, engines_only from tests.example_apps.music.tables import Manager @@ -95,7 +95,7 @@ def test_batch(self): self.assertEqual(iterations, _iterations) -@postgres_only +@engines_only("postgres", "cockroach") class TestBatchNodeArg(TestCase): def test_batch_extra_node(self): """ diff --git a/tests/table/test_join.py b/tests/table/test_join.py index d89dc2402..114edbc1f 100644 --- a/tests/table/test_join.py +++ b/tests/table/test_join.py @@ -1,6 +1,7 @@ import decimal from unittest import TestCase +from tests.base import engine_is from tests.example_apps.music.tables import ( Band, Concert, @@ -66,17 +67,31 @@ def test_join(self): Concert.band_1.manager, ) response = select_query.run_sync() - self.assertEqual( - response, - [ - { - "band_1.name": "Pythonistas", - "band_2.name": "Rustaceans", - "venue.name": "Grand Central", - "band_1.manager": 1, - } - ], - ) + + if engine_is("cockroach"): + self.assertEqual( + response, + [ + { + "band_1.name": "Pythonistas", + "band_2.name": "Rustaceans", + "venue.name": "Grand Central", + "band_1.manager": response[0]["band_1.manager"], + } + ], + ) + else: + self.assertEqual( + response, + [ + { + "band_1.name": "Pythonistas", + "band_2.name": "Rustaceans", + "venue.name": "Grand Central", + "band_1.manager": 1, + } + ], + ) # Now make sure that even deeper joins work: select_query = Concert.select(Concert.band_1.manager.name) @@ -93,14 +108,25 @@ def test_select_all_columns(self): .first() .run_sync() ) - self.assertDictEqual( - result, - { - "name": "Pythonistas", - "manager.id": 1, - "manager.name": "Guido", - }, - ) + + if engine_is("cockroach"): + self.assertDictEqual( + result, + { + "name": "Pythonistas", + "manager.id": result["manager.id"], + "manager.name": "Guido", + }, + ) + else: + self.assertDictEqual( + result, + { + "name": "Pythonistas", + "manager.id": 1, + "manager.name": "Guido", + }, + ) def test_select_all_columns_deep(self): """ @@ -116,18 +142,32 @@ def test_select_all_columns_deep(self): .run_sync() ) - self.assertDictEqual( - result, - { - "venue.id": 1, - "venue.name": "Grand Central", - "venue.capacity": 1000, - "band_1.manager.id": 1, - "band_1.manager.name": "Guido", - "band_2.manager.id": 2, - "band_2.manager.name": "Graydon", - }, - ) + if engine_is("cockroach"): + self.assertDictEqual( + result, + { + "venue.id": result["venue.id"], + "venue.name": "Grand Central", + "venue.capacity": 1000, + "band_1.manager.id": result["band_1.manager.id"], + "band_1.manager.name": "Guido", + "band_2.manager.id": result["band_2.manager.id"], + "band_2.manager.name": "Graydon", + }, + ) + else: + self.assertDictEqual( + result, + { + "venue.id": 1, + "venue.name": "Grand Central", + "venue.capacity": 1000, + "band_1.manager.id": 1, + "band_1.manager.name": "Guido", + "band_2.manager.id": 2, + "band_2.manager.name": "Graydon", + }, + ) def test_select_all_columns_root(self): """ @@ -142,17 +182,31 @@ def test_select_all_columns_root(self): .first() .run_sync() ) - self.assertDictEqual( - result, - { - "id": 1, - "name": "Pythonistas", - "manager": 1, - "popularity": 1000, - "manager.id": 1, - "manager.name": "Guido", - }, - ) + + if engine_is("cockroach"): + self.assertDictEqual( + result, + { + "id": result["id"], + "name": "Pythonistas", + "manager": result["manager"], + "popularity": 1000, + "manager.id": result["manager.id"], + "manager.name": "Guido", + }, + ) + else: + self.assertDictEqual( + result, + { + "id": 1, + "name": "Pythonistas", + "manager": 1, + "popularity": 1000, + "manager.id": 1, + "manager.name": "Guido", + }, + ) def test_select_all_columns_root_nested(self): """ @@ -166,15 +220,29 @@ def test_select_all_columns_root_nested(self): .run_sync() ) - self.assertDictEqual( - result, - { - "id": 1, - "name": "Pythonistas", - "manager": {"id": 1, "name": "Guido"}, - "popularity": 1000, - }, - ) + if engine_is("cockroach"): + self.assertDictEqual( + result, + { + "id": result["id"], + "name": "Pythonistas", + "manager": { + "id": result["manager"]["id"], + "name": "Guido", + }, + "popularity": 1000, + }, + ) + else: + self.assertDictEqual( + result, + { + "id": 1, + "name": "Pythonistas", + "manager": {"id": 1, "name": "Guido"}, + "popularity": 1000, + }, + ) def test_select_all_columns_exclude(self): """ diff --git a/tests/table/test_objects.py b/tests/table/test_objects.py index 61091778f..f8c699ef2 100644 --- a/tests/table/test_objects.py +++ b/tests/table/test_objects.py @@ -1,4 +1,4 @@ -from tests.base import DBTestCase, postgres_only, sqlite_only +from tests.base import DBTestCase, engines_only, sqlite_only from tests.example_apps.music.tables import Band, Manager @@ -27,7 +27,7 @@ def test_get_all(self): class TestOffset(DBTestCase): - @postgres_only + @engines_only("postgres", "cockroach") def test_offset_postgres(self): """ Postgres can do an offset without a limit clause. diff --git a/tests/table/test_output.py b/tests/table/test_output.py index 683e9097c..0fbaafa0d 100644 --- a/tests/table/test_output.py +++ b/tests/table/test_output.py @@ -1,7 +1,7 @@ import json from unittest import TestCase -from tests.base import DBTestCase +from tests.base import DBTestCase, engine_is from tests.example_apps.music.tables import Band, RecordingStudio @@ -45,10 +45,28 @@ def test_select(self): results = RecordingStudio.select().output(load_json=True).run_sync() - self.assertEqual( - results, - [{"id": 1, "facilities": {"a": 123}, "facilities_b": {"a": 123}}], - ) + if engine_is("cockroach"): + self.assertEqual( + results, + [ + { + "id": results[0]["id"], + "facilities": {"a": 123}, + "facilities_b": {"a": 123}, + } + ], + ) + else: + self.assertEqual( + results, + [ + { + "id": 1, + "facilities": {"a": 123}, + "facilities_b": {"a": 123}, + } + ], + ) def test_objects(self): json = {"a": 123} diff --git a/tests/table/test_raw.py b/tests/table/test_raw.py index 12fa372fc..f49f1c9a6 100644 --- a/tests/table/test_raw.py +++ b/tests/table/test_raw.py @@ -1,4 +1,4 @@ -from tests.base import DBTestCase +from tests.base import DBTestCase, engine_is from tests.example_apps.music.tables import Band @@ -8,10 +8,26 @@ def test_raw_without_args(self): response = Band.raw("select * from band").run_sync() - self.assertDictEqual( - response[0], - {"id": 1, "name": "Pythonistas", "manager": 1, "popularity": 1000}, - ) + if engine_is("cockroach"): + self.assertDictEqual( + response[0], + { + "id": response[0]["id"], + "name": "Pythonistas", + "manager": response[0]["manager"], + "popularity": 1000, + }, + ) + else: + self.assertDictEqual( + response[0], + { + "id": 1, + "name": "Pythonistas", + "manager": 1, + "popularity": 1000, + }, + ) def test_raw_with_args(self): self.insert_rows() @@ -21,7 +37,24 @@ def test_raw_with_args(self): ).run_sync() self.assertEqual(len(response), 1) - self.assertDictEqual( - response[0], - {"id": 1, "name": "Pythonistas", "manager": 1, "popularity": 1000}, - ) + + if engine_is("cockroach"): + self.assertDictEqual( + response[0], + { + "id": response[0]["id"], + "name": "Pythonistas", + "manager": response[0]["manager"], + "popularity": 1000, + }, + ) + else: + self.assertDictEqual( + response[0], + { + "id": 1, + "name": "Pythonistas", + "manager": 1, + "popularity": 1000, + }, + ) diff --git a/tests/table/test_repr.py b/tests/table/test_repr.py index a7ff329a1..59ec57ae0 100644 --- a/tests/table/test_repr.py +++ b/tests/table/test_repr.py @@ -11,4 +11,4 @@ def test_repr_postgres(self): self.insert_row() manager = Manager.objects().first().run_sync() - self.assertEqual(manager.__repr__(), "") + self.assertEqual(manager.__repr__(), f"") diff --git a/tests/table/test_select.py b/tests/table/test_select.py index 18c000e6f..247236026 100644 --- a/tests/table/test_select.py +++ b/tests/table/test_select.py @@ -3,7 +3,13 @@ from piccolo.apps.user.tables import BaseUser from piccolo.columns.combination import WhereRaw from piccolo.query.methods.select import Avg, Count, Max, Min, Sum -from tests.base import DBTestCase, postgres_only, sqlite_only +from tests.base import ( + DBTestCase, + engine_is, + engines_only, + engines_skip, + sqlite_only, +) from tests.example_apps.music.tables import Band, Concert, Manager, Venue @@ -13,10 +19,26 @@ def test_query_all_columns(self): response = Band.select().run_sync() - self.assertDictEqual( - response[0], - {"id": 1, "name": "Pythonistas", "manager": 1, "popularity": 1000}, - ) + if engine_is("cockroach"): + self.assertDictEqual( + response[0], + { + "id": response[0]["id"], + "name": "Pythonistas", + "manager": response[0]["manager"], + "popularity": 1000, + }, + ) + else: + self.assertDictEqual( + response[0], + { + "id": 1, + "name": "Pythonistas", + "manager": 1, + "popularity": 1000, + }, + ) def test_query_some_columns(self): self.insert_row() @@ -74,7 +96,7 @@ def test_where_equals(self): response = Band.select(Band.name).where().run_sync() self.assertEqual(response, [{"name": "Pythonistas"}]) - @postgres_only + @engines_only("postgres", "cockroach") def test_where_like_postgres(self): """ Postgres' LIKE is case sensitive. @@ -168,7 +190,7 @@ def test_where_ilike_sqlite(self): .run_sync(), ) - @postgres_only + @engines_only("postgres", "cockroach") def test_where_ilike_postgres(self): """ Only Postgres has ILIKE - it's not in the SQL standard. It's for @@ -375,6 +397,7 @@ def test_where_or(self): response, [{"name": "CSharps"}, {"name": "Rustaceans"}] ) + @engines_skip("cockroach") def test_multiple_where(self): """ Test that chaining multiple where clauses works results in an AND. @@ -392,6 +415,7 @@ def test_multiple_where(self): self.assertEqual(response, [{"name": "Rustaceans"}]) self.assertIn("AND", query.__str__()) + @engines_skip("cockroach") def test_complex_where(self): """ Test a complex where clause - combining AND, and OR. @@ -422,7 +446,7 @@ def test_limit(self): self.assertEqual(response, [{"name": "CSharps"}]) - @postgres_only + @engines_only("postgres", "cockroach") def test_offset_postgres(self): self.insert_rows() @@ -906,7 +930,13 @@ def test_columns(self): .first() .run_sync() ) - self.assertEqual(response, {"id": 1, "name": "Pythonistas"}) + + if engine_is("cockroach"): + self.assertEqual( + response, {"id": response["id"], "name": "Pythonistas"} + ) + else: + self.assertEqual(response, {"id": 1, "name": "Pythonistas"}) def test_call_chain(self): """ @@ -985,5 +1015,10 @@ def test_secret_parameter(self): venue.save().run_sync() venue_dict = Venue.select(exclude_secrets=True).first().run_sync() - self.assertTrue(venue_dict, {"id": 1, "name": "The Garage"}) + if engine_is("cockroach"): + self.assertTrue( + venue_dict, {"id": venue_dict["id"], "name": "The Garage"} + ) + else: + self.assertTrue(venue_dict, {"id": 1, "name": "The Garage"}) self.assertNotIn("capacity", venue_dict.keys()) diff --git a/tests/table/test_str.py b/tests/table/test_str.py index ac3861700..604002f6c 100644 --- a/tests/table/test_str.py +++ b/tests/table/test_str.py @@ -1,18 +1,29 @@ from unittest import TestCase +from tests.base import engine_is from tests.example_apps.music.tables import Manager class TestTableStr(TestCase): def test_str(self): - self.assertEqual( - Manager._table_str(), - ( - "class Manager(Table, tablename='manager'):\n" - " id = Serial(null=False, primary_key=True, unique=False, index=False, index_method=IndexMethod.btree, choices=None, db_column_name='id', secret=False)\n" # noqa: E501 - " name = Varchar(length=50, default='', null=False, primary_key=False, unique=False, index=False, index_method=IndexMethod.btree, choices=None, db_column_name=None, secret=False)\n" # noqa: E501 - ), - ) + if engine_is("cockroach"): + self.assertEqual( + Manager._table_str(), + ( + "class Manager(Table, tablename='manager'):\n" + " id = Serial(null=False, primary_key=True, unique=False, index=False, index_method=IndexMethod.btree, choices=None, db_column_name='id', secret=False)\n" # noqa: E501 + " name = Varchar(length=50, default='', null=False, primary_key=False, unique=False, index=False, index_method=IndexMethod.btree, choices=None, db_column_name=None, secret=False)\n" # noqa: E501 + ), + ) + else: + self.assertEqual( + Manager._table_str(), + ( + "class Manager(Table, tablename='manager'):\n" + " id = Serial(null=False, primary_key=True, unique=False, index=False, index_method=IndexMethod.btree, choices=None, db_column_name='id', secret=False)\n" # noqa: E501 + " name = Varchar(length=50, default='', null=False, primary_key=False, unique=False, index=False, index_method=IndexMethod.btree, choices=None, db_column_name=None, secret=False)\n" # noqa: E501 + ), + ) self.assertEqual( Manager._table_str(abbreviated=True), diff --git a/tests/table/test_update.py b/tests/table/test_update.py index 8a70159f3..23073af2c 100644 --- a/tests/table/test_update.py +++ b/tests/table/test_update.py @@ -20,6 +20,7 @@ from tests.base import ( DBTestCase, engine_version_lt, + engines_skip, is_running_sqlite, sqlite_only, ) @@ -515,6 +516,7 @@ def setUp(self): def tearDown(self): MyTable.alter().drop_table().run_sync() + @engines_skip("cockroach") def test_operators(self): for test_case in TEST_CASES: print(test_case.description) diff --git a/tests/testing/test_model_builder.py b/tests/testing/test_model_builder.py index 9918399b0..6e52289f8 100644 --- a/tests/testing/test_model_builder.py +++ b/tests/testing/test_model_builder.py @@ -4,6 +4,7 @@ from piccolo.columns import Array, Integer, Real, Varchar from piccolo.table import Table from piccolo.testing.model_builder import ModelBuilder +from tests.base import engines_skip from tests.example_apps.music.tables import ( Band, Concert, @@ -22,6 +23,8 @@ class TableWithArrayField(Table): floats = Array(Real()) +# Cockroach Bug: Can turn ON when resolved: https://github.com/cockroachdb/cockroach/issues/71908 # noqa: E501 +@engines_skip("cockroach") class TestModelBuilder(unittest.TestCase): @classmethod def setUpClass(cls): diff --git a/tests/utils/test_lazy_loader.py b/tests/utils/test_lazy_loader.py index 5be0d69ea..32a6be15b 100644 --- a/tests/utils/test_lazy_loader.py +++ b/tests/utils/test_lazy_loader.py @@ -1,14 +1,14 @@ from unittest import TestCase, mock from piccolo.utils.lazy_loader import LazyLoader -from tests.base import postgres_only, sqlite_only +from tests.base import engines_only, sqlite_only class TestLazyLoader(TestCase): def test_lazy_loading_database_driver(self): _ = LazyLoader("asyncpg", globals(), "asyncpg") - @postgres_only + @engines_only("postgres", "cockroach") def test_lazy_loader_asyncpg_exception(self): lazy_loader = LazyLoader("asyncpg", globals(), "asyncpg.connect") diff --git a/tests/utils/test_table_reflection.py b/tests/utils/test_table_reflection.py index 0e540d755..b11858f30 100644 --- a/tests/utils/test_table_reflection.py +++ b/tests/utils/test_table_reflection.py @@ -5,11 +5,11 @@ from piccolo.table import Table from piccolo.table_reflection import TableStorage from piccolo.utils.sync import run_sync -from tests.base import postgres_only +from tests.base import engines_only from tests.example_apps.music.tables import Band, Manager -@postgres_only +@engines_only("postgres", "cockroach") class TestTableStorage(TestCase): def setUp(self) -> None: self.table_storage = TableStorage() From 8ae2accf74e6a5cb40825fe57a0ba67413a50824 Mon Sep 17 00:00:00 2001 From: Miguel Guthridge Date: Tue, 11 Oct 2022 21:17:49 +1000 Subject: [PATCH 378/727] Style improvements for table module (#637) * Fix a couple of type conflicts * Add references to replacement functions for deprecated functions * format with black Co-authored-by: Daniel Townsend --- piccolo/table.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/piccolo/table.py b/piccolo/table.py index 8aeeee6f4..c4a92d3ef 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -1247,16 +1247,19 @@ def create_table_class( For example, `{'my_column': Varchar()}`. """ - return types.new_class( - name=class_name, - bases=bases, - kwds=class_kwargs, - exec_body=lambda namespace: namespace.update(class_members), + return t.cast( + t.Type[Table], + types.new_class( + name=class_name, + bases=bases, + kwds=class_kwargs, + exec_body=lambda namespace: namespace.update(class_members), + ), ) ############################################################################### -# Quickly create or drop database tables from Piccolo `Table` clases. +# Quickly create or drop database tables from Piccolo `Table` classes. async def create_db_tables( @@ -1304,6 +1307,9 @@ def create_tables(*tables: t.Type[Table], if_not_exists: bool = False) -> None: This original implementation has been replaced, because it was synchronous, and felt at odds with the rest of the Piccolo codebase which is async first. + + Instead, use create_db_tables for asynchronous code, or + create_db_tables_sync for synchronous code """ colored_warning( "`create_tables` is deprecated and will be removed in v1 of Piccolo. " @@ -1364,6 +1370,9 @@ def drop_tables(*tables: t.Type[Table]) -> None: This original implementation has been replaced, because it was synchronous, and felt at odds with the rest of the Piccolo codebase which is async first. + + Instead, use drop_db_tables for asynchronous code, or + drop_db_tables_sync for synchronous code """ colored_warning( "`drop_tables` is deprecated and will be removed in v1 of Piccolo. " From d3dd23bcc4afb67e2cbc0c3efe3f426ad85bc4cf Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 11 Oct 2022 12:25:26 +0100 Subject: [PATCH 379/727] bumped version --- CHANGES.rst | 10 ++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 3977ebbb3..3864192d3 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,16 @@ Changes ======= +0.92.0 +------ + +Added initial support for Cockroachdb (thanks to @gnat for this massive +contribution). + +Fixed Pylance warnings (thanks to @MiguelGuthridge for this). + +------------------------------------------------------------------------------- + 0.91.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 716c5a75a..2872dfbee 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.91.0" +__VERSION__ = "0.92.0" From ac7f54ec2016064ac7ad86bfd96c6dcdb2cb4bac Mon Sep 17 00:00:00 2001 From: Nathaniel Sabanski Date: Tue, 11 Oct 2022 05:16:23 -0700 Subject: [PATCH 380/727] Documentation: New as_of() clause (Advanced, added with CockroachDB). (#638) * Documentation: as_of() clause (Added with CockroachDB). * Added reference to formats accepted. --- docs/src/piccolo/query_clauses/as_of.rst | 25 ++++++++++++++++++++++++ docs/src/piccolo/query_clauses/index.rst | 1 + 2 files changed, 26 insertions(+) create mode 100644 docs/src/piccolo/query_clauses/as_of.rst diff --git a/docs/src/piccolo/query_clauses/as_of.rst b/docs/src/piccolo/query_clauses/as_of.rst new file mode 100644 index 000000000..2d8407e42 --- /dev/null +++ b/docs/src/piccolo/query_clauses/as_of.rst @@ -0,0 +1,25 @@ +.. _as_of: + +as_of +===== + +You can use ``as_of`` clause with the following queries: + +* :ref:`Select` +* :ref:`Objects` + +To retrieve historical data from 5 minutes ago: + +.. code-block:: python + + await Band.select().where( + Band.name == 'Pythonistas' + ).as_of('-5min') + +This generates an ``AS OF SYSTEM TIME`` clause. See `documentation `_. + +This clause accepts a wide variety of time and interval `string formats `_. + +This is very useful for performance, as it will reduce transaction contention across a cluster. + +Currently only supported on Cockroach Engine. diff --git a/docs/src/piccolo/query_clauses/index.rst b/docs/src/piccolo/query_clauses/index.rst index 19c4cfbae..ede4da86c 100644 --- a/docs/src/piccolo/query_clauses/index.rst +++ b/docs/src/piccolo/query_clauses/index.rst @@ -20,6 +20,7 @@ by modifying the return values. :maxdepth: 1 :caption: Advanced + ./as_of ./batch ./callback ./distinct From e224e901d81e6076c9f3d574595651975f3b93e3 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 11 Oct 2022 22:04:15 +0100 Subject: [PATCH 381/727] make sure `Time` column is in `piccolo/columns/__init__.py` (#642) --- piccolo/columns/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/piccolo/columns/__init__.py b/piccolo/columns/__init__.py index e84d4c3ad..12a258960 100644 --- a/piccolo/columns/__init__.py +++ b/piccolo/columns/__init__.py @@ -23,6 +23,7 @@ Serial, SmallInt, Text, + Time, Timestamp, Timestamptz, Varchar, @@ -60,6 +61,7 @@ "Serial", "SmallInt", "Text", + "Time", "Timestamp", "Timestamptz", "Varchar", From 56675bb958f3cbb6dfa8f1f5faa5067ad39b0a8a Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 12 Oct 2022 21:00:39 +0100 Subject: [PATCH 382/727] Improve transaction docs (#644) * remove `run` from transaction docs * add a note about why we use `MyTable._meta.db` --- docs/src/piccolo/query_types/transactions.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/src/piccolo/query_types/transactions.rst b/docs/src/piccolo/query_types/transactions.rst index ca0656505..4c72b073c 100644 --- a/docs/src/piccolo/query_types/transactions.rst +++ b/docs/src/piccolo/query_types/transactions.rst @@ -8,6 +8,10 @@ Transactions allow multiple queries to be committed only once successful. This is useful for things like migrations, where you can't have it fail in an inbetween state. +.. note:: + In the examples below we use ``MyTable._meta.db`` to access the ``Engine``, + which is used to create transactions. + ------------------------------------------------------------------------------- Atomic @@ -37,8 +41,8 @@ async. .. code-block:: python async with Band._meta.db.transaction(): - await Manager.create_table().run() - await Concert.create_table().run() + await Manager.create_table() + await Concert.create_table() If an exception is raised within the body of the context manager, then the transaction is automatically rolled back. The exception is still propagated From 7f39e5bafadbd512689156b024e4b1d8c7b39ad2 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 12 Oct 2022 21:27:23 +0100 Subject: [PATCH 383/727] add docs for `add_raw_backwards` (#645) --- docs/src/piccolo/migrations/create.rst | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/docs/src/piccolo/migrations/create.rst b/docs/src/piccolo/migrations/create.rst index 28c078214..404e09648 100644 --- a/docs/src/piccolo/migrations/create.rst +++ b/docs/src/piccolo/migrations/create.rst @@ -7,7 +7,8 @@ controlled way. Each migration belongs to a :ref:`Piccolo app `. You can either manually populate migrations, or allow Piccolo to do it for you automatically. -We recommend using automatic migrations where possible, as it saves you time. +We recommend using :ref:`auto migrations ` where possible, +as it saves you time. ------------------------------------------------------------------------------- @@ -90,10 +91,26 @@ If you want to run raw SQL within your migration, you can do so as follows: description=DESCRIPTION ) + ############################################################# + # This will get run when using `piccolo migrations forwards`: + async def run(): await RawTable.raw('UPDATE band SET popularity={}', 1000) manager.add_raw(run) + + ############################################################# + # If we want to run some code when reversing the migration, + # using `piccolo migrations backwards`: + + async def run_backwards(): + await RawTable.raw('UPDATE band SET popularity={}', 0) + + manager.add_raw_backwards(run_backwards) + + ############################################################# + # We must always return the MigrationManager: + return manager .. hint:: You can learn more about :ref:`raw queries here `. @@ -178,6 +195,8 @@ it's better to copy the relevant tables into your migration file: ------------------------------------------------------------------------------- +.. _AutoMigrations: + Auto migrations --------------- From 3fb22b01d492d9be2cd12d68a2d510d9964109cd Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 19 Oct 2022 21:28:38 +0100 Subject: [PATCH 384/727] Update Postgres and Python in GitHub Actions (#648) * add Python 3.11 to setup.py * add Postgres 15 and Python 3.11, drop Postgres 9.6 from GitHub actions * update min Postgres version in `PostgresEngine` --- .github/workflows/tests.yaml | 4 ++-- piccolo/engine/postgres.py | 2 +- setup.py | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 285d2fcb7..5bcae47a9 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -81,8 +81,8 @@ jobs: timeout-minutes: 30 strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] - postgres-version: [9.6, 10, 11, 12, 13, 14] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11-dev"] + postgres-version: [10, 11, 12, 13, 14, 15] # Service containers to run with `container-job` services: diff --git a/piccolo/engine/postgres.py b/piccolo/engine/postgres.py index fb76714f0..5a3a263da 100644 --- a/piccolo/engine/postgres.py +++ b/piccolo/engine/postgres.py @@ -266,7 +266,7 @@ class PostgresEngine(Engine): ) engine_type = "postgres" - min_version_number = 9.6 + min_version_number = 10 def __init__( self, diff --git a/setup.py b/setup.py index e904e0a41..fa8d25ac3 100644 --- a/setup.py +++ b/setup.py @@ -88,6 +88,7 @@ def extras_require() -> t.Dict[str, t.List[str]]: "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Programming Language :: Python :: Implementation :: CPython", "Framework :: AsyncIO", "Typing :: Typed", From 930b7dfb5795849dd1b9dcb19c75b63a23dbb362 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Acosta?= <33264613+eneacosta@users.noreply.github.com> Date: Fri, 21 Oct 2022 18:12:22 -0300 Subject: [PATCH 385/727] Fix create_pydantic_model for nullable JSON and JSONB (#649) * Fix create_pydantic_model for nullable JSON and JSONB * validation outside try statement, and simple tests for None values in pydantic model added * remove env file * minor tweak - use `assertIsNone` instead of `assert` It gives us slightly better output if an exception occurs. Co-authored-by: Daniel Townsend --- piccolo/utils/pydantic.py | 4 +++- tests/utils/test_pydantic.py | 11 +++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/piccolo/utils/pydantic.py b/piccolo/utils/pydantic.py index 62489e143..fbe63fe4d 100644 --- a/piccolo/utils/pydantic.py +++ b/piccolo/utils/pydantic.py @@ -37,7 +37,9 @@ class Config(pydantic.BaseConfig): arbitrary_types_allowed = True -def pydantic_json_validator(cls, value): +def pydantic_json_validator(cls, value, field): + if not field.required and value is None: + return value try: load_json(value) except json.JSONDecodeError as e: diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index dea4f3e67..394ed4afe 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -265,6 +265,17 @@ class Movie(Table): "json", ) + def test_null_value(self): + class Movie(Table): + meta = JSON(null=True) + meta_b = JSONB(null=True) + + pydantic_model = create_pydantic_model(table=Movie) + movie = pydantic_model(meta=None, meta_b=None) + + self.assertIsNone(movie.meta) + self.assertIsNone(movie.meta_b) + class TestExcludeColumns(TestCase): def test_all(self): From 7ec609db15a17d052d1e94fc99b3b05c47913c68 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 21 Oct 2022 22:21:19 +0100 Subject: [PATCH 386/727] moved CockroachDB specific query clauses into own table of contents --- docs/src/piccolo/query_clauses/index.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/src/piccolo/query_clauses/index.rst b/docs/src/piccolo/query_clauses/index.rst index ede4da86c..abb3fa18b 100644 --- a/docs/src/piccolo/query_clauses/index.rst +++ b/docs/src/piccolo/query_clauses/index.rst @@ -20,7 +20,6 @@ by modifying the return values. :maxdepth: 1 :caption: Advanced - ./as_of ./batch ./callback ./distinct @@ -29,3 +28,9 @@ by modifying the return values. ./offset ./output ./returning + +.. toctree:: + :maxdepth: 1 + :caption: CockroachDB + + ./as_of From fc7694f53dbeae6d8c09f3650a320f767477e2e7 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 21 Oct 2022 22:26:15 +0100 Subject: [PATCH 387/727] bumped version --- CHANGES.rst | 15 +++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 3864192d3..e86da92b9 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,21 @@ Changes ======= +0.93.0 +------ + +* Fixed a bug with nullable ``JSON`` / ``JSONB`` columns and + ``create_pydantic_model`` - thanks to @eneacosta for this fix. +* Made the ``Time`` column type importable from ``piccolo.columns``. +* Python 3.11 is now supported. +* Postgres 9.6 is no longer officially supported, as it's end of life, but + Piccolo should continue to work with it just fine for now. +* Improved docs for transactions, added docs for the ``as_of`` clause in + CockroachDB (thanks to @gnat for this), and added docs for + ``add_raw_backwards``. + +------------------------------------------------------------------------------- + 0.92.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 2872dfbee..11e8c12fc 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.92.0" +__VERSION__ = "0.93.0" From 9da05e9eb6e077a3627fb58dfe0d963cfe782cee Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 23 Oct 2022 21:37:31 +0100 Subject: [PATCH 388/727] fix bug with `SomeTable.objects().create()` and non-null values (#654) --- piccolo/query/methods/objects.py | 15 +++----------- tests/table/instance/test_create.py | 31 +++++++++++++++++++++++------ 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index 82507be87..df3e9c508 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -104,21 +104,12 @@ def prefetch(self, *fk_columns) -> GetOrCreate: @dataclass class Create: - query: Objects + table_class: t.Type[Table] columns: t.Dict[str, t.Any] async def run(self): - instance = self.query.table() - - for column, value in self.columns.items(): - if isinstance(column, str): - column = instance._meta.get_column_by_name(column) - setattr(instance, column._meta.name, value) - + instance = self.table_class(**self.columns) await instance.save().run() - - instance._was_created = True - return instance def __await__(self): @@ -220,7 +211,7 @@ def get_or_create( return GetOrCreate(query=self, where=where, defaults=defaults) def create(self, **columns: t.Any): - return Create(query=self, columns=columns) + return Create(table_class=self.table, columns=columns) def order_by(self, *columns: Column, ascending=True) -> Objects: self.order_by_delegate.order_by(*columns, ascending=ascending) diff --git a/tests/table/instance/test_create.py b/tests/table/instance/test_create.py index 73d028b23..6e4856cc2 100644 --- a/tests/table/instance/test_create.py +++ b/tests/table/instance/test_create.py @@ -1,20 +1,39 @@ from unittest import TestCase -from tests.example_apps.music.tables import Manager +from piccolo.columns import Integer, Varchar +from piccolo.table import Table + + +class Band(Table): + name = Varchar(default=None, null=False) + popularity = Integer() class TestCreate(TestCase): def setUp(self): - Manager.create_table().run_sync() + Band.create_table().run_sync() def tearDown(self): - Manager.alter().drop_table().run_sync() + Band.alter().drop_table().run_sync() def test_create_new(self): """ Make sure that creating a new instance works. """ - Manager.objects().create(name="Maz").run_sync() + Band.objects().create(name="Pythonistas", popularity=1000).run_sync() + + names = [i["name"] for i in Band.select(Band.name).run_sync()] + self.assertTrue("Pythonistas" in names) + + def test_null_values(self): + """ + Make sure we test non-null columns: + https://github.com/piccolo-orm/piccolo/issues/652 + """ + with self.assertRaises(ValueError) as manager: + Band.objects().create().run_sync() + + self.assertEqual(str(manager.exception), "name wasn't provided") - names = [i["name"] for i in Manager.select(Manager.name).run_sync()] - self.assertTrue("Maz" in names) + # Shouldn't raise an exception + Band.objects().create(name="Pythonistas").run_sync() From f6fba4e3b278a21c41eb8650cbddda72fe3cbb9d Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 23 Oct 2022 22:23:53 +0100 Subject: [PATCH 389/727] `__file__` -> `__name__` (#655) --- piccolo/apps/user/tables.py | 2 +- piccolo/engine/base.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/piccolo/apps/user/tables.py b/piccolo/apps/user/tables.py index 43c5e9bda..718531942 100644 --- a/piccolo/apps/user/tables.py +++ b/piccolo/apps/user/tables.py @@ -15,7 +15,7 @@ from piccolo.table import Table from piccolo.utils.sync import run_sync -logger = logging.getLogger(__file__) +logger = logging.getLogger(__name__) class BaseUser(Table, tablename="piccolo_user"): diff --git a/piccolo/engine/base.py b/piccolo/engine/base.py index 43b317ab4..7d55c3a37 100644 --- a/piccolo/engine/base.py +++ b/piccolo/engine/base.py @@ -12,7 +12,7 @@ from piccolo.query.base import Query -logger = logging.getLogger(__file__) +logger = logging.getLogger(__name__) class Batch: From 6c1571cfee945330135974eb1197892649f3c4d2 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 23 Oct 2022 22:31:41 +0100 Subject: [PATCH 390/727] bumped version --- CHANGES.rst | 12 ++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index e86da92b9..e65912fc8 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,18 @@ Changes ======= +0.94.0 +------ + +Fixed a bug with ``MyTable.objects().create()`` and columns which are not +nullable. Thanks to @metakot for reporting this issue. + +We used to use ``logging.getLogger(__file__)``, but as @Drapersniper pointed +out, the Python docs recommend ``logging.getLogger(__name__)``, so it has been +changed. + +------------------------------------------------------------------------------- + 0.93.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 11e8c12fc..4b32b5215 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.93.0" +__VERSION__ = "0.94.0" From 9e9fbfa39a3393f3e67e78f6ce05ee2c9532f1fb Mon Sep 17 00:00:00 2001 From: Haffi Mazhar <53666594+haffi96@users.noreply.github.com> Date: Fri, 28 Oct 2022 10:22:27 +0100 Subject: [PATCH 391/727] Playground: Option to use ipython profile (#656) * Playground: Option to use ipython profile * Playground: use conf params * remove `type: ignore` for iPython import * don't show migrations warning if running the playground * remove mouse support for now * updated examples in playground to use await, and using coloured prints The iPython shell supports top level await, so may as well use the async queries. * added comment about how to set custom iPython colours * tweaked docstring Co-authored-by: Daniel Townsend --- piccolo/apps/playground/commands/run.py | 41 ++++++++++++++++--------- piccolo/main.py | 5 +-- pyproject.toml | 1 + 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/piccolo/apps/playground/commands/run.py b/piccolo/apps/playground/commands/run.py index 6f5d3fb02..8d388fd98 100644 --- a/piccolo/apps/playground/commands/run.py +++ b/piccolo/apps/playground/commands/run.py @@ -24,6 +24,7 @@ from piccolo.engine import PostgresEngine, SQLiteEngine from piccolo.engine.base import Engine from piccolo.table import Table +from piccolo.utils.warnings import colored_string class Manager(Table): @@ -189,6 +190,7 @@ def run( database: str = "piccolo_playground", host: str = "localhost", port: int = 5432, + ipython_profile: bool = False, ): """ Creates a test database to play with. @@ -205,6 +207,10 @@ def run( Postgres host :param port: Postgres port + :param ipython_profile: + Set to true to use your own IPython profile. Located at ~/.ipython/. + For more info see the IPython docs + https://ipython.readthedocs.io/en/stable/config/intro.html. """ try: import IPython @@ -228,29 +234,36 @@ def run( db = SQLiteEngine() for _table in TABLES: _table._meta._db = db - print("Tables:\n") + + print(colored_string("\nTables:\n")) for _table in TABLES: print(_table._table_str(abbreviated=True)) - print("\n") + print("") - print("Try it as a query builder:") - print("Band.select().run_sync()") - print("Band.select(Band.name).run_sync()") - print("Band.select(Band.name, Band.manager.name).run_sync()") + print(colored_string("Try it as a query builder:")) + print("await Band.select()") + print("await Band.select(Band.name)") + print("await Band.select(Band.name, Band.manager.name)") print("\n") - print("Try it as an ORM:") - print( - "b = Band.objects().where(Band.name == 'Pythonistas').first()." - "run_sync()" - ) + print(colored_string("Try it as an ORM:")) + print("b = await Band.objects().where(Band.name == 'Pythonistas').first()") print("b.popularity = 10000") - print("b.save().run_sync()") + print("await b.save()") print("\n") populate() - from IPython.core.interactiveshell import _asyncio_runner # type: ignore + from IPython.core.interactiveshell import _asyncio_runner + + if ipython_profile: + print(colored_string("Using your IPython profile\n")) + # To try this out, set `c.TerminalInteractiveShell.colors = "Linux"` + # in `~/.ipython/profile_default/ipython_config.py` to set the terminal + # color. + conf_args = {} + else: + conf_args = {"colors": "neutral"} - IPython.embed(using=_asyncio_runner) + IPython.embed(using=_asyncio_runner, **conf_args) diff --git a/piccolo/main.py b/piccolo/main.py index a175ddea4..1fc1d28d5 100644 --- a/piccolo/main.py +++ b/piccolo/main.py @@ -104,10 +104,11 @@ def main(): aliases=command.aliases, ) - if "migrations" not in sys.argv: + if not {"playground", "migrations"}.intersection(set(sys.argv)): # Show a warning if any migrations haven't been run. # Don't run it if it looks like the user is running a migration - # command, as this information is redundant. + # command, or using the playground, as this information is + # redundant. try: havent_ran_count = run_sync( diff --git a/pyproject.toml b/pyproject.toml index ad8732598..1fb257c2a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ module = [ "colorama", "dateutil", "IPython", + "IPython.core.interactiveshell", "jinja2", "orjson", "aiosqlite" From 0c089c3cc93aad31af947bac8356feca5da67371 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 28 Oct 2022 10:40:12 +0100 Subject: [PATCH 392/727] bumped version --- CHANGES.rst | 16 ++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index e65912fc8..c36017d71 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,22 @@ Changes ======= +0.95.0 +------ + +Made improvements to the Piccolo playground. + +* Syntax highlighting is now enabled. +* The example queries are now async (iPython supports top level await, so + this works fine). +* You can optionally use your own iPython configuration + ``piccolo playground run --ipython_profile`` (for example if you want a + specific colour scheme, rather than the one we use by default). + +Thanks to @haffi96 for this. See `PR 656 `_. + +------------------------------------------------------------------------------- + 0.94.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 4b32b5215..f5df62219 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.94.0" +__VERSION__ = "0.95.0" From 35bf736ffac8f9660cbf9ddef358492ef1b33622 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 28 Oct 2022 10:46:57 +0100 Subject: [PATCH 393/727] updated playground docs - added iPython section --- docs/src/piccolo/playground/advanced.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/src/piccolo/playground/advanced.rst b/docs/src/piccolo/playground/advanced.rst index 24fb585ab..df090aaed 100644 --- a/docs/src/piccolo/playground/advanced.rst +++ b/docs/src/piccolo/playground/advanced.rst @@ -43,3 +43,17 @@ When you have the database setup, you can connect to it as follows: .. code-block:: bash piccolo playground run --engine=postgres + +iPython +------- + +The playground is built on top of iPython. We provide sensible defaults out of +the box for syntax highlighting etc. However, to use your own custom iPython +profile (located in ``~/.ipython``), do the following: + +.. code-block:: bash + + piccolo playground run --ipython_profile + +See the `iPython docs `_ +for more information. From fbf298f763432859c9682a8197608e633e3326dd Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 30 Oct 2022 21:36:35 +0000 Subject: [PATCH 394/727] use Python 3.11 in GitHub actions (#661) --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 5bcae47a9..827be6f65 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -81,7 +81,7 @@ jobs: timeout-minutes: 30 strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11-dev"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] postgres-version: [10, 11, 12, 13, 14, 15] # Service containers to run with `container-job` From 0456561f6687fce9c55132ce206b9486dddf116b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 8 Nov 2022 20:59:52 +0000 Subject: [PATCH 395/727] added docs for required fields and `create_pydantic_model` (#667) * adds docs for required fields and `create_pydantic_model` * add docs for subclassing Pydantic models * fix integration test failures Need to version pin `di` for now with `xpresso`. * fix linter --- docs/src/piccolo/serialization/index.rst | 72 ++++++++++++++++--- piccolo/apps/asgi/commands/new.py | 7 +- .../templates/app/requirements.txt.jinja | 4 +- piccolo/main.py | 4 +- 4 files changed, 75 insertions(+), 12 deletions(-) diff --git a/docs/src/piccolo/serialization/index.rst b/docs/src/piccolo/serialization/index.rst index 18ac966aa..ceb2761f8 100644 --- a/docs/src/piccolo/serialization/index.rst +++ b/docs/src/piccolo/serialization/index.rst @@ -4,10 +4,13 @@ Serialization Piccolo uses `Pydantic `_ internally to serialize and deserialize data. +Using ``create_pydantic_model`` you can easily create Pydantic models for your +application. + ------------------------------------------------------------------------------- -create_pydantic_model ---------------------- +``create_pydantic_model`` +------------------------- Using ``create_pydantic_model`` we can easily create a `Pydantic model `_ from a Piccolo ``Table``. @@ -52,8 +55,8 @@ We can then create model instances from data we fetch from the database: You have several options for configuring the model, as shown below. -include_columns / exclude_columns -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``include_columns`` / ``exclude_columns`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If we want to exclude the ``popularity`` column from the ``Band`` table: @@ -67,8 +70,8 @@ Conversely, if you only wanted the ``popularity`` column: BandModel = create_pydantic_model(Band, include_columns=(Band.popularity,)) -nested -~~~~~~ +``nested`` +~~~~~~~~~~ Another great feature is ``nested=True``. For each ``ForeignKey`` in the Piccolo ``Table``, the Pydantic model will contain a sub model for the related @@ -122,8 +125,8 @@ To populate a nested Pydantic model with data from the database: There is a `video tutorial on YouTube `_. -include_default_columns -~~~~~~~~~~~~~~~~~~~~~~~ +``include_default_columns`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~ Sometimes you'll want to include the Piccolo ``Table``'s primary key column in the generated Pydantic model. For example, in a ``GET`` endpoint, we usually @@ -151,6 +154,59 @@ By default the primary key column isn't included - you can add it using: BandModel = create_pydantic_model(Band, include_default_columns=True) +Required fields +~~~~~~~~~~~~~~~ + +You can specify which fields are required using the ``required`` +argument of :class:`Column `. For example: + +.. code-block:: python + + class Band(Table): + name = Varchar(required=True) + + BandModel = create_pydantic_model(Band) + + # Omitting the field raises an error: + >>> BandModel() + ValidationError - name field required + +You can override this behaviour using the ``all_optional`` argument. An example +use case is when you have a model which is used for filtering, then you'll want +all fields to be optional. + +.. code-block:: python + + class Band(Table): + name = Varchar(required=True) + + BandFilterModel = create_pydantic_model( + Band, + all_optional=True, + model_name='BandFilterModel', + ) + + # This no longer raises an exception: + >>> BandModel() + +Subclassing the model +~~~~~~~~~~~~~~~~~~~~~ + +If the generated model doesn't perfectly fit your needs, you can subclass it to +add additional fields, and to override existing fields. + +.. code-block:: python + + class Band(Table): + name = Varchar(required=True) + + BandModel = create_pydantic_model(Band) + + class CustomBandModel(BandModel): + genre: str + + >>> CustomBandModel(name="Pythonistas", genre="Rock") + Source ~~~~~~ diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index a95bc3eef..a5f25691a 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -11,7 +11,10 @@ TEMPLATE_DIR = os.path.join(os.path.dirname(__file__), "templates/app/") SERVERS = ["uvicorn", "Hypercorn"] ROUTERS = ["starlette", "fastapi", "blacksheep", "xpresso", "starlite"] -ROUTER_VERSIONS = {"starlite": "==1.23.0", "xpresso": "==0.43.0"} +ROUTER_DEPENDENCIES = { + "starlite": ["starlite==1.23.0"], + "xpresso": ["xpresso==0.43.0", "di==0.72.1"], +} def print_instruction(message: str): @@ -52,7 +55,7 @@ def new(root: str = ".", name: str = "piccolo_project"): template_context = { "router": router, - "router_version": ROUTER_VERSIONS.get(router), + "router_dependencies": ROUTER_DEPENDENCIES.get(router) or [router], "server": get_server(), "project_identifier": name.replace(" ", "_").lower(), } diff --git a/piccolo/apps/asgi/commands/templates/app/requirements.txt.jinja b/piccolo/apps/asgi/commands/templates/app/requirements.txt.jinja index ff7f895d2..7bbb8ba53 100644 --- a/piccolo/apps/asgi/commands/templates/app/requirements.txt.jinja +++ b/piccolo/apps/asgi/commands/templates/app/requirements.txt.jinja @@ -1,4 +1,6 @@ -{{ router }}{% if router_version %}{{ router_version }}{% endif %} +{%- for router_dependency in router_dependencies -%} +{{ router_dependency }} +{% endfor -%} {{ server }} jinja2 piccolo[postgres] diff --git a/piccolo/main.py b/piccolo/main.py index 1fc1d28d5..0766a0aa7 100644 --- a/piccolo/main.py +++ b/piccolo/main.py @@ -104,7 +104,9 @@ def main(): aliases=command.aliases, ) - if not {"playground", "migrations"}.intersection(set(sys.argv)): + if not {"playground", "migrations", "asgi"}.intersection( + set(sys.argv) + ): # Show a warning if any migrations haven't been run. # Don't run it if it looks like the user is running a migration # command, or using the playground, as this information is From 621163333d99c503ddbc3de841a87b083e847a5b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 9 Nov 2022 19:02:14 +0000 Subject: [PATCH 396/727] added `auto_update` option to `Column` (#670) --- piccolo/columns/base.py | 31 ++++++++++++++-- piccolo/table.py | 51 ++++++++++++++++++++++---- tests/table/test_update.py | 74 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 147 insertions(+), 9 deletions(-) diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 14ac5e003..defaecf86 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -154,6 +154,7 @@ class ColumnMeta: help_text: t.Optional[str] = None choices: t.Optional[t.Type[Enum]] = None secret: bool = False + auto_update: t.Any = ... # Used for representing the table in migrations and the playground. params: t.Dict[str, t.Any] = field(default_factory=dict) @@ -437,6 +438,30 @@ class Band(Table): >>> await Band.select(exclude_secrets=True) [{'name': 'Pythonistas'}] + :param auto_update: + Allows you to specify a value to set this column to each time it is + updated (via ``MyTable.update``, or ``MyTable.save`` on an existing + row). A common use case is having a ``modified_on`` column. + + .. code-block:: python + + class Band(Table): + name = Varchar() + popularity = Integer() + # The value can be a function or static value: + modified_on = Timestamp(auto_update=datetime.datetime.now) + + # This will automatically set the `modified_on` column to the + # current timestamp, without having to explicitly set it: + >>> await Band.update({ + ... Band.popularity: Band.popularity + 100 + ... }).where(Band.name == 'Pythonistas') + + Note - this feature is implemented purely within the ORM. If you want + similar functionality on the database level (i.e. if you plan on using + raw SQL to perform updates), then you may be better off creating SQL + triggers instead. + """ value_type: t.Type = int @@ -453,6 +478,7 @@ def __init__( choices: t.Optional[t.Type[Enum]] = None, db_column_name: t.Optional[str] = None, secret: bool = False, + auto_update: t.Any = ..., **kwargs, ) -> None: # This is for backwards compatibility - originally there were two @@ -462,8 +488,8 @@ def __init__( primary_key = True # Used for migrations. - # We deliberately omit 'required', and 'help_text' as they don't effect - # the actual schema. + # We deliberately omit 'required', 'auto_update' and 'help_text' as + # they don't effect the actual schema. # 'choices' isn't used directly in the schema, but may be important # for data migrations. kwargs.update( @@ -494,6 +520,7 @@ def __init__( choices=choices, _db_column_name=db_column_name, secret=secret, + auto_update=auto_update, ) self._alias: t.Optional[str] = None diff --git a/piccolo/table.py b/piccolo/table.py index c4a92d3ef..a6cdc64c4 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -75,6 +75,7 @@ class TableMeta: primary_key: Column = field(default_factory=Column) json_columns: t.List[t.Union[JSON, JSONB]] = field(default_factory=list) secret_columns: t.List[Secret] = field(default_factory=list) + auto_update_columns: t.List[Column] = field(default_factory=list) tags: t.List[str] = field(default_factory=list) help_text: t.Optional[str] = None _db: t.Optional[Engine] = None @@ -136,6 +137,18 @@ def get_column_by_name(self, name: str) -> Column: return column_object + def get_auto_update_values(self) -> t.Dict[Column, t.Any]: + """ + If columns have ``auto_update`` defined, then we retrieve these values. + """ + output: t.Dict[Column, t.Any] = {} + for column in self.auto_update_columns: + value = column._meta.auto_update + if callable(value): + value = value() + output[column] = value + return output + class TableMetaclass(type): def __str__(cls): @@ -219,6 +232,7 @@ def __init_subclass__( secret_columns: t.List[Secret] = [] json_columns: t.List[t.Union[JSON, JSONB]] = [] email_columns: t.List[Email] = [] + auto_update_columns: t.List[Column] = [] primary_key: t.Optional[Column] = None m2m_relationships: t.List[M2M] = [] @@ -264,6 +278,9 @@ def __init_subclass__( if isinstance(column, (JSON, JSONB)): json_columns.append(column) + if column._meta.auto_update is not ...: + auto_update_columns.append(column) + if isinstance(attribute, M2M): attribute._meta._name = attribute_name attribute._meta._table = cls @@ -286,6 +303,7 @@ def __init_subclass__( foreign_key_columns=foreign_key_columns, json_columns=json_columns, secret_columns=secret_columns, + auto_update_columns=auto_update_columns, tags=tags, help_text=help_text, _db=db, @@ -418,6 +436,7 @@ def save( """ cls = self.__class__ + # New row - insert if not self._exists_in_db: return cls.insert(self).returning(cls._meta.primary_key) @@ -436,13 +455,21 @@ def save( i: getattr(self, i._meta.name, None) for i in column_instances } - return ( - cls.update() - .values(values) # type: ignore - .where( - cls._meta.primary_key - == getattr(self, self._meta.primary_key._meta.name) - ) + # Assign any `auto_update` values + if cls._meta.auto_update_columns: + auto_update_values = cls._meta.get_auto_update_values() + values.update(auto_update_values) + for column, value in auto_update_values.items(): + setattr(self, column._meta.name, value) + + return cls.update( + values, # type: ignore + # We've already included the `auto_update` columns, so no need + # to do it again: + use_auto_update=False, + ).where( + cls._meta.primary_key + == getattr(self, self._meta.primary_key._meta.name) ) def remove(self) -> Delete: @@ -1074,6 +1101,7 @@ def update( cls, values: t.Dict[t.Union[Column, str], t.Any] = None, force: bool = False, + use_auto_update: bool = True, **kwargs, ) -> Update: """ @@ -1105,10 +1133,19 @@ def update( Unless set to ``True``, updates aren't allowed without a ``where`` clause, to prevent accidental mass overriding of data. + :param use_auto_update: + Whether to use the ``auto_update`` values on any columns. See + the ``auto_update`` argument on + :class:`Column ` for more information. + """ if values is None: values = {} values = dict(values, **kwargs) + + if use_auto_update and cls._meta.auto_update_columns: + values.update(cls._meta.get_auto_update_values()) # type: ignore + return Update(table=cls, force=force).values(values) @classmethod diff --git a/tests/table/test_update.py b/tests/table/test_update.py index 23073af2c..62c595870 100644 --- a/tests/table/test_update.py +++ b/tests/table/test_update.py @@ -554,3 +554,77 @@ def test_edge_cases(self): # An error should be raised because we can't save at this level # of resolution - 1 millisecond is the minimum. MyTable.timestamp + datetime.timedelta(microseconds=1) + + +############################################################################### +# Test auto_update + + +class AutoUpdateTable(Table, tablename="my_table"): + name = Varchar() + modified_on = Timestamp( + auto_update=datetime.datetime.now, null=True, default=None + ) + + +class TestAutoUpdate(TestCase): + def setUp(self): + AutoUpdateTable.create_table().run_sync() + + def tearDown(self): + AutoUpdateTable.alter().drop_table().run_sync() + + def test_save(self): + """ + Make sure the ``save`` method uses ``auto_update`` columns correctly. + """ + row = AutoUpdateTable(name="test") + + # Saving for the first time is an INSERT, so `auto_update` shouldn't + # be triggered. + row.save().run_sync() + self.assertIsNone(row.modified_on) + + # A subsequent save is an UPDATE, so `auto_update` should be triggered. + row.name = "test 2" + row.save().run_sync() + self.assertIsInstance(row.modified_on, datetime.datetime) + + # If we save it again, `auto_update` should be applied again. + existing_modified_on = row.modified_on + row.name = "test 3" + row.save().run_sync() + self.assertIsInstance(row.modified_on, datetime.datetime) + self.assertGreater(row.modified_on, existing_modified_on) + + def test_update(self): + """ + Make sure the update method uses ``auto_update`` columns correctly. + """ + # Insert a row for us to update + AutoUpdateTable.insert(AutoUpdateTable(name="test")).run_sync() + + self.assertDictEqual( + AutoUpdateTable.select( + AutoUpdateTable.name, AutoUpdateTable.modified_on + ) + .first() + .run_sync(), + {"name": "test", "modified_on": None}, + ) + + # Update the row + AutoUpdateTable.update( + {AutoUpdateTable.name: "test 2"}, force=True + ).run_sync() + + # Retrieve the row + updated_row = ( + AutoUpdateTable.select( + AutoUpdateTable.name, AutoUpdateTable.modified_on + ) + .first() + .run_sync() + ) + self.assertIsInstance(updated_row["modified_on"], datetime.datetime) + self.assertEqual(updated_row["name"], "test 2") From 970bef9ccdefe212fd0d97f1d1c2cff6ea66e8ea Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 9 Nov 2022 19:24:21 +0000 Subject: [PATCH 397/727] bumped version --- CHANGES.rst | 31 +++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index c36017d71..0a51c3030 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,37 @@ Changes ======= +0.96.0 +------ + +Added the ``auto_update`` argument to ``Column``. Its main use case is columns +like ``modified_on`` where we want the value to be updated automatically each +time the row is saved. + +.. code-block:: python + + class Band(Table): + name = Varchar() + popularity = Integer() + modified_on = Timestamp( + null=True, + default=None, + auto_update=datetime.datetime.now + ) + + # The `modified_on` column will automatically be updated to the current + # timestamp: + >>> await Band.update({ + ... Band.popularity: Band.popularity + 100 + ... }).where( + ... Band.name == 'Pythonistas' + ... ) + +It works with ``MyTable.update`` and also when using the ``save`` method on +an existing row. + +------------------------------------------------------------------------------- + 0.95.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index f5df62219..b4ae54b5e 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.95.0" +__VERSION__ = "0.96.0" From 248c209cb3de26181aac907a68552f525dc182f2 Mon Sep 17 00:00:00 2001 From: Miguel Guthridge Date: Fri, 11 Nov 2022 05:52:17 +1100 Subject: [PATCH 398/727] Add error check for order_by mixin columns (#673) * Add error check for order_by mixin columns * fix linter warning, and add simple test Co-authored-by: Daniel Townsend --- piccolo/query/mixins.py | 2 ++ tests/query/test_mixins.py | 19 ++++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index ee36f1f80..2ceb00ba5 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -190,6 +190,8 @@ def get_order_by_columns(self) -> t.List[Column]: return list(self._order_by.columns) if self._order_by else [] def order_by(self, *columns: Column, ascending=True): + if len(columns) < 1: + raise ValueError("At least one column must be passed to order_by.") self._order_by = OrderBy(columns, ascending) diff --git a/tests/query/test_mixins.py b/tests/query/test_mixins.py index dd9110431..9478de668 100644 --- a/tests/query/test_mixins.py +++ b/tests/query/test_mixins.py @@ -1,6 +1,7 @@ import time # For time travel queries. +from unittest import TestCase -from piccolo.query.mixins import ColumnsDelegate +from piccolo.query.mixins import ColumnsDelegate, OrderByDelegate from tests.base import DBTestCase, engines_only from tests.example_apps.music.tables import Band @@ -60,3 +61,19 @@ def test_as_of(self): result = result.run_sync() self.assertTrue(result.name == "Pythonistas") + + +class TestOrderByDelegate(TestCase): + def test_no_columns(self): + """ + An exception should be raised if no columns are passed in. + """ + delegate = OrderByDelegate() + + with self.assertRaises(ValueError) as manager: + delegate.order_by() + + self.assertEqual( + manager.exception.__str__(), + "At least one column must be passed to order_by.", + ) From 54716fe80113f8ee6e4fb71beb52ddf6a89da3e7 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 22 Nov 2022 21:00:11 +0000 Subject: [PATCH 399/727] added `OrderByRaw` (#684) --- docs/src/piccolo/query_clauses/order_by.rst | 52 ++++ piccolo/apps/playground/commands/run.py | 6 + piccolo/query/__init__.py | 23 ++ piccolo/query/methods/objects.py | 16 +- piccolo/query/methods/select.py | 38 ++- piccolo/query/mixins.py | 55 ++++- tests/example_apps/music/tables.py | 35 +-- tests/query/mixins/__init__.py | 0 .../test_columns_delegate.py} | 19 +- tests/query/mixins/test_order_by_delegate.py | 19 ++ tests/table/test_select.py | 230 ++++++++++++++++-- 11 files changed, 404 insertions(+), 89 deletions(-) create mode 100644 tests/query/mixins/__init__.py rename tests/query/{test_mixins.py => mixins/test_columns_delegate.py} (79%) create mode 100644 tests/query/mixins/test_order_by_delegate.py diff --git a/docs/src/piccolo/query_clauses/order_by.rst b/docs/src/piccolo/query_clauses/order_by.rst index f6a66c23b..eb109d2fc 100644 --- a/docs/src/piccolo/query_clauses/order_by.rst +++ b/docs/src/piccolo/query_clauses/order_by.rst @@ -25,6 +25,14 @@ To order by descending: ascending=False ) +You can specify the column name as a string if you prefer: + +.. code-block:: python + + await Band.select().order_by( + 'name' + ) + You can order by multiple columns, and even use joins: .. code-block:: python @@ -33,3 +41,47 @@ You can order by multiple columns, and even use joins: Band.name, Band.manager.name ) + +------------------------------------------------------------------------------- + +Advanced +-------- + +Ascending and descending +~~~~~~~~~~~~~~~~~~~~~~~~ + +If you want to order by multiple columns, with some ascending, and some +descending, then you can do so using multiple ``order_by`` statements: + + +.. code-block:: python + + await Band.select().order_by( + Band.name, + ).order_by( + Band.popularity, + ascending=False + ) + +``OrderByRaw`` +~~~~~~~~~~~~~~ + +SQL's ``ORDER BY`` clause is surprisingly rich in functionality, and there may +be situations where you want to specify the ``ORDER BY`` explicitly using SQL. +To do this use ``OrderByRaw``. + +In the example below, we are ordering the results randomly: + +.. code-block:: python + + from piccolo.query import OrderByRaw + + await Band.select(Band.name).order_by( + OrderByRaw('random()'), + ) + +The above is equivalent to the following SQL: + +.. code-block:: sql + + SELECT "band"."name" FROM band ORDER BY random() ASC diff --git a/piccolo/apps/playground/commands/run.py b/piccolo/apps/playground/commands/run.py index 8d388fd98..092b79c20 100644 --- a/piccolo/apps/playground/commands/run.py +++ b/piccolo/apps/playground/commands/run.py @@ -159,6 +159,12 @@ def populate(): rustaceans = Band(name="Rustaceans", manager=graydon.id, popularity=500) rustaceans.save().run_sync() + anders = Manager(name="Anders") + anders.save().run_sync() + + c_sharps = Band(name="C-Sharps", popularity=700, manager=anders.id) + c_sharps.save().run_sync() + venue = Venue(name="Amazing Venue", capacity=5000) venue.save().run_sync() diff --git a/piccolo/query/__init__.py b/piccolo/query/__init__.py index 410fb0128..8c1a7ccbe 100644 --- a/piccolo/query/__init__.py +++ b/piccolo/query/__init__.py @@ -18,3 +18,26 @@ TableExists, Update, ) +from .mixins import OrderByRaw + +__all__ = [ + "Alter", + "Avg", + "Count", + "Create", + "CreateIndex", + "Delete", + "DropIndex", + "Exists", + "Insert", + "Max", + "Min", + "Objects", + "OrderByRaw", + "Query", + "Raw", + "Select", + "Sum", + "TableExists", + "Update", +] diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index df3e9c508..f71740def 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -15,6 +15,7 @@ LimitDelegate, OffsetDelegate, OrderByDelegate, + OrderByRaw, OutputDelegate, PrefetchDelegate, WhereDelegate, @@ -213,8 +214,19 @@ def get_or_create( def create(self, **columns: t.Any): return Create(table_class=self.table, columns=columns) - def order_by(self, *columns: Column, ascending=True) -> Objects: - self.order_by_delegate.order_by(*columns, ascending=ascending) + def order_by( + self, + *columns: t.Union[Column, str, OrderByRaw], + ascending: bool = True, + ) -> Objects: + _columns: t.List[t.Union[Column, OrderByRaw]] = [] + for column in columns: + if isinstance(column, str): + _columns.append(self.table._meta.get_column_by_name(column)) + else: + _columns.append(column) + + self.order_by_delegate.order_by(*_columns, ascending=ascending) return self def where(self, *where: Combinable) -> Objects: diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 08cdd46c3..772e44af0 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -21,6 +21,7 @@ LimitDelegate, OffsetDelegate, OrderByDelegate, + OrderByRaw, OutputDelegate, WhereDelegate, ) @@ -263,7 +264,7 @@ def distinct(self) -> Select: self.distinct_delegate.distinct() return self - def group_by(self, *columns: Column) -> Select: + def group_by(self, *columns: t.Union[Column, str]) -> Select: _columns: t.List[Column] = [ i for i in self.table._process_column_args(*columns) @@ -448,12 +449,23 @@ async def response_handler(self, response): else: return response - def order_by(self, *columns: Column, ascending=True) -> Select: - _columns: t.List[Column] = [ - i - for i in self.table._process_column_args(*columns) - if isinstance(i, Column) - ] + def order_by( + self, *columns: t.Union[Column, str, OrderByRaw], ascending=True + ) -> Select: + """ + :param columns: + Either a :class:`piccolo.columns.base.Column` instance, a string + representing a column name, or :class:`piccolo.query.OrderByRaw` + which allows you for complex use cases like + ``OrderByRaw('random()')``. + """ + _columns: t.List[t.Union[Column, OrderByRaw]] = [] + for column in columns: + if isinstance(column, str): + _columns.append(self.table._meta.get_column_by_name(column)) + else: + _columns.append(column) + self.order_by_delegate.order_by(*_columns, ascending=ascending) return self @@ -616,7 +628,7 @@ def default_querystrings(self) -> t.Sequence[QueryString]: args: t.List[t.Any] = [] if self.as_of_delegate._as_of: - query += " {}" + query += "{}" args.append(self.as_of_delegate._as_of.querystring) if self.where_delegate._where: @@ -624,11 +636,11 @@ def default_querystrings(self) -> t.Sequence[QueryString]: args.append(self.where_delegate._where.querystring) if self.group_by_delegate._group_by: - query += " {}" + query += "{}" args.append(self.group_by_delegate._group_by.querystring) - if self.order_by_delegate._order_by: - query += " {}" + if self.order_by_delegate._order_by.order_by_items: + query += "{}" args.append(self.order_by_delegate._order_by.querystring) if ( @@ -642,11 +654,11 @@ def default_querystrings(self) -> t.Sequence[QueryString]: ) if self.limit_delegate._limit: - query += " {}" + query += "{}" args.append(self.limit_delegate._limit.querystring) if self.offset_delegate._offset: - query += " {}" + query += "{}" args.append(self.offset_delegate._offset.querystring) querystring = QueryString(query, *args) diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index 2ceb00ba5..36a2fd80f 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import itertools import typing as t from dataclasses import dataclass, field from enum import Enum, auto @@ -74,20 +75,40 @@ def __str__(self) -> str: @dataclass -class OrderBy: +class OrderByRaw: + __slots__ = ("sql",) + + sql: str + + +@dataclass +class OrderByItem: __slots__ = ("columns", "ascending") - columns: t.Sequence[Column] + columns: t.Sequence[t.Union[Column, OrderByRaw]] ascending: bool + +@dataclass +class OrderBy: + order_by_items: t.List[OrderByItem] = field(default_factory=list) + @property def querystring(self) -> QueryString: - order = "ASC" if self.ascending else "DESC" - columns_names = ", ".join( - i._meta.get_full_name(with_alias=False) for i in self.columns - ) + order_by_strings: t.List[str] = [] + for order_by_item in self.order_by_items: + order = "ASC" if order_by_item.ascending else "DESC" + for column in order_by_item.columns: + if isinstance(column, Column): + expression = column._meta.get_full_name(with_alias=False) + elif isinstance(column, OrderByRaw): + expression = column.sql + else: + raise ValueError("Unrecognised order_by") + + order_by_strings.append(f"{expression} {order}") - return QueryString(f" ORDER BY {columns_names} {order}") + return QueryString(f" ORDER BY {', '.join(order_by_strings)}") def __str__(self): return self.querystring.__str__() @@ -184,15 +205,27 @@ def where(self, *where: Combinable): @dataclass class OrderByDelegate: - _order_by: t.Optional[OrderBy] = None + _order_by: OrderBy = field(default_factory=OrderBy) def get_order_by_columns(self) -> t.List[Column]: - return list(self._order_by.columns) if self._order_by else [] + """ + Used to work out which columns are needed for joins. + """ + return [ + i + for i in itertools.chain( + *[i.columns for i in self._order_by.order_by_items] + ) + if isinstance(i, Column) + ] - def order_by(self, *columns: Column, ascending=True): + def order_by(self, *columns: t.Union[Column, OrderByRaw], ascending=True): if len(columns) < 1: raise ValueError("At least one column must be passed to order_by.") - self._order_by = OrderBy(columns, ascending) + + self._order_by.order_by_items.append( + OrderByItem(columns=columns, ascending=ascending) + ) @dataclass diff --git a/tests/example_apps/music/tables.py b/tests/example_apps/music/tables.py index f09f3d8ca..8e2870ea1 100644 --- a/tests/example_apps/music/tables.py +++ b/tests/example_apps/music/tables.py @@ -28,31 +28,18 @@ def get_readable(cls) -> Readable: return Readable(template="%s", columns=[cls.name]) -if engine.engine_type != "cockroach": # type: ignore - - class Band(Table): # type: ignore - name = Varchar(length=50) - manager = ForeignKey(Manager, null=True) - popularity = Integer(default=0) - - @classmethod - def get_readable(cls) -> Readable: - return Readable(template="%s", columns=[cls.name]) - -else: - - class Band(Table): # type: ignore - """ - Special version for Cockroach. - """ - - name = Varchar(length=50) - manager = ForeignKey(Manager, null=True) - popularity = BigInt(default=0) +class Band(Table): + name = Varchar(length=50) + manager = ForeignKey(Manager, null=True) + popularity = ( + BigInt(default=0) + if engine and engine.engine_type == "cockroach" + else Integer(default=0) + ) - @classmethod - def get_readable(cls) -> Readable: - return Readable(template="%s", columns=[cls.name]) + @classmethod + def get_readable(cls) -> Readable: + return Readable(template="%s", columns=[cls.name]) ############################################################################### diff --git a/tests/query/mixins/__init__.py b/tests/query/mixins/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/query/test_mixins.py b/tests/query/mixins/test_columns_delegate.py similarity index 79% rename from tests/query/test_mixins.py rename to tests/query/mixins/test_columns_delegate.py index 9478de668..dd9110431 100644 --- a/tests/query/test_mixins.py +++ b/tests/query/mixins/test_columns_delegate.py @@ -1,7 +1,6 @@ import time # For time travel queries. -from unittest import TestCase -from piccolo.query.mixins import ColumnsDelegate, OrderByDelegate +from piccolo.query.mixins import ColumnsDelegate from tests.base import DBTestCase, engines_only from tests.example_apps.music.tables import Band @@ -61,19 +60,3 @@ def test_as_of(self): result = result.run_sync() self.assertTrue(result.name == "Pythonistas") - - -class TestOrderByDelegate(TestCase): - def test_no_columns(self): - """ - An exception should be raised if no columns are passed in. - """ - delegate = OrderByDelegate() - - with self.assertRaises(ValueError) as manager: - delegate.order_by() - - self.assertEqual( - manager.exception.__str__(), - "At least one column must be passed to order_by.", - ) diff --git a/tests/query/mixins/test_order_by_delegate.py b/tests/query/mixins/test_order_by_delegate.py new file mode 100644 index 000000000..7d2f2c6c4 --- /dev/null +++ b/tests/query/mixins/test_order_by_delegate.py @@ -0,0 +1,19 @@ +from unittest import TestCase + +from piccolo.query.mixins import OrderByDelegate + + +class TestOrderByDelegate(TestCase): + def test_no_columns(self): + """ + An exception should be raised if no columns are passed in. + """ + delegate = OrderByDelegate() + + with self.assertRaises(ValueError) as manager: + delegate.order_by() + + self.assertEqual( + manager.exception.__str__(), + "At least one column must be passed to order_by.", + ) diff --git a/tests/table/test_select.py b/tests/table/test_select.py index 247236026..af287aad0 100644 --- a/tests/table/test_select.py +++ b/tests/table/test_select.py @@ -2,7 +2,9 @@ from piccolo.apps.user.tables import BaseUser from piccolo.columns.combination import WhereRaw +from piccolo.query import OrderByRaw from piccolo.query.methods.select import Avg, Count, Max, Min, Sum +from piccolo.table import create_db_tables_sync, drop_db_tables_sync from tests.base import ( DBTestCase, engine_is, @@ -486,27 +488,6 @@ def test_first(self): self.assertEqual(response, {"name": "CSharps"}) - def test_order_by_ascending(self): - self.insert_rows() - - response = ( - Band.select(Band.name).order_by(Band.name).limit(1).run_sync() - ) - - self.assertEqual(response, [{"name": "CSharps"}]) - - def test_order_by_decending(self): - self.insert_rows() - - response = ( - Band.select(Band.name) - .order_by(Band.name, ascending=False) - .limit(1) - .run_sync() - ) - - self.assertEqual(response, [{"name": "Rustaceans"}]) - def test_count(self): self.insert_rows() @@ -1022,3 +1003,210 @@ def test_secret_parameter(self): else: self.assertTrue(venue_dict, {"id": 1, "name": "The Garage"}) self.assertNotIn("capacity", venue_dict.keys()) + + +class TestSelectOrderBy(TestCase): + """ + We use TestCase, rather than DBTestCase, as we want a lot of data to test + with. + """ + + def setUp(self): + """ + Create tables and lots of test data. + """ + create_db_tables_sync(Band, Manager) + + data = [ + { + "band_name": "Pythonistas", + "manager_name": "Guido", + "popularity": 1000, + }, + { + "band_name": "Rustaceans", + "manager_name": "Graydon", + "popularity": 800, + }, + { + "band_name": "C-Sharps", + "manager_name": "Anders", + "popularity": 800, + }, + { + "band_name": "Rubyists", + "manager_name": "Matz", + "popularity": 820, + }, + ] + + for item in data: + manager = ( + Manager.objects().create(name=item["manager_name"]).run_sync() + ) + + Band.objects().create( + name=item["band_name"], + manager=manager, + popularity=item["popularity"], + ).run_sync() + + def tearDown(self): + drop_db_tables_sync(Band, Manager) + + def test_ascending(self): + response = Band.select(Band.name).order_by(Band.name).run_sync() + + self.assertEqual( + response, + [ + {"name": "C-Sharps"}, + {"name": "Pythonistas"}, + {"name": "Rubyists"}, + {"name": "Rustaceans"}, + ], + ) + + def test_descending(self): + response = ( + Band.select(Band.name) + .order_by(Band.name, ascending=False) + .run_sync() + ) + + self.assertEqual( + response, + [ + {"name": "Rustaceans"}, + {"name": "Rubyists"}, + {"name": "Pythonistas"}, + {"name": "C-Sharps"}, + ], + ) + + def test_string(self): + """ + Make sure strings can be used to identify columns if the user prefers. + """ + response = Band.select(Band.name).order_by("name").run_sync() + + self.assertEqual( + response, + [ + {"name": "C-Sharps"}, + {"name": "Pythonistas"}, + {"name": "Rubyists"}, + {"name": "Rustaceans"}, + ], + ) + + def test_string_unrecognised(self): + """ + Make sure an unrecognised column name raises an Exception. + """ + with self.assertRaises(ValueError) as manager: + Band.select(Band.name).order_by("foo") + + self.assertEqual( + manager.exception.__str__(), + "No matching column found with name == foo", + ) + + def test_multiple_columns_ascending(self): + """ + Make sure we can order by multiple columns. + """ + response = ( + Band.select(Band.popularity, Band.name) + .order_by(Band.popularity, Band.name) + .run_sync() + ) + + self.assertEqual( + response, + [ + {"popularity": 800, "name": "C-Sharps"}, + {"popularity": 800, "name": "Rustaceans"}, + {"popularity": 820, "name": "Rubyists"}, + {"popularity": 1000, "name": "Pythonistas"}, + ], + ) + + def test_multiple_columns_descending(self): + """ + Make sure we can order by multiple columns, descending. + """ + response = ( + Band.select(Band.popularity, Band.name) + .order_by(Band.popularity, Band.name, ascending=False) + .run_sync() + ) + + self.assertEqual( + response, + [ + {"popularity": 1000, "name": "Pythonistas"}, + {"popularity": 820, "name": "Rubyists"}, + {"popularity": 800, "name": "Rustaceans"}, + {"popularity": 800, "name": "C-Sharps"}, + ], + ) + + def test_join(self): + """ + Make sure that we can order using columns in related tables. + """ + response = ( + Band.select(Band.manager.name.as_alias("manager_name"), Band.name) + .order_by(Band.manager.name) + .run_sync() + ) + self.assertEqual( + response, + [ + {"manager_name": "Anders", "name": "C-Sharps"}, + {"manager_name": "Graydon", "name": "Rustaceans"}, + {"manager_name": "Guido", "name": "Pythonistas"}, + {"manager_name": "Matz", "name": "Rubyists"}, + ], + ) + + def test_ascending_descending(self): + """ + Make sure we can combine ascending and descending. + """ + response = ( + Band.select(Band.popularity, Band.name) + .order_by(Band.popularity) + .order_by(Band.name, ascending=False) + .run_sync() + ) + + self.assertEqual( + response, + [ + {"popularity": 800, "name": "Rustaceans"}, + {"popularity": 800, "name": "C-Sharps"}, + {"popularity": 820, "name": "Rubyists"}, + {"popularity": 1000, "name": "Pythonistas"}, + ], + ) + + def test_order_by_raw(self): + """ + Maker sure ``OrderByRaw`` can be used, to order by anything the user + wants. + """ + response = ( + Band.select(Band.name).order_by(OrderByRaw("name")).run_sync() + ) + + self.assertEqual( + response, + [ + {"name": "C-Sharps"}, + {"name": "Pythonistas"}, + {"name": "Rubyists"}, + {"name": "Rustaceans"}, + ], + ) From fafde35bccc3da6bd7cb49de4d960ca5832122af Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 22 Nov 2022 21:06:27 +0000 Subject: [PATCH 400/727] bumped version --- CHANGES.rst | 35 ++++++++++++++++++++++++++++++++++- piccolo/__init__.py | 2 +- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 0a51c3030..da9cd73fa 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,39 @@ Changes ======= +0.97.0 +------ + +Some big improvements to ``order_by`` clauses. + +It's now possible to combine ascending and descending: + +.. code-block:: python + + await Band.select( + Band.name, + Band.popularity + ).order_by( + Band.name + ).order_by( + Band.popularity, + ascending=False + ) + +You can also order by anything you want using ``OrderByRaw``: + +.. code-block:: python + + from piccolo.query import OrderByRaw + + await Band.select( + Band.name + ).order_by( + OrderByRaw('random()') + ) + +------------------------------------------------------------------------------- + 0.96.0 ------ @@ -18,7 +51,7 @@ time the row is saved. default=None, auto_update=datetime.datetime.now ) - + # The `modified_on` column will automatically be updated to the current # timestamp: >>> await Band.update({ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index b4ae54b5e..741bf5914 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.96.0" +__VERSION__ = "0.97.0" From f3db5b3ac1dcb9bae22d7487ed92b3b7f693d74b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 22 Nov 2022 21:34:44 +0000 Subject: [PATCH 401/727] update cockroachdb and python version (#686) --- .github/workflows/tests.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 827be6f65..32a64a259 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -2,7 +2,7 @@ name: Test Suite on: push: - branches: ["master", "cockroachdb"] + branches: ["master"] pull_request: branches: ["master"] @@ -12,7 +12,7 @@ jobs: timeout-minutes: 30 strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 @@ -137,8 +137,8 @@ jobs: timeout-minutes: 30 strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] - cockroachdb-version: ["v22.2.0-beta.2"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + cockroachdb-version: ["v22.2.0-rc.3"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -171,7 +171,7 @@ jobs: timeout-minutes: 30 strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 From f8adb0456cf7deaaf04a07ca0266be60b7c68b79 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 30 Nov 2022 13:05:35 +0000 Subject: [PATCH 402/727] fix Sphinx's duplicate target warning (#696) --- docs/src/piccolo/projects_and_apps/included_apps.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/src/piccolo/projects_and_apps/included_apps.rst b/docs/src/piccolo/projects_and_apps/included_apps.rst index a2590dd05..5762803ae 100644 --- a/docs/src/piccolo/projects_and_apps/included_apps.rst +++ b/docs/src/piccolo/projects_and_apps/included_apps.rst @@ -168,7 +168,7 @@ Here's an example of a generated image: .. note:: - There is a `video tutorial on YouTube `_. + There is a `video tutorial on YouTube `__. ------------------------------------------------------------------------------- @@ -184,7 +184,7 @@ Launches an iPython shell, and automatically imports all of your registered .. note:: - There is a `video tutorial on YouTube `_. + There is a `video tutorial on YouTube `__. ------------------------------------------------------------------------------- @@ -204,7 +204,7 @@ or ``sqlite3`` depending on which you're using). .. note:: - There is a `video tutorial on YouTube `_. + There is a `video tutorial on YouTube `__. ------------------------------------------------------------------------------- From 1f036ad37fc5290412809988537dc606bbaf3cf6 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Wed, 30 Nov 2022 14:24:25 +0100 Subject: [PATCH 403/727] Enabled camelCase column name (#693) * Enabled camelCase column name * added some basic tests for camelCase column names Co-authored-by: Daniel Townsend --- piccolo/query/methods/select.py | 4 +-- tests/query/test_camelcase.py | 61 +++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 tests/query/test_camelcase.py diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 772e44af0..7f10e004d 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -555,9 +555,9 @@ def _get_joins(self, columns: t.Sequence[Selectable]) -> t.List[str]: ]._foreign_key_meta.resolved_target_column._meta.name _joins.append( - f"LEFT JOIN {right_tablename} {table_alias}" + f'LEFT JOIN "{right_tablename}" "{table_alias}"' " ON " - f"({left_tablename}.{key._meta.name} = {table_alias}.{pk_name})" # noqa: E501 + f'("{left_tablename}"."{key._meta.name}" = "{table_alias}"."{pk_name}")' # noqa: E501 ) joins.extend(_joins) diff --git a/tests/query/test_camelcase.py b/tests/query/test_camelcase.py new file mode 100644 index 000000000..3cbc6cf04 --- /dev/null +++ b/tests/query/test_camelcase.py @@ -0,0 +1,61 @@ +from unittest import TestCase + +from piccolo.columns import ForeignKey, Varchar +from piccolo.table import Table, create_db_tables_sync, drop_db_tables_sync + + +class Manager(Table): + theName = Varchar() + + +class Band(Table): + theName = Varchar() + theManager = ForeignKey(Manager) + + +class TestCamelCase(TestCase): + def setUp(self): + create_db_tables_sync(Manager, Band) + + def tearDown(self): + drop_db_tables_sync(Manager, Band) + + def test_queries(self): + """ + Make sure that basic queries work when the columns use camelCase. + """ + manager_names = ("Guido", "Maz", "Graydon") + band_names = ("Pythonistas", "Rubyists", "Rustaceans") + + # Test create + for manager_name, band_name in zip(manager_names, band_names): + manager = Manager.objects().create(theName=manager_name).run_sync() + Band.objects().create( + theName=band_name, theManager=manager + ).run_sync() + + # Test select, with joins + response = ( + Band.select( + Band.theName, + Band.theManager.theName.as_alias("theManagerName"), + ) + .order_by(Band.theName) + .run_sync() + ) + self.assertListEqual( + response, + [ + {"theName": "Pythonistas", "theManagerName": "Guido"}, + {"theName": "Rubyists", "theManagerName": "Maz"}, + {"theName": "Rustaceans", "theManagerName": "Graydon"}, + ], + ) + + # Test delete + Band.delete().where(Band.theName == "Rubyists").run_sync() + + # Test exists + self.assertFalse( + Band.exists().where(Band.theName == "Rubyists").run_sync() + ) From 6a5ef4ebd70540ab2b0b55300e94a6b441487921 Mon Sep 17 00:00:00 2001 From: StitiFatah <68331131+StitiFatah@users.noreply.github.com> Date: Thu, 1 Dec 2022 23:08:20 +0100 Subject: [PATCH 404/727] fix typo for `raw` queries in docs (#699) --- docs/src/piccolo/query_types/raw.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/piccolo/query_types/raw.rst b/docs/src/piccolo/query_types/raw.rst index 71c295855..11ed86781 100644 --- a/docs/src/piccolo/query_types/raw.rst +++ b/docs/src/piccolo/query_types/raw.rst @@ -7,7 +7,7 @@ Should you need to, you can execute raw SQL. .. code-block:: python - >>> await Band.raw('select name from band').run_sync() + >>> await Band.raw('select name from band') [{'name': 'Pythonistas'}] It's recommended that you parameterise any values. Use curly braces ``{}`` as From 13c9d6948613baab44a2e5b993b0709622a465db Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 2 Dec 2022 11:08:45 +0000 Subject: [PATCH 405/727] add `TransactionType` to `SQLiteEngine` (#688) * added ``TransactionType`` * make `deferred` the default, to match SQLite's default behaviour * add docs for SQLite transaction types * added `TestTransactionType` Making sure that SQLite transaction types work correctly. * transaction and atomic improvements * add test for running `add_m2m` inside a transaction --- docs/src/piccolo/engines/sqlite_engine.rst | 8 + docs/src/piccolo/query_types/transactions.rst | 22 +++ docs/src/piccolo/tutorials/index.rst | 1 + .../using_sqlite_and_asyncio_effectively.rst | 103 ++++++++++ piccolo/columns/m2m.py | 63 +++--- piccolo/engine/base.py | 23 ++- piccolo/engine/cockroach.py | 86 +------- piccolo/engine/postgres.py | 91 ++++----- piccolo/engine/sqlite.py | 185 ++++++++++-------- tests/columns/test_m2m.py | 36 ++++ tests/engine/test_transaction.py | 105 +++++++++- 11 files changed, 483 insertions(+), 240 deletions(-) create mode 100644 docs/src/piccolo/tutorials/using_sqlite_and_asyncio_effectively.rst diff --git a/docs/src/piccolo/engines/sqlite_engine.rst b/docs/src/piccolo/engines/sqlite_engine.rst index 7397edca5..bcd5869e9 100644 --- a/docs/src/piccolo/engines/sqlite_engine.rst +++ b/docs/src/piccolo/engines/sqlite_engine.rst @@ -23,3 +23,11 @@ Source .. currentmodule:: piccolo.engine.sqlite .. autoclass:: SQLiteEngine + +------------------------------------------------------------------------------- + +Production tips +--------------- + +If you're planning on using SQLite in production with Piccolo, with lots of +concurrent queries, then here are some :ref:`useful tips `. diff --git a/docs/src/piccolo/query_types/transactions.rst b/docs/src/piccolo/query_types/transactions.rst index 4c72b073c..3db6c854f 100644 --- a/docs/src/piccolo/query_types/transactions.rst +++ b/docs/src/piccolo/query_types/transactions.rst @@ -47,3 +47,25 @@ async. If an exception is raised within the body of the context manager, then the transaction is automatically rolled back. The exception is still propagated though. + +``transaction_exists`` +~~~~~~~~~~~~~~~~~~~~~~ + +You can check whether your code is currently inside a transaction using the +following: + +.. code-block:: python + + >>> Band._meta.db.transaction_exists() + True + +------------------------------------------------------------------------------- + +Transaction types +----------------- + +SQLite +~~~~~~ + +For SQLite you may want to specify the :ref:`transaction type `, +as it can have an effect on how well the database handles concurrent requests. diff --git a/docs/src/piccolo/tutorials/index.rst b/docs/src/piccolo/tutorials/index.rst index 87ccf2ede..c4228b2f1 100644 --- a/docs/src/piccolo/tutorials/index.rst +++ b/docs/src/piccolo/tutorials/index.rst @@ -8,3 +8,4 @@ help you solve common problems: :maxdepth: 1 ./migrate_existing_project + ./using_sqlite_and_asyncio_effectively diff --git a/docs/src/piccolo/tutorials/using_sqlite_and_asyncio_effectively.rst b/docs/src/piccolo/tutorials/using_sqlite_and_asyncio_effectively.rst new file mode 100644 index 000000000..67014dd5e --- /dev/null +++ b/docs/src/piccolo/tutorials/using_sqlite_and_asyncio_effectively.rst @@ -0,0 +1,103 @@ +.. _UsingSQLitAndAsyncioEffectively: + +Using SQLite and asyncio effectively +==================================== + +When using Piccolo with SQLite, there are some best practices to follow. + +asyncio => lots of connections +------------------------------ + +With asyncio, we can potentially open lots of database connections, and attempt +to perform concurrent database writes. + +SQLite doesn't support such concurrent behavior as effectively as Postgres, so +we need to be careful. + +One write at a time +~~~~~~~~~~~~~~~~~~~ + +SQLite can easily support lots of transactions concurrently if they are reading, +but only one write can be performed at a time. + +------------------------------------------------------------------------------- + +.. _SQLiteTransactionTypes: + +Transactions +------------ + +SQLite has several transaction types, as specified by Piccolo's +``TransactionType`` enum: + +.. currentmodule:: piccolo.engine.sqlite + +.. autoclass:: TransactionType + :members: + :undoc-members: + +Which to use? +~~~~~~~~~~~~~ + +When creating a transaction, Piccolo uses ``DEFERRED`` by default (to be +consistent with SQLite). + +This means that the first SQL query executed within the transaction determines +whether it's a **READ** or **WRITE**: + +* **READ** - if the first query is a ``SELECT`` +* **WRITE** - if the first query is something like an ``INSERT`` / ``UPDATE`` / ``DELETE`` + +If a transaction starts off with a ``SELECT``, but then tries to perform an ``INSERT`` / ``UPDATE`` / ``DELETE``, +SQLite tries to 'promote' the transaction so it can write. + +The problem is, if multiple concurrent connections try doing this at the same time, +SQLite will return a database locked error. + +So if you're creating a transaction which you know will perform writes, then +create an ``IMMEDIATE`` transaction: + +.. code-block:: python + + from piccolo.engine.sqlite import TransactionType + + async with Band._meta.db.transaction( + transaction_type=TransactionType.immediate + ): + # We perform a SELECT first, but as it's an IMMEDIATE transaction, + # we can later perform writes without getting a database locked + # error. + if not await Band.exists().where(Band.name == 'Pythonistas'): + await Band.objects().create(name="Pythonistas") + +Multiple ``IMMEDIATE`` transactions can exist concurrently - SQLite uses a lock +to make sure only one of them writes at a time. + +If your transaction will just be performing ``SELECT`` queries, then just use +the default ``DEFERRED`` transactions - you will get improved performance, as +no locking is involved: + +.. code-block:: python + + async with Band._meta.db.transaction(): + bands = await Band.select() + managers = await Manager.select() + +------------------------------------------------------------------------------- + +timeout +------- + +It's recommended to specify the ``timeout`` argument in :class:`SQLiteEngine `. + +.. code-block:: python + + DB = SQLiteEngine(timeout=60) + +Imagine you have a web app, and each endpoint creates a transaction which runs +multiple queries. With SQLite, only a single write operation can happen at a +time, so if several connections are open, they may be queued for a while. + +By increasing ``timeout`` it means that queries are less likely to timeout. + +To find out more about ``timeout`` see the Python :func:`sqlite3 docs `. diff --git a/piccolo/columns/m2m.py b/piccolo/columns/m2m.py index 6cd5a752c..5c696bd2b 100644 --- a/piccolo/columns/m2m.py +++ b/piccolo/columns/m2m.py @@ -254,39 +254,50 @@ def __post_init__(self): for i, j in self.extra_column_values.items() } - async def run(self): + async def _run(self): rows = self.rows unsaved = [i for i in rows if not i._exists_in_db] - async with rows[0]._meta.db.transaction(): - if unsaved: - await rows[0].__class__.insert(*unsaved).run() + if unsaved: + await rows[0].__class__.insert(*unsaved).run() - joining_table = self.m2m._meta.resolved_joining_table + joining_table = self.m2m._meta.resolved_joining_table - joining_table_rows = [] + joining_table_rows = [] - for row in rows: - joining_table_row = joining_table(**self.extra_column_values) - setattr( - joining_table_row, - self.m2m._meta.primary_foreign_key._meta.name, - getattr( - self.target_row, - self.target_row._meta.primary_key._meta.name, - ), - ) - setattr( - joining_table_row, - self.m2m._meta.secondary_foreign_key._meta.name, - getattr( - row, - row._meta.primary_key._meta.name, - ), - ) - joining_table_rows.append(joining_table_row) + for row in rows: + joining_table_row = joining_table(**self.extra_column_values) + setattr( + joining_table_row, + self.m2m._meta.primary_foreign_key._meta.name, + getattr( + self.target_row, + self.target_row._meta.primary_key._meta.name, + ), + ) + setattr( + joining_table_row, + self.m2m._meta.secondary_foreign_key._meta.name, + getattr( + row, + row._meta.primary_key._meta.name, + ), + ) + joining_table_rows.append(joining_table_row) + + return await joining_table.insert(*joining_table_rows).run() - return await joining_table.insert(*joining_table_rows).run() + async def run(self): + """ + Run the queries, making sure they are either within an existing + transaction, or wrapped in a new transaction. + """ + engine = self.rows[0]._meta.db + if engine.transaction_exists(): + await self._run() + else: + async with engine.transaction(): + await self._run() def run_sync(self): return run_sync(self.run()) diff --git a/piccolo/engine/base.py b/piccolo/engine/base.py index 7d55c3a37..f1b43ebc7 100644 --- a/piccolo/engine/base.py +++ b/piccolo/engine/base.py @@ -1,5 +1,6 @@ from __future__ import annotations +import contextvars import logging import typing as t from abc import ABCMeta, abstractmethod @@ -19,7 +20,10 @@ class Batch: pass -class Engine(metaclass=ABCMeta): +TransactionClass = t.TypeVar("TransactionClass") + + +class Engine(t.Generic[TransactionClass], metaclass=ABCMeta): __slots__ = () @@ -116,3 +120,20 @@ async def close_connection_pool(self): The database driver doesn't implement connection pooling. """ self._connection_pool_warning() + + ########################################################################### + + current_transaction: contextvars.ContextVar[t.Optional[TransactionClass]] + + def transaction_exists(self) -> bool: + """ + Find out if a transaction is currently active. + + :returns: + ``True`` if a transaction is already active for the current + asyncio task. This is useful to know, because nested transactions + aren't currently supported, so you can check if an existing + transaction is already active, before creating a new one. + + """ + return self.current_transaction.get() is not None diff --git a/piccolo/engine/cockroach.py b/piccolo/engine/cockroach.py index 59386335f..c12586255 100644 --- a/piccolo/engine/cockroach.py +++ b/piccolo/engine/cockroach.py @@ -1,80 +1,19 @@ from __future__ import annotations -import contextvars import typing as t -from piccolo.engine.exceptions import TransactionError -from piccolo.query.base import Query from piccolo.utils.lazy_loader import LazyLoader from piccolo.utils.warnings import Level, colored_warning -from .postgres import Atomic as PostgresAtomic from .postgres import PostgresEngine -from .postgres import Transaction as PostgresTransaction asyncpg = LazyLoader("asyncpg", globals(), "asyncpg") -if t.TYPE_CHECKING: # pragma: no cover - from asyncpg.pool import Pool - - -############################################################################### - - -class Atomic(PostgresAtomic): - """ - This is useful if you want to build up a transaction programatically, by - adding queries to it. - - Usage:: - - transaction = engine.atomic() - transaction.add(Foo.create_table()) - - # Either: - transaction.run_sync() - await transaction.run() - - """ - - def __init__(self, engine: CockroachEngine): - self.engine = engine - self.queries: t.List[Query] = [] - super(Atomic, self).__init__(engine) - - -############################################################################### - - -class Transaction(PostgresTransaction): - """ - Used for wrapping queries in a transaction, using a context manager. - Currently it's async only. - - Usage:: - - async with engine.transaction(): - # Run some queries: - await Band.select().run() - - """ - - def __init__(self, engine: CockroachEngine): - self.engine = engine - if self.engine.transaction_connection.get(): - raise TransactionError( - "A transaction is already active - nested transactions aren't " - "currently supported." - ) - super(Transaction, self).__init__(engine) - - -############################################################################### - class CockroachEngine(PostgresEngine): """ - An extension of the cockroach backend. + An extension of + :class:`PostgresEngine `. """ engine_type = "cockroach" @@ -85,23 +24,14 @@ def __init__( config: t.Dict[str, t.Any], extensions: t.Sequence[str] = (), log_queries: bool = False, - extra_nodes: t.Dict[str, PostgresEngine] = None, + extra_nodes: t.Dict[str, CockroachEngine] = None, ) -> None: - if extra_nodes is None: - extra_nodes = {} - - self.config = config - self.extensions = extensions - self.log_queries = log_queries - self.extra_nodes = extra_nodes - self.pool: t.Optional[Pool] = None - database_name = config.get("database", "Unknown") - self.transaction_connection = contextvars.ContextVar( - f"pg_transaction_connection_{database_name}", default=None + super().__init__( + config=config, + extensions=extensions, + log_queries=log_queries, + extra_nodes=extra_nodes, ) - super( - PostgresEngine, self - ).__init__() # lgtm[py/super-not-enclosing-class] async def prep_database(self): try: diff --git a/piccolo/engine/postgres.py b/piccolo/engine/postgres.py index 5a3a263da..c8f91f430 100644 --- a/piccolo/engine/postgres.py +++ b/piccolo/engine/postgres.py @@ -7,6 +7,7 @@ from piccolo.engine.base import Batch, Engine from piccolo.engine.exceptions import TransactionError from piccolo.query.base import DDL, Query +from piccolo.query.methods.objects import Create, GetOrCreate from piccolo.querystring import QueryString from piccolo.utils.lazy_loader import LazyLoader from piccolo.utils.sync import run_sync @@ -98,52 +99,30 @@ def __init__(self, engine: PostgresEngine): def add(self, *query: Query): self.queries += list(query) - async def _run_queries(self, connection): - async with connection.transaction(): - for query in self.queries: - if isinstance(query, Query): - for querystring in query.querystrings: - _query, args = querystring.compile_string( - engine_type=self.engine.engine_type - ) - await connection.execute(_query, *args) - elif isinstance(query, DDL): - for ddl in query.ddl: - await connection.execute(ddl) - - self.queries = [] - - async def _run_in_pool(self): - if not self.engine.pool: - raise ValueError("No pool is currently active.") - - async with self.engine.pool.acquire() as connection: - await self._run_queries(connection) - - async def _run_in_new_connection(self): - connection = await asyncpg.connect(**self.engine.config) + async def run(self): try: - await self._run_queries(connection) - except asyncpg.exceptions.PostgresError as exception: - await connection.close() - raise exception - - await connection.close() - - async def run(self, in_pool=True): - if in_pool and self.engine.pool: - await self._run_in_pool() - else: - await self._run_in_new_connection() + async with self.engine.transaction(): + for query in self.queries: + if isinstance(query, (Query, DDL, Create, GetOrCreate)): + await query.run() + else: + raise ValueError("Unrecognised query") + self.queries = [] + except Exception as exception: + self.queries = [] + raise exception from exception def run_sync(self): - return run_sync(self._run_in_new_connection()) + return run_sync(self.run()) + + def __await__(self): + return self.run().__await__() ############################################################################### -class Transaction: +class PostgresTransaction: """ Used for wrapping queries in a transaction, using a context manager. Currently it's async only. @@ -160,7 +139,7 @@ class Transaction: def __init__(self, engine: PostgresEngine): self.engine = engine - if self.engine.transaction_connection.get(): + if self.engine.current_transaction.get(): raise TransactionError( "A transaction is already active - nested transactions aren't " "currently supported." @@ -174,7 +153,7 @@ async def __aenter__(self): self.transaction = self.connection.transaction() await self.transaction.start() - self.context = self.engine.transaction_connection.set(self.connection) + self.context = self.engine.current_transaction.set(self) async def commit(self): await self.transaction.commit() @@ -193,7 +172,7 @@ async def __aexit__(self, exception_type, exception, traceback): else: await self.connection.close() - self.engine.transaction_connection.reset(self.context) + self.engine.current_transaction.reset(self.context) return exception is None @@ -201,9 +180,9 @@ async def __aexit__(self, exception_type, exception, traceback): ############################################################################### -class PostgresEngine(Engine): +class PostgresEngine(Engine[t.Optional[PostgresTransaction]]): """ - Used to connect to Postgresql. + Used to connect to PostgreSQL. :param config: The config dictionary is passed to the underlying database adapter, @@ -262,7 +241,7 @@ class PostgresEngine(Engine): "log_queries", "extra_nodes", "pool", - "transaction_connection", + "current_transaction", ) engine_type = "postgres" @@ -273,7 +252,7 @@ def __init__( config: t.Dict[str, t.Any], extensions: t.Sequence[str] = ("uuid-ossp",), log_queries: bool = False, - extra_nodes: t.Dict[str, PostgresEngine] = None, + extra_nodes: t.Mapping[str, PostgresEngine] = None, ) -> None: if extra_nodes is None: extra_nodes = {} @@ -284,8 +263,8 @@ def __init__( self.extra_nodes = extra_nodes self.pool: t.Optional[Pool] = None database_name = config.get("database", "Unknown") - self.transaction_connection = contextvars.ContextVar( - f"pg_transaction_connection_{database_name}", default=None + self.current_transaction = contextvars.ContextVar( + f"pg_current_transaction_{database_name}", default=None ) super().__init__() @@ -451,9 +430,11 @@ async def run_querystring( print(querystring) # If running inside a transaction: - connection = self.transaction_connection.get() - if connection: - return await connection.fetch(query, *query_args) + current_transaction = self.current_transaction.get() + if current_transaction: + return await current_transaction.connection.fetch( + query, *query_args + ) elif in_pool and self.pool: return await self._run_in_pool(query, query_args) else: @@ -464,9 +445,9 @@ async def run_ddl(self, ddl: str, in_pool: bool = True): print(ddl) # If running inside a transaction: - connection = self.transaction_connection.get() - if connection: - return await connection.fetch(ddl) + current_transaction = self.current_transaction.get() + if current_transaction: + return await current_transaction.connection.fetch(ddl) elif in_pool and self.pool: return await self._run_in_pool(ddl) else: @@ -475,5 +456,5 @@ async def run_ddl(self, ddl: str, in_pool: bool = True): def atomic(self) -> Atomic: return Atomic(engine=self) - def transaction(self) -> Transaction: - return Transaction(engine=self) + def transaction(self) -> PostgresTransaction: + return PostgresTransaction(engine=self) diff --git a/piccolo/engine/sqlite.py b/piccolo/engine/sqlite.py index f7fae03e2..77780e141 100644 --- a/piccolo/engine/sqlite.py +++ b/piccolo/engine/sqlite.py @@ -2,6 +2,7 @@ import contextvars import datetime +import enum import os import sqlite3 import typing as t @@ -12,6 +13,7 @@ from piccolo.engine.base import Batch, Engine from piccolo.engine.exceptions import TransactionError from piccolo.query.base import DDL, Query +from piccolo.query.methods.objects import Create, GetOrCreate from piccolo.querystring import QueryString from piccolo.utils.encoding import dump_json, load_json from piccolo.utils.lazy_loader import LazyLoader @@ -238,6 +240,17 @@ async def __aexit__(self, exception_type, exception, traceback): ############################################################################### +class TransactionType(enum.Enum): + """ + See the `SQLite `_ docs for + more info. + """ + + deferred = "DEFERRED" + immediate = "IMMEDIATE" + exclusive = "EXCLUSIVE" + + class Atomic: """ Usage: @@ -250,67 +263,76 @@ class Atomic: await transaction.run() """ - __slots__ = ("engine", "queries") + __slots__ = ("engine", "queries", "transaction_type") - def __init__(self, engine: SQLiteEngine): + def __init__( + self, + engine: SQLiteEngine, + transaction_type: TransactionType = TransactionType.deferred, + ): self.engine = engine + self.transaction_type = transaction_type self.queries: t.List[Query] = [] def add(self, *query: Query): self.queries += list(query) async def run(self): - connection = await self.engine.get_connection() - await connection.execute("BEGIN") - try: - for query in self.queries: - if isinstance(query, Query): - for querystring in query.querystrings: - await connection.execute( - *querystring.compile_string( - engine_type=self.engine.engine_type - ) - ) - elif isinstance(query, DDL): - for ddl in query.ddl: - await connection.execute(ddl) - + async with self.engine.transaction( + transaction_type=self.transaction_type + ): + for query in self.queries: + if isinstance(query, (Query, DDL, Create, GetOrCreate)): + await query.run() + else: + raise ValueError("Unrecognised query") + self.queries = [] except Exception as exception: - await connection.execute("ROLLBACK") - await connection.close() self.queries = [] raise exception from exception - else: - await connection.execute("COMMIT") - await connection.close() - self.queries = [] def run_sync(self): return run_sync(self.run()) + def __await__(self): + return self.run().__await__() + ############################################################################### -class Transaction: +class SQLiteTransaction: """ Used for wrapping queries in a transaction, using a context manager. Currently it's async only. - Usage: + Usage:: - async with engine.transaction(): - # Run some queries: - await Band.select().run() + async with engine.transaction(): + # Run some queries: + await Band.select().run() """ - __slots__ = ("engine", "context", "connection") + __slots__ = ("engine", "context", "connection", "transaction_type") - def __init__(self, engine: SQLiteEngine): + def __init__( + self, + engine: SQLiteEngine, + transaction_type: TransactionType = TransactionType.deferred, + ): + """ + :param transaction_type: + If your transaction just contains ``SELECT`` queries, then use + ``TransactionType.deferred``. This will give you the best + performance. When performing ``INSERT``, ``UPDATE``, ``DELETE`` + queries, we recommend using ``TransactionType.immediate`` to + avoid database locks. + """ self.engine = engine - if self.engine.transaction_connection.get(): + self.transaction_type = transaction_type + if self.engine.current_transaction.get(): raise TransactionError( "A transaction is already active - nested transactions aren't " "currently supported." @@ -318,8 +340,8 @@ def __init__(self, engine: SQLiteEngine): async def __aenter__(self): self.connection = await self.engine.get_connection() - await self.connection.execute("BEGIN") - self.context = self.engine.transaction_connection.set(self.connection) + await self.connection.execute(f"BEGIN {self.transaction_type.value}") + self.context = self.engine.current_transaction.set(self) async def __aexit__(self, exception_type, exception, traceback): if exception: @@ -328,7 +350,7 @@ async def __aexit__(self, exception_type, exception, traceback): await self.connection.execute("COMMIT") await self.connection.close() - self.engine.transaction_connection.reset(self.context) + self.engine.current_transaction.reset(self.context) return exception is None @@ -340,20 +362,9 @@ def dict_factory(cursor, row) -> t.Dict: return {col[0]: row[idx] for idx, col in enumerate(cursor.description)} -class SQLiteEngine(Engine): - """ - Any connection kwargs are passed into the database adapter. - - See the `SQLite docs `_ - for more info. - - :param log_queries: - If ``True``, all SQL and DDL statements are printed out before being - run. Useful for debugging. +class SQLiteEngine(Engine[t.Optional[SQLiteTransaction]]): - """ # noqa: E501 - - __slots__ = ("connection_kwargs", "transaction_connection", "log_queries") + __slots__ = ("connection_kwargs", "current_transaction", "log_queries") engine_type = "sqlite" min_version_number = 3.25 @@ -362,22 +373,37 @@ def __init__( self, path: str = "piccolo.sqlite", log_queries: bool = False, - detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES, - isolation_level=None, **connection_kwargs, ) -> None: - connection_kwargs.update( - { - "database": path, - "detect_types": detect_types, - "isolation_level": isolation_level, - } - ) + """ + :param path: + A relative or absolute path to the the SQLite database file (it + will be created if it doesn't already exist). + :param log_queries: + If ``True``, all SQL and DDL statements are printed out before + being run. Useful for debugging. + :param connection_kwargs: + These are passed directly to the database adapter. We recommend + setting ``timeout`` if you expect your application to process a + large number of concurrent writes, to prevent queries timing out. + See Python's `sqlite3 docs `_ + for more info. + + """ # noqa: E501 + default_connection_kwargs = { + "database": path, + "detect_types": sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES, + "isolation_level": None, + } + self.log_queries = log_queries - self.connection_kwargs = connection_kwargs + self.connection_kwargs = { + **default_connection_kwargs, + **connection_kwargs, + } - self.transaction_connection = contextvars.ContextVar( - f"sqlite_transaction_connection_{path}", default=None + self.current_transaction = contextvars.ContextVar( + f"sqlite_current_transaction_{path}", default=None ) super().__init__() @@ -404,26 +430,20 @@ async def prep_database(self): def remove_db_file(self): """ - Use with caution - removes the sqlite file. Useful for testing + Use with caution - removes the SQLite file. Useful for testing purposes. """ if os.path.exists(self.path): os.unlink(self.path) - def create_db(self, migrate=False): + def create_db_file(self): """ - Create the database file, with the option to run migrations. Useful - for testing purposes. + Create the database file. Useful for testing purposes. """ if os.path.exists(self.path): raise Exception(f"Database at {self.path} already exists") with open(self.path, "w"): pass - # Commented out for now, as migrations for SQLite aren't as - # well supported as Postgres. - # from piccolo.commands.migration.forwards import ( - # ForwardsMigrationManager, - # ) ########################################################################### @@ -535,10 +555,10 @@ async def run_querystring( ) # If running inside a transaction: - connection = self.transaction_connection.get() - if connection: + current_transaction = self.current_transaction.get() + if current_transaction: return await self._run_in_existing_connection( - connection=connection, + connection=current_transaction.connection, query=query, args=query_args, query_type=querystring.query_type, @@ -561,10 +581,10 @@ async def run_ddl(self, ddl: str, in_pool: bool = False): print(ddl) # If running inside a transaction: - connection = self.transaction_connection.get() - if connection: + current_transaction = self.current_transaction.get() + if current_transaction: return await self._run_in_existing_connection( - connection=connection, + connection=current_transaction.connection, query=ddl, ) @@ -572,8 +592,17 @@ async def run_ddl(self, ddl: str, in_pool: bool = False): query=ddl, ) - def atomic(self) -> Atomic: - return Atomic(engine=self) + def atomic( + self, transaction_type: TransactionType = TransactionType.deferred + ) -> Atomic: + return Atomic(engine=self, transaction_type=transaction_type) - def transaction(self) -> Transaction: - return Transaction(engine=self) + def transaction( + self, transaction_type: TransactionType = TransactionType.deferred + ) -> SQLiteTransaction: + """ + Create a new database transaction. See :class:`Transaction`. + """ + return SQLiteTransaction( + engine=self, transaction_type=transaction_type + ) diff --git a/tests/columns/test_m2m.py b/tests/columns/test_m2m.py index e682ba10c..4dff73430 100644 --- a/tests/columns/test_m2m.py +++ b/tests/columns/test_m2m.py @@ -1,3 +1,4 @@ +import asyncio import datetime import decimal import uuid @@ -542,6 +543,41 @@ def test_add_m2m(self): 1, ) + def test_add_m2m_within_transaction(self): + """ + Make sure we can add items to the joining table, when within an + existing transaction. + + https://github.com/piccolo-orm/piccolo/issues/674 + + """ + engine = Customer._meta.db + + async def add_m2m_in_transaction(): + async with engine.transaction(): + customer: Customer = await Customer.objects().get( + Customer.name == "Bob" + ) + await customer.add_m2m( + Concert(name="Jazzfest"), m2m=Customer.concerts + ) + + asyncio.run(add_m2m_in_transaction()) + + self.assertTrue( + Concert.exists().where(Concert.name == "Jazzfest").run_sync() + ) + + self.assertEqual( + CustomerToConcert.count() + .where( + CustomerToConcert.customer.name == "Bob", + CustomerToConcert.concert.name == "Jazzfest", + ) + .run_sync(), + 1, + ) + def test_get_m2m(self): """ Make sure we can get related items via the joining table. diff --git a/tests/engine/test_transaction.py b/tests/engine/test_transaction.py index 5efe79cb7..c56e802be 100644 --- a/tests/engine/test_transaction.py +++ b/tests/engine/test_transaction.py @@ -1,7 +1,9 @@ import asyncio +import typing as t from unittest import TestCase from piccolo.engine.postgres import Atomic +from piccolo.engine.sqlite import SQLiteEngine, TransactionType from piccolo.table import drop_db_tables_sync from piccolo.utils.sync import run_sync from tests.base import engines_only @@ -126,8 +128,107 @@ async def run_transaction(): return [i[0]["txid_current"] for i in responses] txids = asyncio.run(run_transaction()) - assert len(set(txids)) == 1 + self.assertEqual(len(set(txids)), 1) # Now run it again and make sure the transaction ids differ. next_txids = asyncio.run(run_transaction()) - assert txids != next_txids + self.assertNotEqual(txids, next_txids) + + +class TestTransactionExists(TestCase): + def test_exists(self): + """ + Make sure we can detect when code is within a transaction. + """ + engine = t.cast(SQLiteEngine, Manager._meta.db) + + async def run_inside_transaction(): + async with engine.transaction(): + return engine.transaction_exists() + + self.assertTrue(asyncio.run(run_inside_transaction())) + + async def run_outside_transaction(): + return engine.transaction_exists() + + self.assertFalse(asyncio.run(run_outside_transaction())) + + +@engines_only("sqlite") +class TestTransactionType(TestCase): + def setUp(self): + Manager.create_table().run_sync() + + def tearDown(self): + Manager.alter().drop_table().run_sync() + + def test_transaction(self): + """ + With SQLite, we can specify the transaction type. This helps when + we want to do concurrent writes, to avoid locking the database. + + https://github.com/piccolo-orm/piccolo/issues/687 + """ + engine = t.cast(SQLiteEngine, Manager._meta.db) + + async def run_transaction(name: str): + async with engine.transaction( + transaction_type=TransactionType.immediate + ): + # This does a SELECT followed by an INSERT, so is a good test. + # If using TransactionType.deferred it would fail because + # the database will become locked. + await Manager.objects().get_or_create(Manager.name == name) + + manager_names = [f"Manager_{i}" for i in range(1, 10)] + + async def run_all(): + """ + Run all of the transactions concurrently. + """ + await asyncio.gather( + *[run_transaction(name=name) for name in manager_names] + ) + + asyncio.run(run_all()) + + # Make sure it all ran effectively. + self.assertListEqual( + Manager.select(Manager.name) + .order_by(Manager.name) + .output(as_list=True) + .run_sync(), + manager_names, + ) + + def test_atomic(self): + """ + Similar to above, but with ``Atomic``. + """ + engine = t.cast(SQLiteEngine, Manager._meta.db) + + async def run_atomic(name: str): + atomic = engine.atomic(transaction_type=TransactionType.immediate) + atomic.add(Manager.objects().get_or_create(Manager.name == name)) + await atomic.run() + + manager_names = [f"Manager_{i}" for i in range(1, 10)] + + async def run_all(): + """ + Run all of the transactions concurrently. + """ + await asyncio.gather( + *[run_atomic(name=name) for name in manager_names] + ) + + asyncio.run(run_all()) + + # Make sure it all ran effectively. + self.assertListEqual( + Manager.select(Manager.name) + .order_by(Manager.name) + .output(as_list=True) + .run_sync(), + manager_names, + ) From 4fd91abda934fa9c4cd921e52e2ffb623a4b2a93 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 2 Dec 2022 11:58:27 +0000 Subject: [PATCH 406/727] bumped version --- CHANGES.rst | 27 +++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index da9cd73fa..d8cc9b762 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,33 @@ Changes ======= +0.98.0 +------ + +You can now specify the transaction type for SQLite. + +This is useful when using SQLite in production, as it's possible to get +'database locked' errors if you're running lots of transactions concurrently, +and don't use the correct transaction type. + +In this example we use an ``IMMEDIATE`` transaction: + +.. code-block:: python + + from piccolo.engine.sqlite import TransactionType + + async with Band._meta.db.transaction( + transaction_type=TransactionType.immediate + ): + band = await Band.objects().get_or_create(Band.name == 'Pythonistas') + ... + +We've added a `new tutorial `_ +which explains this in more detail, as well as other tips for using asyncio and +SQLite together effectively. + +------------------------------------------------------------------------------- + 0.97.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 741bf5914..c346757ac 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.97.0" +__VERSION__ = "0.98.0" From e01ba03ec0e252ab83b3d1e070b7d880ee9bd2d5 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 2 Dec 2022 12:10:51 +0000 Subject: [PATCH 407/727] added more info to CHANGES --- CHANGES.rst | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index d8cc9b762..76db6d0e5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,10 +4,13 @@ Changes 0.98.0 ------ +SQLite ``TransactionType`` +~~~~~~~~~~~~~~~~~~~~~~~~~~ + You can now specify the transaction type for SQLite. This is useful when using SQLite in production, as it's possible to get -'database locked' errors if you're running lots of transactions concurrently, +``database locked`` errors if you're running lots of transactions concurrently, and don't use the correct transaction type. In this example we use an ``IMMEDIATE`` transaction: @@ -26,6 +29,17 @@ We've added a `new tutorial Date: Fri, 2 Dec 2022 13:53:23 +0100 Subject: [PATCH 408/727] Add returning clause for delete operation (#701) * returning clause for delete * skip test for old SQLite versions The returning clause is only available in recent SQLite versions. * add link to `returning` clause from `delete` page Co-authored-by: Daniel Townsend --- docs/src/piccolo/query_clauses/returning.rst | 14 ++++++++ docs/src/piccolo/query_types/delete.rst | 5 +++ piccolo/query/methods/delete.py | 35 ++++++++++++++++---- tests/table/test_delete.py | 25 +++++++++++++- 4 files changed, 72 insertions(+), 7 deletions(-) diff --git a/docs/src/piccolo/query_clauses/returning.rst b/docs/src/piccolo/query_clauses/returning.rst index 113554090..6f613e049 100644 --- a/docs/src/piccolo/query_clauses/returning.rst +++ b/docs/src/piccolo/query_clauses/returning.rst @@ -7,6 +7,7 @@ You can use the ``returning`` clause with the following queries: * :ref:`Insert` * :ref:`Update` +* :ref:`Delete` By default, an update query returns an empty list, but using the ``returning`` clause you can retrieve values from the updated rows. @@ -32,6 +33,19 @@ inserted rows: [{'id': 1, 'name': 'Maz'}, {'id': 1, 'name': 'Graydon'}] +As another example, let's use delete and return the full row(s): + +.. code-block:: python + + >>> await Band.delete().where( + ... Band.name == "Pythonistas" + ... ).returning(*Band.all_columns()) + + [{'id': 1, 'name': 'Pythonistas', 'manager': 1, 'popularity': 1000}] + +By counting the number of elements of the returned list, you can find out +how many rows were affected or processed by the operation. + .. warning:: This works for all versions of Postgres, but only `SQLite 3.35.0 `_ and above support the returning clause. See the :ref:`docs ` on diff --git a/docs/src/piccolo/query_types/delete.rst b/docs/src/piccolo/query_types/delete.rst index 90cbff8de..54401f9f3 100644 --- a/docs/src/piccolo/query_types/delete.rst +++ b/docs/src/piccolo/query_types/delete.rst @@ -33,6 +33,11 @@ the data from a table. Query clauses ------------- +returning +~~~~~~~~~ + +See :ref:`Returning`. + where ~~~~~ diff --git a/piccolo/query/methods/delete.py b/piccolo/query/methods/delete.py index 017bd17d2..d5fa07838 100644 --- a/piccolo/query/methods/delete.py +++ b/piccolo/query/methods/delete.py @@ -4,10 +4,11 @@ from piccolo.custom_types import Combinable from piccolo.query.base import Query -from piccolo.query.mixins import WhereDelegate +from piccolo.query.mixins import ReturningDelegate, WhereDelegate from piccolo.querystring import QueryString if t.TYPE_CHECKING: # pragma: no cover + from piccolo.columns import Column from piccolo.table import Table @@ -17,17 +18,26 @@ class DeletionError(Exception): class Delete(Query): - __slots__ = ("force", "where_delegate") + __slots__ = ( + "force", + "returning_delegate", + "where_delegate", + ) def __init__(self, table: t.Type[Table], force: bool = False, **kwargs): super().__init__(table, **kwargs) self.force = force + self.returning_delegate = ReturningDelegate() self.where_delegate = WhereDelegate() def where(self, *where: Combinable) -> Delete: self.where_delegate.where(*where) return self + def returning(self, *columns: Column) -> Delete: + self.returning_delegate.returning(columns) + return self + def _validate(self): """ Don't let a deletion happen unless it has a where clause, or is @@ -44,8 +54,21 @@ def _validate(self): @property def default_querystrings(self) -> t.Sequence[QueryString]: query = f"DELETE FROM {self.table._meta.tablename}" + + querystring = QueryString(query) + if self.where_delegate._where: - query += " WHERE {}" - return [QueryString(query, self.where_delegate._where.querystring)] - else: - return [QueryString(query)] + querystring = QueryString( + "{} WHERE {}", + querystring, + self.where_delegate._where.querystring, + ) + + if self.returning_delegate._returning: + querystring = QueryString( + "{}{}", + querystring, + self.returning_delegate._returning.querystring, + ) + + return [querystring] diff --git a/tests/table/test_delete.py b/tests/table/test_delete.py index 58838bd94..0fb3df112 100644 --- a/tests/table/test_delete.py +++ b/tests/table/test_delete.py @@ -1,5 +1,7 @@ +import pytest + from piccolo.query.methods.delete import DeletionError -from tests.base import DBTestCase +from tests.base import DBTestCase, engine_version_lt, is_running_sqlite from tests.example_apps.music.tables import Band @@ -13,6 +15,27 @@ def test_delete(self): self.assertEqual(response, 0) + @pytest.mark.skipif( + is_running_sqlite() and engine_version_lt(3.35), + reason="SQLite version not supported", + ) + def test_delete_returning(self): + """ + Make sure delete works with the `returning` clause. + """ + + self.insert_rows() + + response = ( + Band.delete() + .where(Band.name == "CSharps") + .returning(Band.name) + .run_sync() + ) + + self.assertEqual(len(response), 1) + self.assertEqual(response, [{"name": "CSharps"}]) + def test_validation(self): """ Make sure you can't delete all the data without forcing it. From eb6ba32e85c2357969e0131a2991b7e50a1dcfb1 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 2 Dec 2022 13:00:02 +0000 Subject: [PATCH 409/727] bumped version --- CHANGES.rst | 23 +++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 76db6d0e5..855260772 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,29 @@ Changes ======= +0.99.0 +------ + +You can now use the ``returning`` clause with ``delete`` queries. + +For example: + +.. code-block:: python + + >>> await Band.delete().where(Band.popularity < 100).returning(Band.name) + [{'name': 'Terrible Band'}, {'name': 'Awful Band'}] + +This also means you can count the number of deleted rows: + +.. code-block:: python + + >>> len(await Band.delete().where(Band.popularity < 100).returning(Band.id)) + 2 + +Thanks to @waldner for adding this feature. + +------------------------------------------------------------------------------- + 0.98.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index c346757ac..1513745e0 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.98.0" +__VERSION__ = "0.99.0" From 112b6edb5819361341dcc5ece17620e33e3cf56a Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 7 Dec 2022 20:01:11 +0000 Subject: [PATCH 410/727] add support for choices with `Array` columns (#704) * add support for choices with `Array` columns * exclude cockroach from the array choice tests * try to optimise `convert_to_sql_value` * add warning about large lists --- docs/src/piccolo/schema/advanced.rst | 36 ++++++++++++ piccolo/columns/base.py | 7 +++ piccolo/columns/column_types.py | 9 ++- piccolo/utils/sql_values.py | 11 ++++ tests/columns/test_choices.py | 87 +++++++++++++++++++++++++++- tests/utils/test_sql_values.py | 43 +++++++++++++- 6 files changed, 187 insertions(+), 6 deletions(-) diff --git a/docs/src/piccolo/schema/advanced.rst b/docs/src/piccolo/schema/advanced.rst index 3ca7c5d1f..0a7147c4d 100644 --- a/docs/src/piccolo/schema/advanced.rst +++ b/docs/src/piccolo/schema/advanced.rst @@ -140,6 +140,42 @@ By using choices, you get the following benefits: * Improved storage efficiency (we can store ``'l'`` instead of ``'large'``). * Piccolo Admin support +``Array`` columns +~~~~~~~~~~~~~~~~~ + +You can also use choices with :class:`Array ` +columns. + +.. code-block:: python + + class Ticket(Table): + class Extras(str, enum.Enum): + drink = "drink" + snack = "snack" + program = "program" + + extras = Array(Varchar(), choices=Extras) + +Note how you pass ``choices`` to ``Array``, and not the ``base_column``: + +.. code-block:: python + + # CORRECT: + Array(Varchar(), choices=Extras) + + # INCORRECT: + Array(Varchar(choices=Extras)) + +We can then use the ``Enum`` in our queries: + +.. code-block:: python + + >>> await Ticket.insert( + ... Ticket(extras=[Extras.drink, Extras.snack]), + ... Ticket(extras=[Extras.program]), + ... ) + + ------------------------------------------------------------------------------- Reflection diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index defaecf86..eae8f4ba3 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -570,6 +570,11 @@ def _validate_choices( """ Make sure the choices value has values of the allowed_type. """ + if getattr(self, "_validated_choices", None): + # If it has previously been validated by a subclass, don't + # validate again. + return True + for element in choices: if isinstance(element.value, allowed_type): continue @@ -582,6 +587,8 @@ def _validate_choices( f"{element.name} doesn't have the correct type" ) + self._validated_choices = True + return True def is_in(self, values: t.List[t.Any]) -> Where: diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 49e86a92a..a1e2693e7 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -2472,8 +2472,15 @@ def __init__( self._validate_default(default, (list, None)) + choices = kwargs.get("choices") + if choices is not None: + self._validate_choices( + choices, allowed_type=base_column.value_type + ) + self._validated_choices = True + # Usually columns are given a name by the Table metaclass, but in this - # case we have to assign one manually. + # case we have to assign one manually to the base column. base_column._meta._name = base_column.__class__.__name__ self.base_column = base_column diff --git a/piccolo/utils/sql_values.py b/piccolo/utils/sql_values.py index 9dde4a5a0..cc372e9e3 100644 --- a/piccolo/utils/sql_values.py +++ b/piccolo/utils/sql_values.py @@ -1,9 +1,11 @@ from __future__ import annotations +import functools import typing as t from enum import Enum from piccolo.utils.encoding import dump_json +from piccolo.utils.warnings import colored_warning if t.TYPE_CHECKING: # pragma: no cover from piccolo.columns import Column @@ -35,5 +37,14 @@ def convert_to_sql_value(value: t.Any, column: Column) -> t.Any: return value.value elif isinstance(column, (JSON, JSONB)) and not isinstance(value, str): return None if value is None else dump_json(value) + elif isinstance(value, list): + if len(value) > 100: + colored_warning( + "When using large lists, consider bypassing the ORM and " + "using SQL directly for improved performance." + ) + # Attempt to do this as performantly as possible. + func = functools.partial(convert_to_sql_value, column=column) + return list(map(func, value)) else: return value diff --git a/tests/columns/test_choices.py b/tests/columns/test_choices.py index 5940ce65a..ccea6b97f 100644 --- a/tests/columns/test_choices.py +++ b/tests/columns/test_choices.py @@ -1,8 +1,19 @@ -from tests.base import DBTestCase +import enum +from unittest import TestCase + +from piccolo.columns.column_types import Array, Varchar +from piccolo.table import Table +from tests.base import engines_only from tests.example_apps.music.tables import Shirt -class TestChoices(DBTestCase): +class TestChoices(TestCase): + def setUp(self): + Shirt.create_table().run_sync() + + def tearDown(self): + Shirt.alter().drop_table().run_sync() + def _insert_shirts(self): Shirt.insert( Shirt(size=Shirt.Size.small), @@ -63,3 +74,75 @@ def test_objects_where(self): ) self.assertEqual(len(shirts), 1) self.assertEqual(shirts[0].size, "s") + + +class Ticket(Table): + class Extras(str, enum.Enum): + drink = "drink" + snack = "snack" + program = "program" + + extras = Array(Varchar(), choices=Extras) + + +@engines_only("postgres", "sqlite") +class TestArrayChoices(TestCase): + """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + """ # noqa: E501 + + def setUp(self): + Ticket.create_table().run_sync() + + def tearDown(self): + Ticket.alter().drop_table().run_sync() + + def test_string(self): + """ + Make sure strings can be passed in as choices. + """ + ticket = Ticket(extras=["drink", "snack", "program"]) + ticket.save().run_sync() + + self.assertListEqual( + Ticket.select(Ticket.extras).run_sync(), + [{"extras": ["drink", "snack", "program"]}], + ) + + def test_enum(self): + """ + Make sure enums can be passed in as choices. + """ + ticket = Ticket( + extras=[ + Ticket.Extras.drink, + Ticket.Extras.snack, + Ticket.Extras.program, + ] + ) + ticket.save().run_sync() + + self.assertListEqual( + Ticket.select(Ticket.extras).run_sync(), + [{"extras": ["drink", "snack", "program"]}], + ) + + def test_invalid_choices(self): + """ + Make sure an invalid choices Enum is rejected. + """ + with self.assertRaises(ValueError) as manager: + + class Ticket(Table): + # This will be rejected, because the values are ints, and the + # Array's base_column is Varchar. + class Extras(int, enum.Enum): + drink = 1 + snack = 2 + program = 3 + + extras = Array(Varchar(), choices=Extras) + + self.assertEqual( + manager.exception.__str__(), "drink doesn't have the correct type" + ) diff --git a/tests/utils/test_sql_values.py b/tests/utils/test_sql_values.py index 299cf96a0..a5bc0416d 100644 --- a/tests/utils/test_sql_values.py +++ b/tests/utils/test_sql_values.py @@ -1,7 +1,10 @@ +import time from enum import Enum from unittest import TestCase -from piccolo.columns.column_types import JSON, JSONB, Integer, Varchar +import pytest + +from piccolo.columns.column_types import JSON, JSONB, Array, Integer, Varchar from piccolo.table import Table from piccolo.utils.sql_values import convert_to_sql_value @@ -48,10 +51,10 @@ def test_convert_enum(self): """ class Colour(Enum): - red = "red" + red = "r" self.assertEqual( - convert_to_sql_value(value=Colour.red, column=Varchar()), "red" + convert_to_sql_value(value=Colour.red, column=Varchar()), "r" ) def test_other(self): @@ -62,3 +65,37 @@ def test_other(self): convert_to_sql_value(value=1, column=Integer()), 1, ) + + def test_convert_enum_list(self): + """ + It's possible to have a list of enums when using ``Array`` columns. + """ + + class Colour(Enum): + red = "r" + green = "g" + blue = "b" + + self.assertEqual( + convert_to_sql_value( + value=[Colour.red, Colour.green, Colour.blue], + column=Array(Varchar()), + ), + ["r", "g", "b"], + ) + + @pytest.mark.speed + def test_convert_large_list(self): + """ + Large lists are problematic. We need to check each value in the list, + but as efficiently as possible. + """ + start = time.time() + + convert_to_sql_value( + value=[i for i in range(1000)], + column=Array(Varchar()), + ) + + duration = time.time() - start + print(duration) From 94c55f803f7757d266b9dd2d2498e44c52467224 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 7 Dec 2022 20:12:54 +0000 Subject: [PATCH 411/727] bumped version --- CHANGES.rst | 28 ++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 855260772..ffaac324f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,34 @@ Changes ======= +0.100.0 +------- + +``Array`` columns now support choices. + +.. code-block:: python + + class Ticket(Table): + class Extras(str, enum.Enum): + drink = "drink" + snack = "snack" + program = "program" + + extras = Array(Varchar(), choices=Extras) + +We can then use the ``Enum`` in our queries: + +.. code-block:: python + + >>> await Ticket.insert( + ... Ticket(extras=[Extras.drink, Extras.snack]), + ... Ticket(extras=[Extras.program]), + ... ) + +This will also be supported in Piccolo Admin in the next release. + +------------------------------------------------------------------------------- + 0.99.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 1513745e0..58eab199b 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.99.0" +__VERSION__ = "0.100.0" From 901b6f30dcf2427d10a054a09206723d4b0c113e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 8 Dec 2022 20:12:12 +0000 Subject: [PATCH 412/727] when loading fixtures, sort data based on foreign key (#710) * when loading fixtures, sort data based on foreign key * make sure subsequent inserts work --- piccolo/apps/fixtures/commands/load.py | 44 +++++++++------- .../apps/fixtures/commands/test_dump_load.py | 50 +++++++++++++------ 2 files changed, 62 insertions(+), 32 deletions(-) diff --git a/piccolo/apps/fixtures/commands/load.py b/piccolo/apps/fixtures/commands/load.py index 8d1d2cc60..12d22f204 100644 --- a/piccolo/apps/fixtures/commands/load.py +++ b/piccolo/apps/fixtures/commands/load.py @@ -1,11 +1,14 @@ from __future__ import annotations +import typing as t + from piccolo.apps.fixtures.commands.shared import ( FixtureConfig, create_pydantic_fixture_model, ) from piccolo.conf.apps import Finder from piccolo.engine import engine_finder +from piccolo.table import Table, sort_table_classes from piccolo.utils.encoding import load_json @@ -40,24 +43,31 @@ async def load_json_string(json_string: str): if not engine: raise Exception("Unable to find the engine.") + # This is what we want to the insert into the database: + data: t.Dict[t.Type[Table], t.List[Table]] = {} + + for app_name in app_names: + app_model = getattr(fixture_pydantic_model, app_name) + + for ( + table_class_name, + model_instance_list, + ) in app_model.__dict__.items(): + table_class = finder.get_table_with_name( + app_name, table_class_name + ) + data[table_class] = [ + table_class.from_dict(row.__dict__) + for row in model_instance_list + ] + + # We have to sort the table classes based on foreign key, so we insert + # the data in the right order. + sorted_table_classes = sort_table_classes(list(data.keys())) + async with engine.transaction(): - for app_name in app_names: - app_model = getattr(fixture_pydantic_model, app_name) - - for ( - table_class_name, - model_instance_list, - ) in app_model.__dict__.items(): - table_class = finder.get_table_with_name( - app_name, table_class_name - ) - - await table_class.insert( - *[ - table_class.from_dict(row.__dict__) - for row in model_instance_list - ] - ).run() + for table_class in sorted_table_classes: + await table_class.insert(*data[table_class]).run() async def load(path: str = "fixture.json"): diff --git a/tests/apps/fixtures/commands/test_dump_load.py b/tests/apps/fixtures/commands/test_dump_load.py index 253958a59..eee930ac1 100644 --- a/tests/apps/fixtures/commands/test_dump_load.py +++ b/tests/apps/fixtures/commands/test_dump_load.py @@ -1,5 +1,6 @@ import datetime import decimal +import typing as t import uuid from unittest import TestCase @@ -29,10 +30,12 @@ def tearDown(self): for table_class in (MegaTable, SmallTable): table_class.alter().drop_table().run_sync() - def insert_row(self): + def insert_rows(self): small_table = SmallTable(varchar_col="Test") small_table.save().run_sync() + SmallTable(varchar_col="Test 2").save().run_sync() + mega_table = MegaTable( bigint_col=1, boolean_col=True, @@ -60,27 +63,22 @@ def insert_row(self): ) mega_table.save().run_sync() - @engines_only("postgres", "sqlite") - def test_dump_load(self): - """ - Make sure we can dump some rows into a JSON fixture, then load them - back into the database. - """ - self.insert_row() + def _run_comparison(self, table_class_names: t.List[str]): + self.insert_rows() json_string = run_sync( dump_to_json_string( fixture_configs=[ FixtureConfig( app_name="mega", - table_class_names=["SmallTable", "MegaTable"], + table_class_names=table_class_names, ) ] ) ) # We need to clear the data out now, otherwise when loading the data - # back in, there will be a constraint errors over clashing primary + # back in, there will be constraint errors over clashing primary # keys. SmallTable.delete(force=True).run_sync() MegaTable.delete(force=True).run_sync() @@ -89,7 +87,10 @@ def test_dump_load(self): self.assertEqual( SmallTable.select().run_sync(), - [{"id": 1, "varchar_col": "Test"}], + [ + {"id": 1, "varchar_col": "Test"}, + {"id": 2, "varchar_col": "Test 2"}, + ], ) mega_table_data = MegaTable.select().run_sync() @@ -138,13 +139,32 @@ def test_dump_load(self): }, ) - @engines_only("cockroach") - def test_dump_load_alt(self): + # Make sure subsequent inserts work. + SmallTable().save().run_sync() + + @engines_only("postgres", "sqlite") + def test_dump_load(self): """ Make sure we can dump some rows into a JSON fixture, then load them back into the database. """ - self.insert_row() + self._run_comparison(table_class_names=["SmallTable", "MegaTable"]) + + @engines_only("postgres", "sqlite") + def test_dump_load_ordering(self): + """ + Similar to `test_dump_load` - but we need to make sure it inserts + the data in the correct order, so foreign key constraints don't fail. + """ + self._run_comparison(table_class_names=["MegaTable", "SmallTable"]) + + @engines_only("cockroach") + def test_dump_load_cockroach(self): + """ + Similar to `test_dump_load`, except the schema is slightly different + for CockroachDB. + """ + self.insert_rows() json_string = run_sync( dump_to_json_string( @@ -158,7 +178,7 @@ def test_dump_load_alt(self): ) # We need to clear the data out now, otherwise when loading the data - # back in, there will be a constraint errors over clashing primary + # back in, there will be constraint errors over clashing primary # keys. SmallTable.delete(force=True).run_sync() MegaTable.delete(force=True).run_sync() From 4bcb77043b37254980d2af4de38d74095a97524a Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 8 Dec 2022 20:18:45 +0000 Subject: [PATCH 413/727] bumped version --- CHANGES.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index ffaac324f..4ade4004b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,14 @@ Changes ======= +0.101.0 +------- + +``piccolo fixtures load`` is now more intelligent about how it loads data, to +avoid foreign key constraint errors. + +------------------------------------------------------------------------------- + 0.100.0 ------- From f8a7eee99fda67ed94627ce5e4410406bd1a2070 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 8 Dec 2022 20:27:44 +0000 Subject: [PATCH 414/727] add correct version --- piccolo/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 58eab199b..6e56794bf 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.100.0" +__VERSION__ = "0.101.0" From 828544984bf9c8299e2249a44e46c51f1e6db246 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 9 Dec 2022 20:21:59 +0000 Subject: [PATCH 415/727] Change migration filenames to make them importable (#712) * wip * added a test --- piccolo/apps/migrations/commands/new.py | 12 +++++++-- tests/apps/migrations/commands/test_new.py | 30 ++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/piccolo/apps/migrations/commands/new.py b/piccolo/apps/migrations/commands/new.py index 3cc04fbb0..9081f0e59 100644 --- a/piccolo/apps/migrations/commands/new.py +++ b/piccolo/apps/migrations/commands/new.py @@ -60,6 +60,13 @@ class NewMigrationMeta: migration_path: str +def now(): + """ + In a separate function so it's easier to patch in tests. + """ + return datetime.datetime.now() + + def _generate_migration_meta(app_config: AppConfig) -> NewMigrationMeta: """ Generates the migration ID and filename. @@ -68,13 +75,14 @@ def _generate_migration_meta(app_config: AppConfig) -> NewMigrationMeta: # chance that the IDs would clash if the migrations are generated # programatically in quick succession (e.g. in a unit test), so they had # to be added. The trade off is a longer ID. - _id = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S:%f") + _id = now().strftime("%Y-%m-%dT%H:%M:%S:%f") # Originally we just used the _id as the filename, but colons aren't # supported in Windows, so we need to sanitize it. We don't want to # change the _id format though, as it would break existing migrations. # The filename doesn't have any special significance - only the id matters. - filename = _id.replace(":", "-") + cleaned_id = _id.replace(":", "_").replace("-", "_").lower() + filename = f"{app_config.app_name}_{cleaned_id}" path = os.path.join(app_config.migrations_folder_path, f"{filename}.py") diff --git a/tests/apps/migrations/commands/test_new.py b/tests/apps/migrations/commands/test_new.py index bb802b1df..bf94a3468 100644 --- a/tests/apps/migrations/commands/test_new.py +++ b/tests/apps/migrations/commands/test_new.py @@ -1,3 +1,4 @@ +import datetime import os import shutil import tempfile @@ -7,6 +8,7 @@ from piccolo.apps.migrations.commands.new import ( BaseMigrationManager, _create_new_migration, + _generate_migration_meta, new, ) from piccolo.conf.apps import AppConfig @@ -56,3 +58,31 @@ def test_new_command(self, print_: MagicMock): self.assertTrue( print_.mock_calls[-1] == call("No changes detected - exiting.") ) + + +class TestGenerateMigrationMeta(TestCase): + @patch("piccolo.apps.migrations.commands.new.now") + def test_filename(self, now: MagicMock): + now.return_value = datetime.datetime( + year=2022, + month=1, + day=10, + hour=7, + minute=15, + second=20, + microsecond=3000, + ) + migration_meta = _generate_migration_meta( + app_config=AppConfig( + app_name="app_name", + migrations_folder_path="/tmp/", + ) + ) + self.assertEqual( + migration_meta.migration_filename, + "app_name_2022_01_10t07_15_20_003000", + ) + self.assertEqual( + migration_meta.migration_path, + "/tmp/app_name_2022_01_10t07_15_20_003000.py", + ) From f4e5622600d6938e3f65be7dd132f124a1520dc1 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Mon, 12 Dec 2022 12:13:48 +0100 Subject: [PATCH 416/727] update Starlite asgi template (#714) --- piccolo/apps/asgi/commands/new.py | 2 +- .../templates/app/_starlite_app.py.jinja | 17 ++++------------- .../app/home/templates/home.html.jinja_raw | 2 +- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index a5f25691a..499c22d31 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -12,7 +12,7 @@ SERVERS = ["uvicorn", "Hypercorn"] ROUTERS = ["starlette", "fastapi", "blacksheep", "xpresso", "starlite"] ROUTER_DEPENDENCIES = { - "starlite": ["starlite==1.23.0"], + "starlite": ["starlite>=1.45.0"], "xpresso": ["xpresso==0.43.0", "di==0.72.1"], } diff --git a/piccolo/apps/asgi/commands/templates/app/_starlite_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_starlite_app.py.jinja index f73b53d24..97dbd8623 100644 --- a/piccolo/apps/asgi/commands/templates/app/_starlite_app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/_starlite_app.py.jinja @@ -2,7 +2,6 @@ import typing as t from piccolo.engine import engine_finder from piccolo_admin.endpoints import create_admin -from starlette.routing import Mount, Router from starlite import ( MediaType, Response, @@ -23,18 +22,10 @@ from home.endpoints import home from home.piccolo_app import APP_CONFIG from home.tables import Task - -@asgi(path="/{admin:path}") -async def admin(scope: Scope, receive: Receive, send: Send) -> None: - router = Router( - [ - Mount( - path="/admin", - app=create_admin(tables=APP_CONFIG.table_classes), - ), - ] - ) - await router(scope=scope, receive=receive, send=send) +# mounting Piccolo Admin +@asgi("/admin/", is_mount=True) +async def admin(scope: "Scope", receive: "Receive", send: "Send") -> None: + await create_admin(tables=APP_CONFIG.table_classes)(scope, receive, send) @get("/tasks", tags=["Task"]) diff --git a/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw b/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw index ed5934f9f..cbc65de87 100644 --- a/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw +++ b/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw @@ -58,7 +58,7 @@

    Starlite

    From 60251e4c08d684138f17272273e5d3fd97ad6c59 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 13 Dec 2022 09:55:16 +0000 Subject: [PATCH 417/727] 715-get-or-create-with-null make sure that `get_or_create` works correctly with null values --- piccolo/columns/combination.py | 24 +++++++++++++++++------- tests/columns/test_combination.py | 18 ++++++++++++++++++ 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/piccolo/columns/combination.py b/piccolo/columns/combination.py index a753957d3..fec8c3767 100644 --- a/piccolo/columns/combination.py +++ b/piccolo/columns/combination.py @@ -2,7 +2,11 @@ import typing as t -from piccolo.columns.operators.comparison import ComparisonOperator, Equal +from piccolo.columns.operators.comparison import ( + ComparisonOperator, + Equal, + IsNull, +) from piccolo.custom_types import Combinable, Iterable from piccolo.querystring import QueryString from piccolo.utils.sql_values import convert_to_sql_value @@ -52,18 +56,22 @@ def get_column_values(self) -> t.Dict[Column, t.Any]: This is used by `get_or_create` to know which values to assign if the row doesn't exist in the database. - For example, if we have: + For example, if we have:: - (Band.name == 'Pythonistas') & (Band.popularity == 1000) + (Band.name == 'Pythonistas') & (Band.popularity == 1000) - We will return {Band.name: 'Pythonistas', Band.popularity: 1000}. + We will return:: + + {Band.name: 'Pythonistas', Band.popularity: 1000}. If the operator is anything besides equals, we don't return it, for - example: + example:: + + (Band.name == 'Pythonistas') & (Band.popularity > 1000) - (Band.name == 'Pythonistas') & (Band.popularity > 1000) + Returns:: - Returns {Band.name: 'Pythonistas'} + {Band.name: 'Pythonistas'} """ output = {} @@ -71,6 +79,8 @@ def get_column_values(self) -> t.Dict[Column, t.Any]: if isinstance(combinable, Where): if combinable.operator == Equal: output[combinable.column] = combinable.value + elif combinable.operator == IsNull: + output[combinable.column] = None elif isinstance(combinable, And): output.update(combinable.get_column_values()) diff --git a/tests/columns/test_combination.py b/tests/columns/test_combination.py index 032830d6e..c17c55d99 100644 --- a/tests/columns/test_combination.py +++ b/tests/columns/test_combination.py @@ -36,3 +36,21 @@ def test_not_in(self): with self.assertRaises(ValueError): Band.name.not_in([]) + + +class TestAnd(unittest.TestCase): + def test_get_column_values(self): + """ + Make sure that we can extract the column values from an ``And``. + + There was a bug with ``None`` values not working: + + https://github.com/piccolo-orm/piccolo/issues/715 + + """ + And_ = (Band.manager.is_null()) & (Band.name == "Pythonistas") + column_values = And_.get_column_values() + + self.assertDictEqual( + column_values, {Band.name: "Pythonistas", Band.manager: None} + ) From 7911252e916579e9acb99cadcf9bad50b0825cf2 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 13 Dec 2022 10:01:50 +0000 Subject: [PATCH 418/727] Revert "715-get-or-create-with-null" This reverts commit 60251e4c08d684138f17272273e5d3fd97ad6c59. --- piccolo/columns/combination.py | 24 +++++++----------------- tests/columns/test_combination.py | 18 ------------------ 2 files changed, 7 insertions(+), 35 deletions(-) diff --git a/piccolo/columns/combination.py b/piccolo/columns/combination.py index fec8c3767..a753957d3 100644 --- a/piccolo/columns/combination.py +++ b/piccolo/columns/combination.py @@ -2,11 +2,7 @@ import typing as t -from piccolo.columns.operators.comparison import ( - ComparisonOperator, - Equal, - IsNull, -) +from piccolo.columns.operators.comparison import ComparisonOperator, Equal from piccolo.custom_types import Combinable, Iterable from piccolo.querystring import QueryString from piccolo.utils.sql_values import convert_to_sql_value @@ -56,22 +52,18 @@ def get_column_values(self) -> t.Dict[Column, t.Any]: This is used by `get_or_create` to know which values to assign if the row doesn't exist in the database. - For example, if we have:: + For example, if we have: - (Band.name == 'Pythonistas') & (Band.popularity == 1000) + (Band.name == 'Pythonistas') & (Band.popularity == 1000) - We will return:: - - {Band.name: 'Pythonistas', Band.popularity: 1000}. + We will return {Band.name: 'Pythonistas', Band.popularity: 1000}. If the operator is anything besides equals, we don't return it, for - example:: - - (Band.name == 'Pythonistas') & (Band.popularity > 1000) + example: - Returns:: + (Band.name == 'Pythonistas') & (Band.popularity > 1000) - {Band.name: 'Pythonistas'} + Returns {Band.name: 'Pythonistas'} """ output = {} @@ -79,8 +71,6 @@ def get_column_values(self) -> t.Dict[Column, t.Any]: if isinstance(combinable, Where): if combinable.operator == Equal: output[combinable.column] = combinable.value - elif combinable.operator == IsNull: - output[combinable.column] = None elif isinstance(combinable, And): output.update(combinable.get_column_values()) diff --git a/tests/columns/test_combination.py b/tests/columns/test_combination.py index c17c55d99..032830d6e 100644 --- a/tests/columns/test_combination.py +++ b/tests/columns/test_combination.py @@ -36,21 +36,3 @@ def test_not_in(self): with self.assertRaises(ValueError): Band.name.not_in([]) - - -class TestAnd(unittest.TestCase): - def test_get_column_values(self): - """ - Make sure that we can extract the column values from an ``And``. - - There was a bug with ``None`` values not working: - - https://github.com/piccolo-orm/piccolo/issues/715 - - """ - And_ = (Band.manager.is_null()) & (Band.name == "Pythonistas") - column_values = And_.get_column_values() - - self.assertDictEqual( - column_values, {Band.name: "Pythonistas", Band.manager: None} - ) From c1a95aeeef442f181e5bd36488d257c29dfd4006 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 13 Dec 2022 10:30:27 +0000 Subject: [PATCH 419/727] 715-get-or-create-with-null (#716) make sure that `get_or_create` works correctly with null values --- piccolo/columns/combination.py | 24 +++++++++++++++++------- tests/columns/test_combination.py | 18 ++++++++++++++++++ 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/piccolo/columns/combination.py b/piccolo/columns/combination.py index a753957d3..fec8c3767 100644 --- a/piccolo/columns/combination.py +++ b/piccolo/columns/combination.py @@ -2,7 +2,11 @@ import typing as t -from piccolo.columns.operators.comparison import ComparisonOperator, Equal +from piccolo.columns.operators.comparison import ( + ComparisonOperator, + Equal, + IsNull, +) from piccolo.custom_types import Combinable, Iterable from piccolo.querystring import QueryString from piccolo.utils.sql_values import convert_to_sql_value @@ -52,18 +56,22 @@ def get_column_values(self) -> t.Dict[Column, t.Any]: This is used by `get_or_create` to know which values to assign if the row doesn't exist in the database. - For example, if we have: + For example, if we have:: - (Band.name == 'Pythonistas') & (Band.popularity == 1000) + (Band.name == 'Pythonistas') & (Band.popularity == 1000) - We will return {Band.name: 'Pythonistas', Band.popularity: 1000}. + We will return:: + + {Band.name: 'Pythonistas', Band.popularity: 1000}. If the operator is anything besides equals, we don't return it, for - example: + example:: + + (Band.name == 'Pythonistas') & (Band.popularity > 1000) - (Band.name == 'Pythonistas') & (Band.popularity > 1000) + Returns:: - Returns {Band.name: 'Pythonistas'} + {Band.name: 'Pythonistas'} """ output = {} @@ -71,6 +79,8 @@ def get_column_values(self) -> t.Dict[Column, t.Any]: if isinstance(combinable, Where): if combinable.operator == Equal: output[combinable.column] = combinable.value + elif combinable.operator == IsNull: + output[combinable.column] = None elif isinstance(combinable, And): output.update(combinable.get_column_values()) diff --git a/tests/columns/test_combination.py b/tests/columns/test_combination.py index 032830d6e..c17c55d99 100644 --- a/tests/columns/test_combination.py +++ b/tests/columns/test_combination.py @@ -36,3 +36,21 @@ def test_not_in(self): with self.assertRaises(ValueError): Band.name.not_in([]) + + +class TestAnd(unittest.TestCase): + def test_get_column_values(self): + """ + Make sure that we can extract the column values from an ``And``. + + There was a bug with ``None`` values not working: + + https://github.com/piccolo-orm/piccolo/issues/715 + + """ + And_ = (Band.manager.is_null()) & (Band.name == "Pythonistas") + column_values = And_.get_column_values() + + self.assertDictEqual( + column_values, {Band.name: "Pythonistas", Band.manager: None} + ) From f041ec91ece88f7aae12dc5c1d2894a64861e04d Mon Sep 17 00:00:00 2001 From: waldner Date: Tue, 13 Dec 2022 21:46:43 +0100 Subject: [PATCH 420/727] Expose pydantic config class to user (#705) `create_pydantic_model` accepts a new argument specifying the base class to use for the generated model config. --- docs/src/piccolo/serialization/index.rst | 23 ++++++++++++++++ piccolo/utils/pydantic.py | 7 ++++- tests/utils/test_pydantic.py | 35 ++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/docs/src/piccolo/serialization/index.rst b/docs/src/piccolo/serialization/index.rst index ceb2761f8..da240fad8 100644 --- a/docs/src/piccolo/serialization/index.rst +++ b/docs/src/piccolo/serialization/index.rst @@ -154,6 +154,29 @@ By default the primary key column isn't included - you can add it using: BandModel = create_pydantic_model(Band, include_default_columns=True) +``pydantic_config_class`` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can specify a custom class to use as base for the pydantic model's config. +This class should be a subclass of ``pydantic.BaseConfig``. +For example, let's set the ``extra`` parameter to tell pydantic how to treat +extra fields (that is, fields that would not otherwise be in the generated model). +The allowed values are:: + +* ``'ignore'`` (default): silently ignore extra fields +* ``'allow'``: accept the extra fields and assigns them to the model +* ``'forbid'``: fail validation if extra fields are present + +So if we want to disallow extra fields, we can do: + +.. code-block:: python + + class MyPydanticConfig(pydantic.BaseConfig): + extra = 'forbid' + + model = create_pydantic_model(table=MyTable, pydantic_config_class=MyPydanticConfig, ...) + + Required fields ~~~~~~~~~~~~~~~ diff --git a/piccolo/utils/pydantic.py b/piccolo/utils/pydantic.py index fbe63fe4d..13e7e82e2 100644 --- a/piccolo/utils/pydantic.py +++ b/piccolo/utils/pydantic.py @@ -90,6 +90,7 @@ def create_pydantic_model( deserialize_json: bool = False, recursion_depth: int = 0, max_recursion_depth: int = 5, + pydantic_config_class: t.Type[pydantic.BaseConfig] = pydantic.BaseConfig, **schema_extra_kwargs, ) -> t.Type[pydantic.BaseModel]: """ @@ -132,6 +133,10 @@ def create_pydantic_model( Not to be set by the user - used internally to track recursion. :param max_recursion_depth: If using nested models, this specifies the max amount of recursion. + :param pydantic_config_class: + Config class to use as base for the generated pydantic model + (default: ``pydantic.BaseConfig``). You can create your own + subclass of ``pydantic.BaseConfig`` and pass it here. :param schema_extra_kwargs: This can be used to add additional fields to the schema. This is very useful when using Pydantic's JSON Schema features. For example: @@ -301,7 +306,7 @@ def create_pydantic_model( columns[column_name] = (_type, field) - class CustomConfig(Config): + class CustomConfig(Config, pydantic_config_class): # type: ignore schema_extra = { "help_text": table._meta.help_text, **schema_extra_kwargs, diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index 394ed4afe..38a5cdfe7 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -2,6 +2,7 @@ from unittest import TestCase import pydantic +import pytest from pydantic import ValidationError from piccolo.columns import ( @@ -701,3 +702,37 @@ class Band(Table): model = create_pydantic_model(Band, visible_columns=("name",)) self.assertEqual(model.schema()["visible_columns"], ("name",)) + + +class TestPydanticExtraFields(TestCase): + def test_pydantic_extra_fields(self): + """ + Make sure that the value of ``pydantic_extra_fields`` is correctly + propagated to the generated model. + """ + + class Band(Table): + name = Varchar() + + for v in ["ignore", "allow", "forbid"]: + + class MyConfig(pydantic.BaseConfig): + extra = v + + model = create_pydantic_model(Band, pydantic_config_class=MyConfig) + self.assertEqual(model.Config.extra, v) + + def test_pydantic_invalid_extra_fields(self): + """ + Make sure that invalid values for ``pydantic_extra_fields`` + are rejected. + """ + + class Band(Table): + name = Varchar() + + class MyConfig(pydantic.BaseConfig): + extra = "foobar" + + with pytest.raises(ValueError): + create_pydantic_model(Band, pydantic_config_class=MyConfig) From 52501c1d3ae5451e4af11e09949050c1928c81db Mon Sep 17 00:00:00 2001 From: waldner Date: Tue, 13 Dec 2022 23:21:33 +0100 Subject: [PATCH 421/727] fix test descriptions (#717) --- tests/utils/test_pydantic.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index 38a5cdfe7..7103e3db6 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -707,8 +707,8 @@ class Band(Table): class TestPydanticExtraFields(TestCase): def test_pydantic_extra_fields(self): """ - Make sure that the value of ``pydantic_extra_fields`` is correctly - propagated to the generated model. + Make sure that the value of ``extra`` in the config class + is correctly propagated to the generated model. """ class Band(Table): @@ -724,7 +724,7 @@ class MyConfig(pydantic.BaseConfig): def test_pydantic_invalid_extra_fields(self): """ - Make sure that invalid values for ``pydantic_extra_fields`` + Make sure that invalid values for ``extra`` in the config class are rejected. """ From 14012e59759c849ad732b4c4e4fb40636dc88aa6 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 13 Dec 2022 23:25:39 +0000 Subject: [PATCH 422/727] tweak docs for Pydantic base class (#718) --- docs/src/piccolo/serialization/index.rst | 18 +++++++++++------- piccolo/utils/pydantic.py | 13 ++++++++----- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/docs/src/piccolo/serialization/index.rst b/docs/src/piccolo/serialization/index.rst index da240fad8..64c6aa936 100644 --- a/docs/src/piccolo/serialization/index.rst +++ b/docs/src/piccolo/serialization/index.rst @@ -157,11 +157,12 @@ By default the primary key column isn't included - you can add it using: ``pydantic_config_class`` ~~~~~~~~~~~~~~~~~~~~~~~~~ -You can specify a custom class to use as base for the pydantic model's config. +You can specify a custom class to use as the base for the Pydantic model's config. This class should be a subclass of ``pydantic.BaseConfig``. + For example, let's set the ``extra`` parameter to tell pydantic how to treat extra fields (that is, fields that would not otherwise be in the generated model). -The allowed values are:: +The allowed values are: * ``'ignore'`` (default): silently ignore extra fields * ``'allow'``: accept the extra fields and assigns them to the model @@ -174,7 +175,10 @@ So if we want to disallow extra fields, we can do: class MyPydanticConfig(pydantic.BaseConfig): extra = 'forbid' - model = create_pydantic_model(table=MyTable, pydantic_config_class=MyPydanticConfig, ...) + model = create_pydantic_model( + table=MyTable, + pydantic_config_class=MyPydanticConfig + ) Required fields @@ -187,7 +191,7 @@ argument of :class:`Column `. For example: class Band(Table): name = Varchar(required=True) - + BandModel = create_pydantic_model(Band) # Omitting the field raises an error: @@ -202,7 +206,7 @@ all fields to be optional. class Band(Table): name = Varchar(required=True) - + BandFilterModel = create_pydantic_model( Band, all_optional=True, @@ -222,12 +226,12 @@ add additional fields, and to override existing fields. class Band(Table): name = Varchar(required=True) - + BandModel = create_pydantic_model(Band) class CustomBandModel(BandModel): genre: str - + >>> CustomBandModel(name="Pythonistas", genre="Rock") Source diff --git a/piccolo/utils/pydantic.py b/piccolo/utils/pydantic.py index 13e7e82e2..c637b401f 100644 --- a/piccolo/utils/pydantic.py +++ b/piccolo/utils/pydantic.py @@ -90,7 +90,7 @@ def create_pydantic_model( deserialize_json: bool = False, recursion_depth: int = 0, max_recursion_depth: int = 5, - pydantic_config_class: t.Type[pydantic.BaseConfig] = pydantic.BaseConfig, + pydantic_config_class: t.Optional[t.Type[pydantic.BaseConfig]] = None, **schema_extra_kwargs, ) -> t.Type[pydantic.BaseModel]: """ @@ -134,9 +134,8 @@ def create_pydantic_model( :param max_recursion_depth: If using nested models, this specifies the max amount of recursion. :param pydantic_config_class: - Config class to use as base for the generated pydantic model - (default: ``pydantic.BaseConfig``). You can create your own - subclass of ``pydantic.BaseConfig`` and pass it here. + Config class to use as base for the generated pydantic model. You can + create your own subclass of ``pydantic.BaseConfig`` and pass it here. :param schema_extra_kwargs: This can be used to add additional fields to the schema. This is very useful when using Pydantic's JSON Schema features. For example: @@ -306,7 +305,11 @@ def create_pydantic_model( columns[column_name] = (_type, field) - class CustomConfig(Config, pydantic_config_class): # type: ignore + base_classes: t.List[t.Type[pydantic.BaseConfig]] = [Config] + if pydantic_config_class: + base_classes.append(pydantic_config_class) + + class CustomConfig(*base_classes): # type: ignore schema_extra = { "help_text": table._meta.help_text, **schema_extra_kwargs, From 4fe9b4856abdd0d20e4f4625d029d2c88027c21d Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 16 Dec 2022 23:43:30 +0000 Subject: [PATCH 423/727] Update migration docs and tests (#721) * updated docs for new migration file naming convention * added extra tests for new migration file naming * fix black error --- docs/src/piccolo/migrations/create.rst | 13 +++++++++---- docs/src/piccolo/migrations/running.rst | 9 +++++++-- piccolo/apps/migrations/commands/new.py | 17 +++++++++++++++-- tests/apps/migrations/commands/test_new.py | 19 +++++++++++++++++++ 4 files changed, 50 insertions(+), 8 deletions(-) diff --git a/docs/src/piccolo/migrations/create.rst b/docs/src/piccolo/migrations/create.rst index 404e09648..c769f3c1c 100644 --- a/docs/src/piccolo/migrations/create.rst +++ b/docs/src/piccolo/migrations/create.rst @@ -21,15 +21,20 @@ First, let's create an empty migration: piccolo migrations new my_app -This creates a new migration file in the migrations folder of the app. The -migration filename is a timestamp: +This creates a new migration file in the migrations folder of the app. By +default, the migration filename is the name of the app, followed by a timestamp, +but you can rename it to anything you want: .. code-block:: bash piccolo_migrations/ - 2022-02-26T17-38-44-758593.py + my_app_2022_12_06T13_58_23_024723.py -.. hint:: You can rename this file if you like to make it more memorable. +.. note:: + We changed the naming convention for migration files in version ``0.102.0`` + (previously they were like ``2022-12-06T13-58-23-024723.py``). As mentioned, + the name isn't important - change it to anything you want. The new format + was chosen because a Python file should start with a letter by convention. The contents of an empty migration file looks like this: diff --git a/docs/src/piccolo/migrations/running.rst b/docs/src/piccolo/migrations/running.rst index e7b38289a..278d48ca5 100644 --- a/docs/src/piccolo/migrations/running.rst +++ b/docs/src/piccolo/migrations/running.rst @@ -3,6 +3,7 @@ Running migrations .. hint:: To see all available options for these commands, use the ``--help`` flag, for example ``piccolo migrations forwards --help``. + .. hint:: To see the SQL queries of a migration without actually running them , use the ``--preview`` flag, for example: ``piccolo migrations forwards my_app --preview`` or ``piccolo migrations backwards 2018-09-04T19:44:09 --preview``. @@ -20,11 +21,15 @@ When the migration is run, the forwards function is executed. To do this: Reversing migrations -------------------- -To reverse the migration, run this: +To reverse the migration, run the following command, specifying the ID of a +migration: .. code-block:: bash - piccolo migrations backwards 2018-09-04T19:44:09 + piccolo migrations backwards my_app 2018-09-04T19:44:09 + +Piccolo will then reverse the migrations for the given app, starting with the +most recent migration, up to and including the migration with the specified ID. You can try going forwards and backwards a few times to make sure it works as expected. diff --git a/piccolo/apps/migrations/commands/new.py b/piccolo/apps/migrations/commands/new.py index 9081f0e59..ec3812139 100644 --- a/piccolo/apps/migrations/commands/new.py +++ b/piccolo/apps/migrations/commands/new.py @@ -2,6 +2,7 @@ import datetime import os +import string import sys import typing as t from dataclasses import dataclass @@ -31,9 +32,10 @@ loader=jinja2.FileSystemLoader(searchpath=TEMPLATE_DIRECTORY), ) - MIGRATION_MODULES: t.Dict[str, ModuleType] = {} +VALID_PYTHON_MODULE_CHARACTERS = string.ascii_lowercase + string.digits + "_" + def render_template(**kwargs): template = JINJA_ENV.get_template("migration.py.jinja") @@ -82,7 +84,18 @@ def _generate_migration_meta(app_config: AppConfig) -> NewMigrationMeta: # change the _id format though, as it would break existing migrations. # The filename doesn't have any special significance - only the id matters. cleaned_id = _id.replace(":", "_").replace("-", "_").lower() - filename = f"{app_config.app_name}_{cleaned_id}" + + # Just in case the app name contains characters which aren't valid for + # a Python module. + cleaned_app_name = "".join( + [ + i + for i in app_config.app_name.lower().replace("-", "_") + if i in VALID_PYTHON_MODULE_CHARACTERS + ] + ) + + filename = f"{cleaned_app_name}_{cleaned_id}" path = os.path.join(app_config.migrations_folder_path, f"{filename}.py") diff --git a/tests/apps/migrations/commands/test_new.py b/tests/apps/migrations/commands/test_new.py index bf94a3468..08c549ebd 100644 --- a/tests/apps/migrations/commands/test_new.py +++ b/tests/apps/migrations/commands/test_new.py @@ -72,6 +72,9 @@ def test_filename(self, now: MagicMock): second=20, microsecond=3000, ) + + # Try with an app name which already contains valid characters for a + # Python module. migration_meta = _generate_migration_meta( app_config=AppConfig( app_name="app_name", @@ -86,3 +89,19 @@ def test_filename(self, now: MagicMock): migration_meta.migration_path, "/tmp/app_name_2022_01_10t07_15_20_003000.py", ) + + # Try with an app name with invalid characters for a Python module. + migration_meta = _generate_migration_meta( + app_config=AppConfig( + app_name="App-Name!", + migrations_folder_path="/tmp/", + ) + ) + self.assertEqual( + migration_meta.migration_filename, + "app_name_2022_01_10t07_15_20_003000", + ) + self.assertEqual( + migration_meta.migration_path, + "/tmp/app_name_2022_01_10t07_15_20_003000.py", + ) From 35d33465484dddbbc5a9b59e774bb75db3d18319 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 17 Dec 2022 00:06:47 +0000 Subject: [PATCH 424/727] bumped version (#722) --- CHANGES.rst | 51 +++++++++++++++++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 4ade4004b..5347d9f0c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,57 @@ Changes ======= +0.102.0 +------- + +Migration file names +~~~~~~~~~~~~~~~~~~~~ + +The naming convention for migrations has changed slightly. It used to be just +a timestamp - for example: + +.. code-block:: text + + 2021-09-06T13-58-23-024723.py + +By convention Python files should start with a letter, and only contain +``a-z``, ``0-9`` and ``_``, so the new format is: + +.. code-block:: text + + my_app_2021_09_06T13_58_23_024723.py + +.. note:: You can name a migration file anything you want (it's the ``ID`` + value inside it which is important), so this change doesn't break anything. + +Enhanced Pydantic configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We now expose all of Pydantic's configuration options to +``create_pydantic_model``: + +.. code-block:: python + + class MyPydanticConfig(pydantic.BaseConfig): + extra = 'forbid' + + model = create_pydantic_model( + table=MyTable, + pydantic_config_class=MyPydanticConfig + ) + +Thanks to @waldner for this. + +Other changes +~~~~~~~~~~~~~ + +* Fixed a bug with ``get_or_create`` and null columns (thanks to @powellnorma + for reporting this issue). +* Updated the Starlite ASGI template, so it uses the latest syntax for mounting + Piccolo Admin (thanks to @sinisaos for this, and the Starlite team). + +------------------------------------------------------------------------------- + 0.101.0 ------- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 6e56794bf..cc9a1072b 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.101.0" +__VERSION__ = "0.102.0" From b56192b001950a0121f66f47db0c01957efdbca5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominic=20D=C4=85browski?= <44851017+blazerman53@users.noreply.github.com> Date: Sat, 17 Dec 2022 16:17:05 +0100 Subject: [PATCH 425/727] Removed reference to LGTM (#726) --- piccolo/columns/defaults/interval.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piccolo/columns/defaults/interval.py b/piccolo/columns/defaults/interval.py index 5d12fdda3..f3daba639 100644 --- a/piccolo/columns/defaults/interval.py +++ b/piccolo/columns/defaults/interval.py @@ -7,7 +7,7 @@ from .base import Default -class IntervalCustom(Default): # lgtm [py/missing-equals] +class IntervalCustom(Default): def __init__( self, weeks: int = 0, From 3dfef677dfd409b5067a499901c6f30e3c32a7c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominic=20D=C4=85browski?= <44851017+blazerman53@users.noreply.github.com> Date: Sat, 17 Dec 2022 16:17:32 +0100 Subject: [PATCH 426/727] Removed references to LGTM (#725) --- piccolo/columns/column_types.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index a1e2693e7..854a2af78 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -1217,7 +1217,7 @@ def __set__(self, obj, value: t.Union[time, None]): obj.__dict__[self._meta.name] = value -class Interval(Column): # lgtm [py/missing-equals] +class Interval(Column): """ Used for storing timedeltas. Uses the ``timedelta`` type for values. @@ -1637,7 +1637,7 @@ class ForeignKeySetupResponse: is_lazy: bool -class ForeignKey(Column): # lgtm [py/missing-equals] +class ForeignKey(Column): """ Used to reference another table. Uses the same type as the primary key column on the table it references. @@ -2179,7 +2179,7 @@ def __set__(self, obj, value: t.Any): ############################################################################### -class JSON(Column): # lgtm[py/missing-equals] +class JSON(Column): """ Used for storing JSON strings. The data is stored as text. This can be preferable to JSONB if you just want to store and retrieve JSON without From d5123e94337fe4ee05816b890cfdbcc4a897009c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominic=20D=C4=85browski?= <44851017+blazerman53@users.noreply.github.com> Date: Sat, 17 Dec 2022 16:17:49 +0100 Subject: [PATCH 427/727] Removed references to LGTM (#724) --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 2d7b1eef1..75168c26a 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,6 @@ ![Release](https://github.com/piccolo-orm/piccolo/actions/workflows/release.yaml/badge.svg) [![Documentation Status](https://readthedocs.org/projects/piccolo-orm/badge/?version=latest)](https://piccolo-orm.readthedocs.io/en/latest/?badge=latest) [![PyPI](https://img.shields.io/pypi/v/piccolo?color=%2334D058&label=pypi)](https://pypi.org/project/piccolo/) -[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/piccolo-orm/piccolo.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/piccolo-orm/piccolo/context:python) -[![Total alerts](https://img.shields.io/lgtm/alerts/g/piccolo-orm/piccolo.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/piccolo-orm/piccolo/alerts/) [![codecov](https://codecov.io/gh/piccolo-orm/piccolo/branch/master/graph/badge.svg?token=V19CWH7MXX)](https://codecov.io/gh/piccolo-orm/piccolo) Piccolo is a fast, user friendly ORM and query builder which supports asyncio. [Read the docs](https://piccolo-orm.readthedocs.io/en/latest/). From aa5ed9fb6346b8c5a7b84d97845357f596f57479 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 19 Dec 2022 14:30:53 +0000 Subject: [PATCH 428/727] minor tweak to `M2M` so it flattens `all_columns` (#730) --- piccolo/columns/m2m.py | 20 +++++++++++++------- piccolo/query/mixins.py | 9 ++------- piccolo/utils/list.py | 27 +++++++++++++++++++++++++++ tests/columns/test_m2m.py | 35 +++++++++++++++++++++++++++++++++++ tests/utils/test_list.py | 8 ++++++++ 5 files changed, 85 insertions(+), 14 deletions(-) create mode 100644 piccolo/utils/list.py create mode 100644 tests/utils/test_list.py diff --git a/piccolo/columns/m2m.py b/piccolo/columns/m2m.py index 5c696bd2b..bb7f30b39 100644 --- a/piccolo/columns/m2m.py +++ b/piccolo/columns/m2m.py @@ -12,6 +12,7 @@ ForeignKey, LazyTableReference, ) +from piccolo.utils.list import flatten from piccolo.utils.sync import run_sync if t.TYPE_CHECKING: @@ -45,9 +46,9 @@ def __init__( self.m2m = m2m self.load_json = load_json - safe_types = [int, str] + safe_types = (int, str) - # If the columns can be serialised / deserialise as JSON, then we + # If the columns can be serialised / deserialised as JSON, then we # can fetch the data all in one go. self.serialisation_safe = all( (column.__class__.value_type in safe_types) @@ -408,7 +409,10 @@ def __init__( ) def __call__( - self, *columns: Column, as_list: bool = False, load_json: bool = False + self, + *columns: t.Union[Column, t.List[Column]], + as_list: bool = False, + load_json: bool = False, ) -> M2MSelect: """ :param columns: @@ -420,14 +424,16 @@ def __call__( :param load_json: If ``True``, any JSON strings are loaded as Python objects. """ - if not columns: - columns = tuple(self._meta.secondary_table._meta.columns) + columns_ = flatten(columns) + + if not columns_: + columns_ = self._meta.secondary_table._meta.columns - if as_list and len(columns) != 1: + if as_list and len(columns_) != 1: raise ValueError( "`as_list` is only valid with a single column argument" ) return M2MSelect( - *columns, m2m=self, as_list=as_list, load_json=load_json + *columns_, m2m=self, as_list=as_list, load_json=load_json ) diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index 36a2fd80f..582c8bbb1 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -10,6 +10,7 @@ from piccolo.columns.column_types import ForeignKey from piccolo.custom_types import Combinable from piccolo.querystring import QueryString +from piccolo.utils.list import flatten from piccolo.utils.sql_values import convert_to_sql_value if t.TYPE_CHECKING: # pragma: no cover @@ -440,13 +441,7 @@ def columns(self, *columns: t.Union[Selectable, t.List[Selectable]]): in which case we flatten the list. """ - _columns = [] - for column in columns: - if isinstance(column, list): - _columns.extend(column) - else: - _columns.append(column) - + _columns = flatten(columns) combined = list(self.selected_columns) + _columns self.selected_columns = combined diff --git a/piccolo/utils/list.py b/piccolo/utils/list.py new file mode 100644 index 000000000..b26847adc --- /dev/null +++ b/piccolo/utils/list.py @@ -0,0 +1,27 @@ +import typing as t + +ElementType = t.TypeVar("ElementType") + + +def flatten( + items: t.Sequence[t.Union[ElementType, t.List[ElementType]]] +) -> t.List[ElementType]: + """ + Takes a sequence of elements, and flattens it out. For example:: + + >>> flatten(['a', ['b', 'c']]) + ['a', 'b', 'c'] + + We need this for situations like this:: + + await Band.select(Band.name, Band.manager.all_columns()) + + """ + _items: t.List[ElementType] = [] + for item in items: + if isinstance(item, list): + _items.extend(item) + else: + _items.append(item) + + return _items diff --git a/tests/columns/test_m2m.py b/tests/columns/test_m2m.py index 4dff73430..653202ed8 100644 --- a/tests/columns/test_m2m.py +++ b/tests/columns/test_m2m.py @@ -273,6 +273,41 @@ def test_select_id(self): ], ) + @engines_skip("cockroach") + def test_select_all_columns(self): + """ + Make sure ``all_columns`` can be passed in as an argument. ``M2M`` + should flatten the arguments. Reported here: + + https://github.com/piccolo-orm/piccolo/issues/728 + + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + + """ # noqa: E501 + response = Band.select( + Band.name, Band.genres(Genre.all_columns(exclude=(Genre.id,))) + ).run_sync() + self.assertEqual( + response, + [ + { + "name": "Pythonistas", + "genres": [ + {"name": "Rock"}, + {"name": "Folk"}, + ], + }, + {"name": "Rustaceans", "genres": [{"name": "Folk"}]}, + { + "name": "C-Sharps", + "genres": [ + {"name": "Rock"}, + {"name": "Classical"}, + ], + }, + ], + ) + def test_add_m2m(self): """ Make sure we can add items to the joining table. diff --git a/tests/utils/test_list.py b/tests/utils/test_list.py new file mode 100644 index 000000000..e0211c6f7 --- /dev/null +++ b/tests/utils/test_list.py @@ -0,0 +1,8 @@ +from unittest import TestCase + +from piccolo.utils.list import flatten + + +class TestFlatten(TestCase): + def test_flatten(self): + self.assertListEqual(flatten(["a", ["b", "c"]]), ["a", "b", "c"]) From feaaf231b7a86a2765acbca90fe9487e7e384a54 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 19 Dec 2022 14:41:52 +0000 Subject: [PATCH 429/727] bump cockroach (#732) --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 32a64a259..8e32e1ac5 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -138,7 +138,7 @@ jobs: strategy: matrix: python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] - cockroachdb-version: ["v22.2.0-rc.3"] + cockroachdb-version: ["v22.2.0"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} From 1092d490a51401e42bc253bdb477aa96fe656c18 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 19 Dec 2022 16:20:34 +0000 Subject: [PATCH 430/727] added `SelectRaw` (#621) * added `SelectRaw` * fix slots error * skip `SelectRaw` tests for older SQLite versions * skip cockroach in test * make import absolute Was getting a warning from Pylance about the import overshadowing Python's `select` module. * make `SelectRaw` importable from `piccolo.query` * added warning about only using trusted SQL with `SelectRaw` --- docs/src/piccolo/query_types/select.rst | 22 +++++++++++++++++++ piccolo/columns/base.py | 4 ++++ piccolo/columns/combination.py | 17 ++++++++++----- piccolo/query/__init__.py | 2 ++ piccolo/query/methods/count.py | 3 +-- piccolo/query/methods/select.py | 22 +++++++++++++++++++ tests/table/test_select.py | 29 ++++++++++++++++++++++++- 7 files changed, 90 insertions(+), 9 deletions(-) diff --git a/docs/src/piccolo/query_types/select.rst b/docs/src/piccolo/query_types/select.rst index 643aa789e..f570c28ec 100644 --- a/docs/src/piccolo/query_types/select.rst +++ b/docs/src/piccolo/query_types/select.rst @@ -255,6 +255,28 @@ And can use aliases for aggregate functions like this: ------------------------------------------------------------------------------- +SelectRaw +--------- + +In certain situations you may want to have raw SQL in your select query. + +For example, if there's a Postgres function which you want to access, which +isn't supported by Piccolo: + +.. code-block:: python + + from piccolo.query import SelectRaw + + >>> await Band.select( + ... Band.name, + ... SelectRaw("log(popularity) AS log_popularity") + ... ) + [{'name': 'Pythonistas', 'log_popularity': 3.0}] + +.. warning:: Only use SQL that you trust. + +------------------------------------------------------------------------------- + Query clauses ------------- diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index eae8f4ba3..adbfb6248 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -344,6 +344,10 @@ def __deepcopy__(self, memo) -> ColumnMeta: class Selectable(metaclass=ABCMeta): + """ + Anything which inherits from this can be used in a select query. + """ + _alias: t.Optional[str] @abstractmethod diff --git a/piccolo/columns/combination.py b/piccolo/columns/combination.py index fec8c3767..c263e5e51 100644 --- a/piccolo/columns/combination.py +++ b/piccolo/columns/combination.py @@ -105,15 +105,20 @@ def __init__(self, sql: str, *args: t.Any) -> None: """ Execute raw SQL queries in your where clause. Use with caution! - await Band.where( - WhereRaw("name = 'Pythonistas'") - ) + .. code-block:: python + + await Band.where( + WhereRaw("name = 'Pythonistas'") + ) Or passing in parameters: - await Band.where( - WhereRaw("name = {}", 'Pythonistas') - ) + .. code-block:: python + + await Band.where( + WhereRaw("name = {}", 'Pythonistas') + ) + """ self.querystring = QueryString(sql, *args) diff --git a/piccolo/query/__init__.py b/piccolo/query/__init__.py index 8c1a7ccbe..ed4ef65d4 100644 --- a/piccolo/query/__init__.py +++ b/piccolo/query/__init__.py @@ -18,6 +18,7 @@ TableExists, Update, ) +from .methods.select import SelectRaw from .mixins import OrderByRaw __all__ = [ @@ -37,6 +38,7 @@ "Query", "Raw", "Select", + "SelectRaw", "Sum", "TableExists", "Update", diff --git a/piccolo/query/methods/count.py b/piccolo/query/methods/count.py index 7292793f2..426a773cd 100644 --- a/piccolo/query/methods/count.py +++ b/piccolo/query/methods/count.py @@ -4,11 +4,10 @@ from piccolo.custom_types import Combinable from piccolo.query.base import Query +from piccolo.query.methods.select import Select from piccolo.query.mixins import WhereDelegate from piccolo.querystring import QueryString -from .select import Select - if t.TYPE_CHECKING: # pragma: no cover from piccolo.table import Table diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 7f10e004d..9c8f3ce4d 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -39,6 +39,28 @@ def is_numeric_column(column: Column) -> bool: return column.value_type in (int, decimal.Decimal, float) +class SelectRaw(Selectable): + def __init__(self, sql: str, *args: t.Any) -> None: + """ + Execute raw SQL in your select query. + + .. code-block:: python + + >>> await Band.select( + ... Band.name, + ... SelectRaw("log(popularity) AS log_popularity") + ... ) + [{'name': 'Pythonistas', 'log_popularity': 3.0}] + + """ + self.querystring = QueryString(sql, *args) + + def get_select_string( + self, engine_type: str, with_alias: bool = True + ) -> str: + return self.querystring.__str__() + + class Avg(Selectable): """ ``AVG()`` SQL function. Column type must be numeric to run the query. diff --git a/tests/table/test_select.py b/tests/table/test_select.py index af287aad0..b8aad0c23 100644 --- a/tests/table/test_select.py +++ b/tests/table/test_select.py @@ -1,15 +1,20 @@ from unittest import TestCase +import pytest + from piccolo.apps.user.tables import BaseUser from piccolo.columns.combination import WhereRaw from piccolo.query import OrderByRaw -from piccolo.query.methods.select import Avg, Count, Max, Min, Sum +from piccolo.query.methods.select import Avg, Count, Max, Min, SelectRaw, Sum from piccolo.table import create_db_tables_sync, drop_db_tables_sync from tests.base import ( DBTestCase, engine_is, + engine_version_lt, engines_only, engines_skip, + is_running_cockroach, + is_running_sqlite, sqlite_only, ) from tests.example_apps.music.tables import Band, Concert, Manager, Venue @@ -960,6 +965,28 @@ def test_as_alias_with_where_clause(self): response, [{"name": "Pythonistas", "manager_name": "Guido"}] ) + @pytest.mark.skipif( + is_running_sqlite() and engine_version_lt(3.35), + reason="SQLite doesn't have math functions in this version.", + ) + @pytest.mark.skipif( + is_running_cockroach(), + reason=( + "Cockroach raises an error when trying to use the log function." + ), + ) + def test_select_raw(self): + """ + Make sure ``SelectRaw`` can be used in select queries. + """ + self.insert_row() + response = Band.select( + Band.name, SelectRaw("round(log(popularity)) AS popularity_log") + ).run_sync() + self.assertListEqual( + response, [{"name": "Pythonistas", "popularity_log": 3.0}] + ) + class TestSelectSecret(TestCase): def setUp(self): From aee9388a50789ac5d737f16f907be34f5cc9a2cf Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 21 Dec 2022 12:59:28 +0000 Subject: [PATCH 431/727] fixtures insert rows in batches (#734) --- piccolo/apps/fixtures/commands/load.py | 21 +++++++++++++--- piccolo/utils/list.py | 34 ++++++++++++++++++++++++++ tests/utils/test_list.py | 22 ++++++++++++++++- 3 files changed, 72 insertions(+), 5 deletions(-) diff --git a/piccolo/apps/fixtures/commands/load.py b/piccolo/apps/fixtures/commands/load.py index 12d22f204..26363451e 100644 --- a/piccolo/apps/fixtures/commands/load.py +++ b/piccolo/apps/fixtures/commands/load.py @@ -10,9 +10,10 @@ from piccolo.engine import engine_finder from piccolo.table import Table, sort_table_classes from piccolo.utils.encoding import load_json +from piccolo.utils.list import batch -async def load_json_string(json_string: str): +async def load_json_string(json_string: str, chunk_size: int = 1000): """ Parses the JSON string, and inserts the parsed data into the database. """ @@ -67,14 +68,26 @@ async def load_json_string(json_string: str): async with engine.transaction(): for table_class in sorted_table_classes: - await table_class.insert(*data[table_class]).run() + rows = data[table_class] + for chunk in batch(data=rows, chunk_size=chunk_size): + await table_class.insert(*chunk).run() -async def load(path: str = "fixture.json"): + +async def load(path: str = "fixture.json", chunk_size: int = 1000): """ Reads the fixture file, and loads the contents into the database. + + :param path: + The path of the fixture file. + + :param chunk_size: + The maximum number of rows to insert at a time. This is usually + determined by the database adapter, which has a max number of + parameters per query. + """ with open(path, "r") as f: contents = f.read() - await load_json_string(contents) + await load_json_string(contents, chunk_size=chunk_size) diff --git a/piccolo/utils/list.py b/piccolo/utils/list.py index b26847adc..dbd61b289 100644 --- a/piccolo/utils/list.py +++ b/piccolo/utils/list.py @@ -25,3 +25,37 @@ def flatten( _items.append(item) return _items + + +def batch( + data: t.List[ElementType], chunk_size: int +) -> t.List[t.List[ElementType]]: + """ + Breaks the list down into sublists of the given ``chunk_size``. The last + sublist may have fewer elements than ``chunk_size``:: + + >>> batch(['a', 'b', 'c'], 2) + [['a', 'b'], ['c']] + + :param data: + The data to break up into sublists. + :param chunk_size: + How large each sublist should be. + + """ + # TODO: Replace with itertools.batch when available: + # https://docs.python.org/3.12/library/itertools.html#itertools.batched + + if chunk_size <= 0: + raise ValueError("`chunk_size` must be greater than 0.") + + row_count = len(data) + + iterations = int(row_count / chunk_size) + if row_count % chunk_size > 0: + iterations += 1 + + return [ + data[(i * chunk_size) : ((i + 1) * chunk_size)] # noqa: E203 + for i in range(0, iterations) + ] diff --git a/tests/utils/test_list.py b/tests/utils/test_list.py index e0211c6f7..c21cec6b4 100644 --- a/tests/utils/test_list.py +++ b/tests/utils/test_list.py @@ -1,8 +1,28 @@ +import string from unittest import TestCase -from piccolo.utils.list import flatten +from piccolo.utils.list import batch, flatten class TestFlatten(TestCase): def test_flatten(self): self.assertListEqual(flatten(["a", ["b", "c"]]), ["a", "b", "c"]) + + +class TestBatch(TestCase): + def test_batch(self): + self.assertListEqual( + batch([i for i in string.ascii_lowercase], chunk_size=5), + [ + ["a", "b", "c", "d", "e"], + ["f", "g", "h", "i", "j"], + ["k", "l", "m", "n", "o"], + ["p", "q", "r", "s", "t"], + ["u", "v", "w", "x", "y"], + ["z"], + ], + ) + + def test_zero(self): + with self.assertRaises(ValueError): + batch([1, 2, 3], chunk_size=0) From 5e5cabc403e106c0ae1985a35c0d2449ccb76b3c Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 22 Dec 2022 22:34:06 +0000 Subject: [PATCH 432/727] bumped version (#736) --- CHANGES.rst | 27 +++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 5347d9f0c..8ac63254f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,33 @@ Changes ======= +0.103.0 +------- + +``SelectRaw`` +~~~~~~~~~~~~~ + +This allows you to access features in the database which aren't exposed +directly by Piccolo. For example, Postgres functions: + +.. code-block:: python + + from piccolo.query import SelectRaw + + >>> await Band.select( + ... Band.name, + ... SelectRaw("log(popularity) AS log_popularity") + ... ) + [{'name': 'Pythonistas', 'log_popularity': 3.0}] + +Large fixtures +~~~~~~~~~~~~~~ + +Piccolo can now load large fixtures using ``piccolo fixtures load``. The +rows are inserted in batches, so the database adapter doesn't raise any errors. + +------------------------------------------------------------------------------- + 0.102.0 ------- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index cc9a1072b..d7f5ee996 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.102.0" +__VERSION__ = "0.103.0" From fe307d50b78378d799085953daf87277d987a6a1 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 26 Dec 2022 22:30:01 +0000 Subject: [PATCH 433/727] fix starlite integration test (#739) --- piccolo/apps/asgi/commands/new.py | 2 +- .../apps/asgi/commands/templates/app/_starlite_app.py.jinja | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index 499c22d31..4e7d7efd0 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -12,7 +12,7 @@ SERVERS = ["uvicorn", "Hypercorn"] ROUTERS = ["starlette", "fastapi", "blacksheep", "xpresso", "starlite"] ROUTER_DEPENDENCIES = { - "starlite": ["starlite>=1.45.0"], + "starlite": ["starlite>=1.46.0"], "xpresso": ["xpresso==0.43.0", "di==0.72.1"], } diff --git a/piccolo/apps/asgi/commands/templates/app/_starlite_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_starlite_app.py.jinja index 97dbd8623..8db51c61a 100644 --- a/piccolo/apps/asgi/commands/templates/app/_starlite_app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/_starlite_app.py.jinja @@ -15,14 +15,14 @@ from starlite import ( post, ) from starlite.plugins.piccolo_orm import PiccoloORMPlugin -from starlite.template.jinja import JinjaTemplateEngine +from starlite.contrib.jinja import JinjaTemplateEngine from starlite.types import Receive, Scope, Send from home.endpoints import home from home.piccolo_app import APP_CONFIG from home.tables import Task -# mounting Piccolo Admin +# mounting Piccolo Admin @asgi("/admin/", is_mount=True) async def admin(scope: "Scope", receive: "Receive", send: "Send") -> None: await create_admin(tables=APP_CONFIG.table_classes)(scope, receive, send) From 49ea9ca9d74fef148784a40422d7db2a5908407f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Dec 2022 23:04:41 +0000 Subject: [PATCH 434/727] Bump wheel from 0.37.1 to 0.38.1 in /requirements (#738) Bumps [wheel](https://github.com/pypa/wheel) from 0.37.1 to 0.38.1. - [Release notes](https://github.com/pypa/wheel/releases) - [Changelog](https://github.com/pypa/wheel/blob/main/docs/news.rst) - [Commits](https://github.com/pypa/wheel/compare/0.37.1...0.38.1) --- updated-dependencies: - dependency-name: wheel dependency-type: direct:development ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Daniel Townsend --- requirements/dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt index d5a6967cc..511d87655 100644 --- a/requirements/dev-requirements.txt +++ b/requirements/dev-requirements.txt @@ -7,4 +7,4 @@ slotscheck==0.14.0 twine==3.8.0 mypy==0.942 pip-upgrader==1.4.15 -wheel==0.37.1 +wheel==0.38.1 From 36d4bee3a759c8274ec5a4e09494e7ae100ed28b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 2 Jan 2023 18:54:34 +0000 Subject: [PATCH 435/727] improve typing for `await Band.objects().first()` (#562) * improve typing for `await MyTable.objects().first()` * adding type tests * add proper types for `await Band.objects()` * fix type error * fix LGTM warning * add generic types to `Create` class * wip * add `_data` argument to `Table.__init__` * add `assert_type` test for `get_or_create` * add returning clause to update query * wip * add missing import * fix mypy errors * add more type checks * fix pytest warning about unrecognised marker * make proxying work correctly * fix callbacks * fix frozen query tests * add correct types for `select` queries * added more type tests * fix mypy error * fix frozen query performance * fix cockroach error * fix starlite integration test * proxy `__str__` method to query * make frozen query test more predictable in terms of speed * remove unused import * fix circular imports * added proxy * use `Self` in a few more places --- .gitignore | 5 + piccolo/apps/migrations/commands/clean.py | 2 +- piccolo/custom_types.py | 5 + piccolo/engine/postgres.py | 3 +- piccolo/engine/sqlite.py | 3 +- piccolo/query/base.py | 97 ++++----- piccolo/query/methods/alter.py | 5 +- piccolo/query/methods/count.py | 10 +- piccolo/query/methods/delete.py | 7 +- piccolo/query/methods/exists.py | 14 +- piccolo/query/methods/insert.py | 16 +- piccolo/query/methods/objects.py | 248 ++++++++++++++-------- piccolo/query/methods/refresh.py | 9 +- piccolo/query/methods/select.py | 159 +++++++++++--- piccolo/query/methods/table_exists.py | 3 +- piccolo/query/methods/update.py | 9 +- piccolo/query/mixins.py | 4 - piccolo/query/proxy.py | 69 ++++++ piccolo/table.py | 25 ++- pyproject.toml | 1 + requirements/dev-requirements.txt | 2 +- requirements/requirements.txt | 2 +- tests/query/test_freeze.py | 29 ++- tests/table/test_objects.py | 25 +++ tests/table/test_output.py | 2 +- tests/type_checking.py | 91 ++++++++ 26 files changed, 630 insertions(+), 215 deletions(-) create mode 100644 piccolo/query/proxy.py create mode 100644 tests/type_checking.py diff --git a/.gitignore b/.gitignore index c14d1f2f0..b318c8ebb 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,8 @@ prof/ .env/ .venv/ result.json + +# CockroachDB +cockroach-data/ +heap_profiler/ +goroutine_dump/ diff --git a/piccolo/apps/migrations/commands/clean.py b/piccolo/apps/migrations/commands/clean.py index 08c075672..e7ef22091 100644 --- a/piccolo/apps/migrations/commands/clean.py +++ b/piccolo/apps/migrations/commands/clean.py @@ -37,7 +37,7 @@ def get_migration_ids_to_remove(self) -> t.List[str]: if len(migration_ids) > 0: query = query.where(Migration.name.not_in(migration_ids)) - return query.run_sync() + return t.cast(t.List[str], query.run_sync()) async def run(self): print("Checking the migration table ...") diff --git a/piccolo/custom_types.py b/piccolo/custom_types.py index 0efd1b3d3..2101bc689 100644 --- a/piccolo/custom_types.py +++ b/piccolo/custom_types.py @@ -4,12 +4,17 @@ if t.TYPE_CHECKING: # pragma: no cover from piccolo.columns.combination import And, Or, Where, WhereRaw # noqa + from piccolo.table import Table Combinable = t.Union["Where", "WhereRaw", "And", "Or"] Iterable = t.Iterable[t.Any] +TableInstance = t.TypeVar("TableInstance", bound="Table") +QueryResponseType = t.TypeVar("QueryResponseType", bound=t.Any) + + ############################################################################### # For backwards compatibility: diff --git a/piccolo/engine/postgres.py b/piccolo/engine/postgres.py index c8f91f430..41ffb7731 100644 --- a/piccolo/engine/postgres.py +++ b/piccolo/engine/postgres.py @@ -7,7 +7,6 @@ from piccolo.engine.base import Batch, Engine from piccolo.engine.exceptions import TransactionError from piccolo.query.base import DDL, Query -from piccolo.query.methods.objects import Create, GetOrCreate from piccolo.querystring import QueryString from piccolo.utils.lazy_loader import LazyLoader from piccolo.utils.sync import run_sync @@ -100,6 +99,8 @@ def add(self, *query: Query): self.queries += list(query) async def run(self): + from piccolo.query.methods.objects import Create, GetOrCreate + try: async with self.engine.transaction(): for query in self.queries: diff --git a/piccolo/engine/sqlite.py b/piccolo/engine/sqlite.py index 77780e141..441dff8c9 100644 --- a/piccolo/engine/sqlite.py +++ b/piccolo/engine/sqlite.py @@ -13,7 +13,6 @@ from piccolo.engine.base import Batch, Engine from piccolo.engine.exceptions import TransactionError from piccolo.query.base import DDL, Query -from piccolo.query.methods.objects import Create, GetOrCreate from piccolo.querystring import QueryString from piccolo.utils.encoding import dump_json, load_json from piccolo.utils.lazy_loader import LazyLoader @@ -278,6 +277,8 @@ def add(self, *query: Query): self.queries += list(query) async def run(self): + from piccolo.query.methods.objects import Create, GetOrCreate + try: async with self.engine.transaction( transaction_type=self.transaction_type diff --git a/piccolo/query/base.py b/piccolo/query/base.py index 3beae8017..a7761b8c6 100644 --- a/piccolo/query/base.py +++ b/piccolo/query/base.py @@ -1,18 +1,18 @@ from __future__ import annotations -import itertools import typing as t from time import time from piccolo.columns.column_types import JSON, JSONB -from piccolo.query.mixins import CallbackType, ColumnsDelegate +from piccolo.custom_types import QueryResponseType, TableInstance +from piccolo.query.mixins import ColumnsDelegate from piccolo.querystring import QueryString -from piccolo.utils.encoding import dump_json, load_json +from piccolo.utils.encoding import load_json from piccolo.utils.objects import make_nested_object from piccolo.utils.sync import run_sync if t.TYPE_CHECKING: # pragma: no cover - from piccolo.query.mixins import CallbackDelegate, OutputDelegate + from piccolo.query.mixins import OutputDelegate from piccolo.table import Table # noqa @@ -25,13 +25,13 @@ def __exit__(self, exception_type, exception, traceback): print(f"Duration: {self.end - self.start}s") -class Query: +class Query(t.Generic[TableInstance, QueryResponseType]): __slots__ = ("table", "_frozen_querystrings") def __init__( self, - table: t.Type[Table], + table: t.Type[TableInstance], frozen_querystrings: t.Optional[t.Sequence[QueryString]] = None, ): self.table = table @@ -45,7 +45,7 @@ def engine_type(self) -> str: else: raise ValueError("Engine isn't defined.") - async def _process_results(self, results): # noqa: C901 + async def _process_results(self, results): if results: keys = results[0].keys() keys = [i.replace("$", ".") for i in keys] @@ -117,35 +117,13 @@ async def _process_results(self, results): # noqa: C901 if output: if output._output.as_objects: - # When using .first() we get a single row, not a list - # of rows. - if type(raw) is list: - if output._output.nested: - raw = [ - make_nested_object(row, self.table) for row in raw - ] - else: - raw = [ - self.table(**columns, _exists_in_db=True) - for columns in raw - ] - elif raw is not None: - if output._output.nested: - raw = make_nested_object(raw, self.table) - else: - raw = self.table(**raw, _exists_in_db=True) - elif type(raw) is list: - if output._output.as_list: - if len(raw) == 0: - return [] - if len(raw[0].keys()) != 1: - raise ValueError( - "Each row returned more than one value" - ) - else: - raw = list(itertools.chain(*[j.values() for j in raw])) - if output._output.as_json: - raw = dump_json(raw) + if output._output.nested: + raw = [make_nested_object(row, self.table) for row in raw] + else: + raw = [ + self.table(**columns, _exists_in_db=True) + for columns in raw + ] return raw @@ -157,14 +135,16 @@ def _validate(self): """ pass - def __await__(self): + def __await__(self) -> t.Generator[None, None, QueryResponseType]: """ If the user doesn't explicity call .run(), proxy to it as a convenience. """ return self.run().__await__() - async def run(self, node: t.Optional[str] = None, in_pool: bool = True): + async def _run( + self, node: t.Optional[str] = None, in_pool: bool = True + ) -> QueryResponseType: """ Run the query on the database. @@ -195,22 +175,11 @@ async def run(self, node: t.Optional[str] = None, in_pool: bool = True): querystrings = self.querystrings - callback: t.Optional[CallbackDelegate] = getattr( - self, "callback_delegate", None - ) - if len(querystrings) == 1: results = await engine.run_querystring( querystrings[0], in_pool=in_pool ) - processed_results = await self._process_results(results) - - if callback: - processed_results = await callback.invoke( - processed_results, kind=CallbackType.success - ) - - return processed_results + return await self._process_results(results) else: responses = [] for querystring in querystrings: @@ -219,15 +188,20 @@ async def run(self, node: t.Optional[str] = None, in_pool: bool = True): ) processed_results = await self._process_results(results) - if callback: - processed_results = await callback.invoke( - processed_results, kind=CallbackType.success - ) - responses.append(processed_results) - return responses + return t.cast(QueryResponseType, responses) - def run_sync(self, timed=False, in_pool=False, *args, **kwargs): + async def run( + self, node: t.Optional[str] = None, in_pool: bool = True + ) -> QueryResponseType: + return await self._run(node=node, in_pool=in_pool) + + def run_sync( + self, + node: t.Optional[str] = None, + timed: bool = False, + in_pool: bool = False, + ) -> QueryResponseType: """ A convenience method for running the coroutine synchronously. @@ -241,7 +215,7 @@ def run_sync(self, timed=False, in_pool=False, *args, **kwargs): `issue 505 `_. """ - coroutine = self.run(in_pool=in_pool, *args, **kwargs) + coroutine = self.run(node=node, in_pool=in_pool) if not timed: return run_sync(coroutine) @@ -366,6 +340,9 @@ def __str__(self) -> str: return "; ".join([i.__str__() for i in self.querystrings]) +############################################################################### + + class FrozenQuery: def __init__(self, query: Query): self.query = query @@ -389,6 +366,9 @@ def __str__(self) -> str: return self.query.__str__() +############################################################################### + + class DDL: __slots__ = ("table",) @@ -464,7 +444,6 @@ async def run(self, in_pool=True): if len(self.ddl) == 1: return await engine.run_ddl(self.ddl[0], in_pool=in_pool) responses = [] - # TODO - run in a transaction for ddl in self.ddl: response = await engine.run_ddl(ddl, in_pool=in_pool) responses.append(response) diff --git a/piccolo/query/methods/alter.py b/piccolo/query/methods/alter.py index 115a35d0d..c51caa14d 100644 --- a/piccolo/query/methods/alter.py +++ b/piccolo/query/methods/alter.py @@ -295,7 +295,7 @@ def __init__(self, table: t.Type[Table], **kwargs): self._set_null: t.List[SetNull] = [] self._set_unique: t.List[SetUnique] = [] - def add_column(self, name: str, column: Column) -> Alter: + def add_column(self: Self, name: str, column: Column) -> Self: """ Band.alter().add_column(‘members’, Integer()) """ @@ -531,3 +531,6 @@ def default_ddl(self) -> t.Sequence[str]: query += ",".join(f" {i}" for i in alterations) return [query] + + +Self = t.TypeVar("Self", bound=Alter) diff --git a/piccolo/query/methods/count.py b/piccolo/query/methods/count.py index 426a773cd..75a6dd467 100644 --- a/piccolo/query/methods/count.py +++ b/piccolo/query/methods/count.py @@ -19,10 +19,15 @@ def __init__(self, table: t.Type[Table], **kwargs): super().__init__(table, **kwargs) self.where_delegate = WhereDelegate() - def where(self, *where: Combinable) -> Count: + ########################################################################### + # Clauses + + def where(self: Self, *where: Combinable) -> Self: self.where_delegate.where(*where) return self + ########################################################################### + async def response_handler(self, response) -> bool: return response[0]["count"] @@ -36,3 +41,6 @@ def default_querystrings(self) -> t.Sequence[QueryString]: select.querystrings[0], ) ] + + +Self = t.TypeVar("Self", bound=Count) diff --git a/piccolo/query/methods/delete.py b/piccolo/query/methods/delete.py index d5fa07838..121375c23 100644 --- a/piccolo/query/methods/delete.py +++ b/piccolo/query/methods/delete.py @@ -30,11 +30,11 @@ def __init__(self, table: t.Type[Table], force: bool = False, **kwargs): self.returning_delegate = ReturningDelegate() self.where_delegate = WhereDelegate() - def where(self, *where: Combinable) -> Delete: + def where(self: Self, *where: Combinable) -> Self: self.where_delegate.where(*where) return self - def returning(self, *columns: Column) -> Delete: + def returning(self: Self, *columns: Column) -> Self: self.returning_delegate.returning(columns) return self @@ -72,3 +72,6 @@ def default_querystrings(self) -> t.Sequence[QueryString]: ) return [querystring] + + +Self = t.TypeVar("Self", bound=Delete) diff --git a/piccolo/query/methods/exists.py b/piccolo/query/methods/exists.py index 01e0ddb7e..26d25e03e 100644 --- a/piccolo/query/methods/exists.py +++ b/piccolo/query/methods/exists.py @@ -2,24 +2,21 @@ import typing as t -from piccolo.custom_types import Combinable +from piccolo.custom_types import Combinable, TableInstance from piccolo.query.base import Query from piccolo.query.methods.select import Select from piccolo.query.mixins import WhereDelegate from piccolo.querystring import QueryString -if t.TYPE_CHECKING: # pragma: no cover - from piccolo.table import Table - -class Exists(Query): +class Exists(Query[TableInstance, bool]): __slots__ = ("where_delegate",) - def __init__(self, table: t.Type[Table], **kwargs): + def __init__(self, table: t.Type[TableInstance], **kwargs): super().__init__(table, **kwargs) self.where_delegate = WhereDelegate() - def where(self, *where: Combinable) -> Exists: + def where(self: Self, *where: Combinable) -> Self: self.where_delegate.where(*where) return self @@ -36,3 +33,6 @@ def default_querystrings(self) -> t.Sequence[QueryString]: 'SELECT EXISTS({}) AS "exists"', select.querystrings[0] ) ] + + +Self = t.TypeVar("Self", bound=Exists) diff --git a/piccolo/query/methods/insert.py b/piccolo/query/methods/insert.py index b1b439626..9f31f445a 100644 --- a/piccolo/query/methods/insert.py +++ b/piccolo/query/methods/insert.py @@ -2,6 +2,7 @@ import typing as t +from piccolo.custom_types import TableInstance from piccolo.query.base import Query from piccolo.query.mixins import AddDelegate, ReturningDelegate from piccolo.querystring import QueryString @@ -11,10 +12,14 @@ from piccolo.table import Table -class Insert(Query): +class Insert( + t.Generic[TableInstance], Query[TableInstance, t.List[t.Dict[str, t.Any]]] +): __slots__ = ("add_delegate", "returning_delegate") - def __init__(self, table: t.Type[Table], *instances: Table, **kwargs): + def __init__( + self, table: t.Type[TableInstance], *instances: TableInstance, **kwargs + ): super().__init__(table, **kwargs) self.add_delegate = AddDelegate() self.returning_delegate = ReturningDelegate() @@ -23,11 +28,11 @@ def __init__(self, table: t.Type[Table], *instances: Table, **kwargs): ########################################################################### # Clauses - def add(self, *instances: Table) -> Insert: + def add(self: Self, *instances: Table) -> Self: self.add_delegate.add(*instances, table_class=self.table) return self - def returning(self, *columns: Column) -> Insert: + def returning(self: Self, *columns: Column) -> Self: self.returning_delegate.returning(columns) return self @@ -81,3 +86,6 @@ def default_querystrings(self) -> t.Sequence[QueryString]: ] return [querystring] + + +Self = t.TypeVar("Self", bound=Insert) diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index f71740def..db9e43ccc 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -1,13 +1,13 @@ from __future__ import annotations import typing as t -from dataclasses import dataclass from piccolo.columns.column_types import ForeignKey from piccolo.columns.combination import And, Where -from piccolo.custom_types import Combinable +from piccolo.custom_types import Combinable, TableInstance from piccolo.engine.base import Batch from piccolo.query.base import Query +from piccolo.query.methods.select import Select from piccolo.query.mixins import ( AsOfDelegate, CallbackDelegate, @@ -20,30 +20,49 @@ PrefetchDelegate, WhereDelegate, ) +from piccolo.query.proxy import Proxy from piccolo.querystring import QueryString from piccolo.utils.dictionary import make_nested from piccolo.utils.sync import run_sync -from .select import Select - if t.TYPE_CHECKING: # pragma: no cover from piccolo.columns import Column - from piccolo.table import Table -@dataclass -class GetOrCreate: - query: Objects - where: Combinable - defaults: t.Dict[t.Union[Column, str], t.Any] +############################################################################### + + +class GetOrCreate( + Proxy["Objects[TableInstance]", TableInstance], t.Generic[TableInstance] +): + def __init__( + self, + query: Objects[TableInstance], + table_class: t.Type[TableInstance], + where: Combinable, + defaults: t.Dict[Column, t.Any], + ): + self.query = query + self.table_class = table_class + self.where = where + self.defaults = defaults + + async def run( + self, node: t.Optional[str] = None, in_pool: bool = True + ) -> TableInstance: + """ + :raises ValueError: + If more than one matching row is found. - async def run(self): - instance = await self.query.get(self.where).run() + """ + instance = await self.query.get(self.where).run( + node=node, in_pool=in_pool + ) if instance: instance._was_created = False return instance - instance = self.query.table() + instance = self.table_class(_data=self.defaults) # If it's a complex `where`, there can be several column values to # extract e.g. (Band.name == 'Pythonistas') & (Band.popularity == 1000) @@ -60,12 +79,7 @@ async def run(self): # to this table. setattr(instance, column._meta.name, value) - for column, value in self.defaults.items(): - if isinstance(column, str): - column = instance._meta.get_column_by_name(column) - setattr(instance, column._meta.name, value) - - await instance.save().run() + await instance.save().run(node=node, in_pool=in_pool) # If the user wants us to prefetch related objects, for example: # @@ -85,46 +99,86 @@ async def run(self): .run() ) + instance = t.cast(TableInstance, instance) instance._was_created = True return instance - def __await__(self): - """ - If the user doesn't explicity call .run(), proxy to it as a - convenience. - """ - return self.run().__await__() - def run_sync(self): - return run_sync(self.run()) +class Get( + Proxy["First[TableInstance]", t.Optional[TableInstance]], + t.Generic[TableInstance], +): + pass + + +class First( + Proxy["Objects[TableInstance]", t.Optional[TableInstance]], + t.Generic[TableInstance], +): + async def run( + self, node: t.Optional[str] = None, in_pool: bool = True + ) -> t.Optional[TableInstance]: + objects = await self.query.run( + node=node, in_pool=in_pool, use_callbacks=False + ) + + results = objects[0] if objects else None + + modified_response: t.Optional[ + TableInstance + ] = await self.query.callback_delegate.invoke( + results=results, kind=CallbackType.success + ) + return modified_response - def prefetch(self, *fk_columns) -> GetOrCreate: - self.query.prefetch(*fk_columns) - return self +class Create(t.Generic[TableInstance]): + """ + This is provided as a simple convenience. Rather than running:: + + band = Band(name='Pythonistas') + await band.save() + + We can instead do it in a single line:: -@dataclass -class Create: - table_class: t.Type[Table] - columns: t.Dict[str, t.Any] + band = Band.objects().create(name='Pythonistas') + + """ - async def run(self): + def __init__( + self, + table_class: t.Type[TableInstance], + columns: t.Dict[str, t.Any], + ): + self.table_class = table_class + self.columns = columns + + async def run( + self, + node: t.Optional[str] = None, + in_pool: bool = True, + ) -> TableInstance: instance = self.table_class(**self.columns) - await instance.save().run() + await instance.save().run(node=node, in_pool=in_pool) return instance - def __await__(self): + def __await__(self) -> t.Generator[None, None, TableInstance]: """ If the user doesn't explicity call .run(), proxy to it as a convenience. """ return self.run().__await__() - def run_sync(self): - return run_sync(self.run()) + def run_sync(self, *args, **kwargs) -> TableInstance: + return run_sync(self.run(*args, **kwargs)) + +############################################################################### -class Objects(Query): + +class Objects( + Query[TableInstance, t.List[TableInstance]], t.Generic[TableInstance] +): """ Almost identical to select, except you have to select all fields, and table instances are returned, rather than just data. @@ -144,7 +198,7 @@ class Objects(Query): def __init__( self, - table: t.Type[Table], + table: t.Type[TableInstance], prefetch: t.Sequence[t.Union[ForeignKey, t.List[ForeignKey]]] = (), **kwargs, ): @@ -160,18 +214,18 @@ def __init__( self.prefetch(*prefetch) self.where_delegate = WhereDelegate() - def output(self, load_json: bool = False) -> Objects: + def output(self: Self, load_json: bool = False) -> Self: self.output_delegate.output( as_list=False, as_json=False, load_json=load_json ) return self def callback( - self, + self: Self, callbacks: t.Union[t.Callable, t.List[t.Callable]], *, on: CallbackType = CallbackType.success, - ) -> Objects: + ) -> Self: self.callback_delegate.callback(callbacks, on=on) return self @@ -179,46 +233,23 @@ def as_of(self, interval: str = "-1s") -> Objects: self.as_of_delegate.as_of(interval) return self - def limit(self, number: int) -> Objects: + def limit(self: Self, number: int) -> Self: self.limit_delegate.limit(number) return self - def first(self) -> Objects: - self.limit_delegate.first() - return self - def prefetch( - self, *fk_columns: t.Union[ForeignKey, t.List[ForeignKey]] - ) -> Objects: + self: Self, *fk_columns: t.Union[ForeignKey, t.List[ForeignKey]] + ) -> Self: self.prefetch_delegate.prefetch(*fk_columns) return self - def get(self, where: Combinable) -> Objects: - self.where_delegate.where(where) - self.limit_delegate.first() - return self - - def offset(self, number: int) -> Objects: + def offset(self: Self, number: int) -> Self: self.offset_delegate.offset(number) return self - def get_or_create( - self, - where: Combinable, - defaults: t.Dict[t.Union[Column, str], t.Any] = None, - ): - if defaults is None: - defaults = {} - return GetOrCreate(query=self, where=where, defaults=defaults) - - def create(self, **columns: t.Any): - return Create(table_class=self.table, columns=columns) - def order_by( - self, - *columns: t.Union[Column, str, OrderByRaw], - ascending: bool = True, - ) -> Objects: + self: Self, *columns: t.Union[Column, str, OrderByRaw], ascending=True + ) -> Self: _columns: t.List[t.Union[Column, OrderByRaw]] = [] for column in columns: if isinstance(column, str): @@ -229,12 +260,39 @@ def order_by( self.order_by_delegate.order_by(*_columns, ascending=ascending) return self - def where(self, *where: Combinable) -> Objects: + def where(self: Self, *where: Combinable) -> Self: self.where_delegate.where(*where) return self + ########################################################################### + + def first(self: Self) -> First[TableInstance]: + self.limit_delegate.limit(1) + return First[TableInstance](query=self) + + def get(self: Self, where: Combinable) -> Get[TableInstance]: + self.where_delegate.where(where) + self.limit_delegate.limit(1) + return Get[TableInstance](query=First[TableInstance](query=self)) + + def get_or_create( + self: Self, + where: Combinable, + defaults: t.Dict[Column, t.Any] = None, + ) -> GetOrCreate[TableInstance]: + if defaults is None: + defaults = {} + return GetOrCreate[TableInstance]( + query=self, table_class=self.table, where=where, defaults=defaults + ) + + def create(self: Self, **columns: t.Any) -> Create[TableInstance]: + return Create[TableInstance](table_class=self.table, columns=columns) + + ########################################################################### + async def batch( - self, + self: Self, batch_size: t.Optional[int] = None, node: t.Optional[str] = None, **kwargs, @@ -246,14 +304,7 @@ async def batch( return await self.table._meta.db.batch(self, **kwargs) async def response_handler(self, response): - if self.limit_delegate._first: - if len(response) == 0: - return None - if self.output_delegate._output.nested: - return make_nested(response[0]) - else: - return response[0] - elif self.output_delegate._output.nested: + if self.output_delegate._output.nested: return [make_nested(i) for i in response] else: return response @@ -287,3 +338,34 @@ def default_querystrings(self) -> t.Sequence[QueryString]: select.output_delegate.output(nested=True) return select.querystrings + + ########################################################################### + + async def run( + self, + node: t.Optional[str] = None, + in_pool: bool = True, + use_callbacks: bool = True, + ) -> t.List[TableInstance]: + results = await super().run(node=node, in_pool=in_pool) + + if use_callbacks: + # With callbacks, the user can return any data that they want. + # Assume that most of the time they will still return a list of + # Table instances. + modified: t.List[ + TableInstance + ] = await self.callback_delegate.invoke( + results, kind=CallbackType.success + ) + return modified + else: + return results + + def __await__( + self, + ) -> t.Generator[None, None, t.List[TableInstance]]: + return super().__await__() + + +Self = t.TypeVar("Self", bound=Objects) diff --git a/piccolo/query/methods/refresh.py b/piccolo/query/methods/refresh.py index 9152638e9..cc7f3764f 100644 --- a/piccolo/query/methods/refresh.py +++ b/piccolo/query/methods/refresh.py @@ -40,7 +40,9 @@ def _columns(self) -> t.Sequence[Column]: i for i in self.instance._meta.columns if not i._meta.primary_key ] - async def run(self) -> Table: + async def run( + self, in_pool: bool = True, node: t.Optional[str] = None + ) -> Table: """ Run it asynchronously. For example:: @@ -73,6 +75,7 @@ async def run(self) -> Table: await instance.select(*columns) .where(pk_column == primary_key_value) .first() + .run(node=node, in_pool=in_pool) ) if updated_values is None: @@ -92,11 +95,11 @@ def __await__(self): """ return self.run().__await__() - def run_sync(self) -> Table: + def run_sync(self, *args, **kwargs) -> Table: """ Run it synchronously. For example:: my_instance.refresh().run_sync() """ - return run_sync(self.run()) + return run_sync(self.run(*args, **kwargs)) diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 9c8f3ce4d..62a5a1759 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -9,6 +9,7 @@ from piccolo.columns.column_types import JSON, JSONB, PrimaryKey from piccolo.columns.m2m import M2MSelect from piccolo.columns.readable import Readable +from piccolo.custom_types import TableInstance from piccolo.engine.base import Batch from piccolo.query.base import Query from piccolo.query.mixins import ( @@ -25,9 +26,10 @@ OutputDelegate, WhereDelegate, ) +from piccolo.query.proxy import Proxy from piccolo.querystring import QueryString from piccolo.utils.dictionary import make_nested -from piccolo.utils.encoding import load_json +from piccolo.utils.encoding import dump_json, load_json from piccolo.utils.warnings import colored_warning if t.TYPE_CHECKING: # pragma: no cover @@ -236,7 +238,76 @@ def get_select_string( return f'SUM({column_name}) AS "{self._alias}"' -class Select(Query): +OptionalDict = t.Optional[t.Dict[str, t.Any]] + + +class First(Proxy["Select", OptionalDict]): + """ + This is for static typing purposes. + """ + + def __init__(self, query: Select): + self.query = query + + async def run( + self, + node: t.Optional[str] = None, + in_pool: bool = True, + ) -> OptionalDict: + rows = await self.query.run( + node=node, in_pool=in_pool, use_callbacks=False + ) + results = rows[0] if rows else None + + modified_response = await self.query.callback_delegate.invoke( + results=results, kind=CallbackType.success + ) + return modified_response + + +class SelectList(Proxy["Select", t.List]): + """ + This is for static typing purposes. + """ + + async def run( + self, + node: t.Optional[str] = None, + in_pool: bool = True, + ) -> t.List: + rows = await self.query.run( + node=node, in_pool=in_pool, use_callbacks=False + ) + + if len(rows) == 0: + response = [] + else: + if len(rows[0].keys()) != 1: + raise ValueError("Each row returned more than one value") + + response = list(itertools.chain(*[j.values() for j in rows])) + + modified_response = await self.query.callback_delegate.invoke( + results=response, kind=CallbackType.success + ) + return modified_response + + +class SelectJSON(Proxy["Select", str]): + """ + This is for static typing purposes. + """ + + async def run( + self, + node: t.Optional[str] = None, + in_pool: bool = True, + ) -> str: + rows = await self.query.run(node=node, in_pool=in_pool) + return dump_json(rows) + + +class Select(Query[TableInstance, t.List[t.Dict[str, t.Any]]]): __slots__ = ( "columns_list", "exclude_secrets", @@ -254,7 +325,7 @@ class Select(Query): def __init__( self, - table: t.Type[Table], + table: t.Type[TableInstance], columns_list: t.Sequence[t.Union[Selectable, str]] = None, exclude_secrets: bool = False, **kwargs, @@ -277,16 +348,16 @@ def __init__( self.columns(*columns_list) - def columns(self, *columns: t.Union[Selectable, str]) -> Select: + def columns(self: Self, *columns: t.Union[Selectable, str]) -> Self: _columns = self.table._process_column_args(*columns) self.columns_delegate.columns(*_columns) return self - def distinct(self) -> Select: + def distinct(self: Self) -> Self: self.distinct_delegate.distinct() return self - def group_by(self, *columns: t.Union[Column, str]) -> Select: + def group_by(self: Self, *columns: t.Union[Column, str]) -> Self: _columns: t.List[Column] = [ i for i in self.table._process_column_args(*columns) @@ -295,19 +366,19 @@ def group_by(self, *columns: t.Union[Column, str]) -> Select: self.group_by_delegate.group_by(*_columns) return self - def as_of(self, interval: str = "-1s") -> Select: + def as_of(self: Self, interval: str = "-1s") -> Self: self.as_of_delegate.as_of(interval) return self - def limit(self, number: int) -> Select: + def limit(self: Self, number: int) -> Self: self.limit_delegate.limit(number) return self - def first(self) -> Select: - self.limit_delegate.first() - return self + def first(self) -> First: + self.limit_delegate.limit(1) + return First(query=self) - def offset(self, number: int) -> Select: + def offset(self: Self, number: int) -> Self: self.offset_delegate.offset(number) return self @@ -458,22 +529,14 @@ async def response_handler(self, response): # no columns were selected from related tables. was_select_star = len(self.columns_delegate.selected_columns) == 0 - if self.limit_delegate._first: - if len(response) == 0: - return None - - if self.output_delegate._output.nested and not was_select_star: - return make_nested(response[0]) - else: - return response[0] - elif self.output_delegate._output.nested and not was_select_star: + if self.output_delegate._output.nested and not was_select_star: return [make_nested(i) for i in response] else: return response def order_by( - self, *columns: t.Union[Column, str, OrderByRaw], ascending=True - ) -> Select: + self: Self, *columns: t.Union[Column, str, OrderByRaw], ascending=True + ) -> Self: """ :param columns: Either a :class:`piccolo.columns.base.Column` instance, a string @@ -491,31 +554,53 @@ def order_by( self.order_by_delegate.order_by(*_columns, ascending=ascending) return self + @t.overload + def output(self: Self, *, as_list: bool) -> SelectList: + ... + + @t.overload + def output(self: Self, *, as_json: bool) -> SelectJSON: + ... + + @t.overload + def output(self: Self, *, load_json: bool) -> Self: + ... + + @t.overload + def output(self: Self, *, nested: bool) -> Self: + ... + def output( - self, + self: Self, + *, as_list: bool = False, as_json: bool = False, load_json: bool = False, nested: bool = False, - ) -> Select: + ): self.output_delegate.output( as_list=as_list, as_json=as_json, load_json=load_json, nested=nested, ) + if as_list: + return SelectList(query=self) + elif as_json: + return SelectJSON(query=self) + return self def callback( - self, + self: Self, callbacks: t.Union[t.Callable, t.List[t.Callable]], *, on: CallbackType = CallbackType.success, - ) -> Select: + ) -> Self: self.callback_delegate.callback(callbacks, on=on) return self - def where(self, *where: Combinable) -> Select: + def where(self: Self, *where: Combinable) -> Self: self.where_delegate.where(*where) return self @@ -686,3 +771,21 @@ def default_querystrings(self) -> t.Sequence[QueryString]: querystring = QueryString(query, *args) return [querystring] + + async def run( + self, + node: t.Optional[str] = None, + in_pool: bool = True, + use_callbacks: bool = True, + **kwargs, + ) -> t.List[t.Dict[str, t.Any]]: + results = await super().run(node=node, in_pool=in_pool) + if use_callbacks: + return await self.callback_delegate.invoke( + results, kind=CallbackType.success + ) + else: + return results + + +Self = t.TypeVar("Self", bound=Select) diff --git a/piccolo/query/methods/table_exists.py b/piccolo/query/methods/table_exists.py index bf9afb08a..81332faf3 100644 --- a/piccolo/query/methods/table_exists.py +++ b/piccolo/query/methods/table_exists.py @@ -2,11 +2,12 @@ import typing as t +from piccolo.custom_types import TableInstance from piccolo.query.base import Query from piccolo.querystring import QueryString -class TableExists(Query): +class TableExists(Query[TableInstance, bool]): __slots__: t.Tuple = () diff --git a/piccolo/query/methods/update.py b/piccolo/query/methods/update.py index 6a09c7b9d..b4c980edb 100644 --- a/piccolo/query/methods/update.py +++ b/piccolo/query/methods/update.py @@ -2,7 +2,7 @@ import typing as t -from piccolo.custom_types import Combinable +from piccolo.custom_types import Combinable, TableInstance from piccolo.query.base import Query from piccolo.query.mixins import ( ReturningDelegate, @@ -13,14 +13,13 @@ if t.TYPE_CHECKING: # pragma: no cover from piccolo.columns import Column - from piccolo.table import Table class UpdateError(Exception): pass -class Update(Query): +class Update(Query[TableInstance, t.List[t.Any]]): __slots__ = ( "force", @@ -29,7 +28,9 @@ class Update(Query): "where_delegate", ) - def __init__(self, table: t.Type[Table], force: bool = False, **kwargs): + def __init__( + self, table: t.Type[TableInstance], force: bool = False, **kwargs + ): super().__init__(table, **kwargs) self.force = force self.returning_delegate = ReturningDelegate() diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index 582c8bbb1..397f84f99 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -238,10 +238,6 @@ class LimitDelegate: def limit(self, number: int): self._limit = Limit(number) - def first(self): - self.limit(1) - self._first = True - def copy(self) -> LimitDelegate: _limit = self._limit.copy() if self._limit is not None else None return self.__class__(_limit=_limit, _first=self._first) diff --git a/piccolo/query/proxy.py b/piccolo/query/proxy.py new file mode 100644 index 000000000..7ded47b84 --- /dev/null +++ b/piccolo/query/proxy.py @@ -0,0 +1,69 @@ +import inspect +import typing as t + +from typing_extensions import Protocol + +from piccolo.query.base import FrozenQuery +from piccolo.utils.sync import run_sync + + +class Runnable(Protocol): + async def run(self, node: t.Optional[str] = None, in_pool: bool = True): + ... + + +QueryType = t.TypeVar("QueryType", bound=Runnable) +ResponseType = t.TypeVar("ResponseType") + + +class Proxy(t.Generic[QueryType, ResponseType]): + def __init__(self, query: QueryType): + self.query = query + + async def run( + self, + node: t.Optional[str] = None, + in_pool: bool = True, + ) -> ResponseType: + return await self.query.run(node=node, in_pool=in_pool) + + def run_sync(self, *args, **kwargs) -> ResponseType: + return run_sync(self.run(*args, **kwargs)) + + def __await__( + self, + ) -> t.Generator[None, None, ResponseType]: + """ + If the user doesn't explicity call .run(), proxy to it as a + convenience. + """ + return self.run().__await__() + + def freeze(self): + self.query.freeze() + return FrozenQuery(query=self) + + def __getattr__(self, name: str): + """ + Proxy any attributes to the underlying query, so all of the query + clauses continue to work. + """ + attr = getattr(self.query, name) + + if inspect.ismethod(attr): + # We do this to preserve the fluent interface. + + def proxy(*args, **kwargs): + response = attr(*args, **kwargs) + if isinstance(response, self.query.__class__): + self.query = response + return self + else: + return response + + return proxy + else: + return attr + + def __str__(self) -> str: + return self.query.__str__() diff --git a/piccolo/table.py b/piccolo/table.py index a6cdc64c4..2982180bd 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -26,6 +26,7 @@ ) from piccolo.columns.readable import Readable from piccolo.columns.reference import LAZY_COLUMN_REFERENCES +from piccolo.custom_types import TableInstance from piccolo.engine import Engine, engine_finder from piccolo.query import ( Alter, @@ -43,6 +44,7 @@ ) from piccolo.query.methods.create_index import CreateIndex from piccolo.query.methods.indexes import Indexes +from piccolo.query.methods.objects import First from piccolo.query.methods.refresh import Refresh from piccolo.querystring import QueryString, Unquoted from piccolo.utils import _camel_to_snake @@ -363,6 +365,10 @@ def __init__( self._exists_in_db = _exists_in_db + # This is used by get_or_create to indicate to the user whether it + # was an existing row or not. + self._was_created: t.Optional[bool] = None + for column in self._meta.columns: value = _data.get(column, ...) @@ -404,7 +410,9 @@ def _create_serial_primary_key(cls) -> Serial: return pk @classmethod - def from_dict(cls, data: t.Dict[str, t.Any]) -> Table: + def from_dict( + cls: t.Type[TableInstance], data: t.Dict[str, t.Any] + ) -> TableInstance: """ Used when loading fixtures. It can be overriden by subclasses in case they have specific logic / validation which needs running when loading @@ -513,7 +521,9 @@ def refresh( """ return Refresh(instance=self, columns=columns) - def get_related(self, foreign_key: t.Union[ForeignKey, str]) -> Objects: + def get_related( + self: TableInstance, foreign_key: t.Union[ForeignKey, str] + ) -> First[Table]: """ Used to fetch a ``Table`` instance, for the target of a foreign key. @@ -895,7 +905,9 @@ def ref(cls, column_name: str) -> Column: return _reference_column @classmethod - def insert(cls, *rows: "Table") -> Insert: + def insert( + cls: t.Type[TableInstance], *rows: TableInstance + ) -> Insert[TableInstance]: """ Insert rows into the database. @@ -1022,8 +1034,9 @@ def alter(cls) -> Alter: @classmethod def objects( - cls, *prefetch: t.Union[ForeignKey, t.List[ForeignKey]] - ) -> Objects: + cls: t.Type[TableInstance], + *prefetch: t.Union[ForeignKey, t.List[ForeignKey]], + ) -> Objects[TableInstance]: """ Returns a list of table instances (each representing a row), which you can modify and then call 'save' on, or can delete by calling 'remove'. @@ -1058,7 +1071,7 @@ def objects( """ - return Objects(table=cls, prefetch=prefetch) + return Objects[TableInstance](table=cls, prefetch=prefetch) @classmethod def count(cls) -> Count: diff --git a/pyproject.toml b/pyproject.toml index 1fb257c2a..521e112c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ ignore_missing_imports = true [tool.pytest.ini_options] markers = [ "integration", + "speed" ] [tool.coverage.run] diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt index 511d87655..bd8a17e15 100644 --- a/requirements/dev-requirements.txt +++ b/requirements/dev-requirements.txt @@ -5,6 +5,6 @@ flake8==4.0.1 isort==5.10.1 slotscheck==0.14.0 twine==3.8.0 -mypy==0.942 +mypy==0.961 pip-upgrader==1.4.15 wheel==0.38.1 diff --git a/requirements/requirements.txt b/requirements/requirements.txt index c38e71fb4..f6dbc3d2f 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -3,5 +3,5 @@ colorama>=0.4.0 Jinja2>=2.11.0 targ>=0.3.7 inflection>=0.5.1 -typing-extensions>=3.10.0.0 +typing-extensions>=4.3.0 pydantic[email]>=1.6 diff --git a/tests/query/test_freeze.py b/tests/query/test_freeze.py index 9b264012c..29cb5271f 100644 --- a/tests/query/test_freeze.py +++ b/tests/query/test_freeze.py @@ -1,9 +1,12 @@ import timeit import typing as t from dataclasses import dataclass +from unittest import mock +from piccolo.columns import Integer, Varchar from piccolo.query.base import Query -from tests.base import DBTestCase, sqlite_only +from piccolo.table import Table +from tests.base import AsyncMock, DBTestCase, sqlite_only from tests.example_apps.music.tables import Band @@ -79,12 +82,26 @@ def test_frozen_performance(self): The frozen query performance should exceed the non-frozen. If not, there's a problem. - Only test this on SQLite, as the latency from the database itself - is more predictable than with Postgres, and the test runs quickly. + We mock out the database to make the performance more predictable. """ + db = mock.MagicMock() + db.engine_type = "sqlite" + db.run_querystring = AsyncMock() + db.run_querystring.return_value = [ + {"name": "Pythonistas", "popularity": 1000} + ] + + class Band(Table, db=db): + name = Varchar() + popularity = Integer() + iterations = 50 - query = Band.select().where(Band.name == "Pythonistas").first() + query = ( + Band.select(Band.name) + .where(Band.popularity > 900) + .order_by(Band.name) + ) query_duration = timeit.repeat( lambda: query.run_sync(), repeat=iterations, number=1 ) @@ -96,8 +113,8 @@ def test_frozen_performance(self): # Remove the outliers before comparing self.assertGreater( - sum(sorted(query_duration)[5:-5]), - sum(sorted(frozen_query_duration)[5:-5]), + sum(sorted(query_duration)[10:-10]), + sum(sorted(frozen_query_duration)[10:-10]), ) def test_attribute_access(self): diff --git a/tests/table/test_objects.py b/tests/table/test_objects.py index f8c699ef2..f4d2849df 100644 --- a/tests/table/test_objects.py +++ b/tests/table/test_objects.py @@ -1,3 +1,7 @@ +import typing as t + +from typing_extensions import assert_type + from tests.base import DBTestCase, engines_only, sqlite_only from tests.example_apps.music.tables import Band, Manager @@ -259,3 +263,24 @@ def test_prefetch_new_object(self): self.assertIsInstance(band.manager, Manager) self.assertEqual(band.name, "New Band 2") self.assertEqual(band.manager.name, "Guido") + + +if t.TYPE_CHECKING: + # Making sure the types are inferred correctly by MyPy. + + assert_type(Band.objects().first().run_sync(), t.Optional[Band]) + + assert_type( + Band.objects().get(Band.name == "Pythonistas").run_sync(), + t.Optional[Band], + ) + + assert_type( + Band.objects().get_or_create(Band.name == "Pythonistas").run_sync(), + Band, + ) + + assert_type( + Band.objects().run_sync(), + t.List[Band], + ) diff --git a/tests/table/test_output.py b/tests/table/test_output.py index 0fbaafa0d..97256dfcb 100644 --- a/tests/table/test_output.py +++ b/tests/table/test_output.py @@ -101,6 +101,6 @@ def test_output_nested_with_first(self): .output(nested=True) .run_sync() ) - self.assertEqual( + self.assertDictEqual( response, {"name": "Pythonistas", "manager": {"name": "Guido"}} ) diff --git a/tests/type_checking.py b/tests/type_checking.py new file mode 100644 index 000000000..5c7dffbc9 --- /dev/null +++ b/tests/type_checking.py @@ -0,0 +1,91 @@ +""" +Making sure the types are inferred correctly by MyPy. + +Note: We need type annotations on the function, otherwise MyPy treats every +type inside the function as Any. +""" + +import typing as t + +from typing_extensions import assert_type + +from .example_apps.music.tables import Band, Manager + +if t.TYPE_CHECKING: + + async def objects() -> None: + query = Band.objects() + assert_type(await query, t.List[Band]) + assert_type(await query.run(), t.List[Band]) + assert_type(query.run_sync(), t.List[Band]) + + async def objects_first() -> None: + query = Band.objects().first() + assert_type(await query, t.Optional[Band]) + assert_type(await query.run(), t.Optional[Band]) + assert_type(query.run_sync(), t.Optional[Band]) + + async def get() -> None: + query = Band.objects().get(Band.name == "Pythonistas") + assert_type(await query, t.Optional[Band]) + assert_type(await query.run(), t.Optional[Band]) + assert_type(query.run_sync(), t.Optional[Band]) + + async def get_or_create() -> None: + query = Band.objects().get_or_create(Band.name == "Pythonistas") + assert_type(await query, Band) + assert_type(await query.run(), Band) + assert_type(query.run_sync(), Band) + + async def select() -> None: + query = Band.select() + assert_type(await query, t.List[t.Dict[str, t.Any]]) + assert_type(await query.run(), t.List[t.Dict[str, t.Any]]) + assert_type(query.run_sync(), t.List[t.Dict[str, t.Any]]) + + async def select_first() -> None: + query = Band.select().first() + assert_type(await query, t.Optional[t.Dict[str, t.Any]]) + assert_type(await query.run(), t.Optional[t.Dict[str, t.Any]]) + assert_type(query.run_sync(), t.Optional[t.Dict[str, t.Any]]) + + async def select_list() -> None: + query = Band.select(Band.name).output(as_list=True) + assert_type(await query, t.List) + assert_type(await query.run(), t.List) + assert_type(query.run_sync(), t.List) + # The next step would be to detect that it's t.List[str], but might not + # be possible. + + async def select_as_json() -> None: + query = Band.select(Band.name).output(as_json=True) + assert_type(await query, str) + assert_type(await query.run(), str) + assert_type(query.run_sync(), str) + + async def exists() -> None: + query = Band.exists() + assert_type(await query, bool) + assert_type(await query.run(), bool) + assert_type(query.run_sync(), bool) + + async def table_exists() -> None: + query = Band.table_exists() + assert_type(await query, bool) + assert_type(await query.run(), bool) + assert_type(query.run_sync(), bool) + + async def from_dict() -> None: + assert_type(Band.from_dict(data={}), Band) + + async def update() -> None: + query = Band.update() + assert_type(await query, t.List[t.Any]) + assert_type(await query.run(), t.List[t.Any]) + assert_type(query.run_sync(), t.List[t.Any]) + + async def insert() -> None: + # This is correct: + Band.insert(Band()) + # This is an error: + Band.insert(Manager()) # type: ignore From fdb703f4abf461dc323776d9f2611a1dc92a6c92 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 2 Jan 2023 20:16:12 +0000 Subject: [PATCH 436/727] v0.104.0 (#744) * remove duplicate type tests * bumped version * remove unused import --- CHANGES.rst | 17 +++++++++++++++++ piccolo/__init__.py | 2 +- tests/table/test_objects.py | 25 ------------------------- 3 files changed, 18 insertions(+), 26 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 8ac63254f..50a24defd 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,23 @@ Changes ======= +0.104.0 +------- + +Major improvements to Piccolo's typing / auto completion support. + +For example: + +.. code-block:: python + + >>> bands = await Band.objects() # List[Band] + + >>> band = await Band.objects().first() # Optional[Band] + + >>> bands = await Band.select().output(as_json=True) # str + +------------------------------------------------------------------------------- + 0.103.0 ------- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index d7f5ee996..43d367ac6 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.103.0" +__VERSION__ = "0.104.0" diff --git a/tests/table/test_objects.py b/tests/table/test_objects.py index f4d2849df..f8c699ef2 100644 --- a/tests/table/test_objects.py +++ b/tests/table/test_objects.py @@ -1,7 +1,3 @@ -import typing as t - -from typing_extensions import assert_type - from tests.base import DBTestCase, engines_only, sqlite_only from tests.example_apps.music.tables import Band, Manager @@ -263,24 +259,3 @@ def test_prefetch_new_object(self): self.assertIsInstance(band.manager, Manager) self.assertEqual(band.name, "New Band 2") self.assertEqual(band.manager.name, "Guido") - - -if t.TYPE_CHECKING: - # Making sure the types are inferred correctly by MyPy. - - assert_type(Band.objects().first().run_sync(), t.Optional[Band]) - - assert_type( - Band.objects().get(Band.name == "Pythonistas").run_sync(), - t.Optional[Band], - ) - - assert_type( - Band.objects().get_or_create(Band.name == "Pythonistas").run_sync(), - Band, - ) - - assert_type( - Band.objects().run_sync(), - t.List[Band], - ) From 300e84ac380f02690705cd6839e31b02cf8c9d65 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 7 Jan 2023 15:39:16 +0000 Subject: [PATCH 437/727] Fix performance issue with deeply nested foreign keys (#746) * fix * add test --- piccolo/columns/column_types.py | 4 +++- tests/table/test_join.py | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 854a2af78..5adabe3d3 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -2125,7 +2125,7 @@ def __getattribute__(self, name: str) -> t.Union[Column, t.Any]: raise Exception("Call chain too long!") foreign_key_meta: ForeignKeyMeta = object.__getattribute__( - self, "_foreign_key_meta" + new_column, "_foreign_key_meta" ) for proxy_column in foreign_key_meta.proxy_columns: @@ -2134,6 +2134,8 @@ def __getattribute__(self, name: str) -> t.Union[Column, t.Any]: except Exception: pass + foreign_key_meta.proxy_columns = [] + for ( column ) in value._foreign_key_meta.resolved_references._meta.columns: diff --git a/tests/table/test_join.py b/tests/table/test_join.py index 114edbc1f..72651c198 100644 --- a/tests/table/test_join.py +++ b/tests/table/test_join.py @@ -169,6 +169,24 @@ def test_select_all_columns_deep(self): }, ) + def test_proxy_columns(self): + """ + Make sure that ``proxy_columns`` are set correctly. + + There used to be a bug which meant queries got slower over time: + + https://github.com/piccolo-orm/piccolo/issues/691 + + """ + # We call it multiple times to make sure it doesn't change with time. + for _ in range(2): + self.assertEqual( + len(Concert.band_1.manager._foreign_key_meta.proxy_columns), 2 + ) + self.assertEqual( + len(Concert.band_1._foreign_key_meta.proxy_columns), 4 + ) + def test_select_all_columns_root(self): """ Make sure that using ``all_columns`` at the root doesn't interfere From 667b8c3ec026cee917dc9451ec9a568e922613e9 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 7 Jan 2023 16:01:56 +0000 Subject: [PATCH 438/727] bumped version (#747) --- CHANGES.rst | 8 ++++++++ piccolo/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 50a24defd..aa6e7ee3e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,14 @@ Changes ======= +0.105.0 +------- + +Improved the performance of select queries with complex joins. Many thanks to +@powellnorma and @sinisaos for their help with this. + +------------------------------------------------------------------------------- + 0.104.0 ------- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 43d367ac6..8367cc383 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.104.0" +__VERSION__ = "0.105.0" From 928a154164add2be0d7505f19f40d0af428f70f6 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 29 Jan 2023 22:48:16 +0000 Subject: [PATCH 439/727] make note about using double quotes when installing on Windows (#753) --- docs/src/piccolo/getting_started/installing_piccolo.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/src/piccolo/getting_started/installing_piccolo.rst b/docs/src/piccolo/getting_started/installing_piccolo.rst index ce60967c2..0cf936e0f 100644 --- a/docs/src/piccolo/getting_started/installing_piccolo.rst +++ b/docs/src/piccolo/getting_started/installing_piccolo.rst @@ -40,3 +40,7 @@ Now install Piccolo, ideally inside a `virtualenv Date: Mon, 30 Jan 2023 18:58:26 +0000 Subject: [PATCH 440/727] fix code block language - should be `bash` not `python` (#754) --- docs/src/piccolo/getting_started/installing_piccolo.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/piccolo/getting_started/installing_piccolo.rst b/docs/src/piccolo/getting_started/installing_piccolo.rst index 0cf936e0f..c5cad56d3 100644 --- a/docs/src/piccolo/getting_started/installing_piccolo.rst +++ b/docs/src/piccolo/getting_started/installing_piccolo.rst @@ -13,7 +13,7 @@ Pip Now install Piccolo, ideally inside a `virtualenv `_: -.. code-block:: python +.. code-block:: bash # Optional - creating a virtualenv on Unix: python3 -m venv my_project From 528160dbde478b5c817389f421203a4b3007a4cd Mon Sep 17 00:00:00 2001 From: Pieterjan De Potter Date: Tue, 31 Jan 2023 09:24:25 +0100 Subject: [PATCH 441/727] fix output of alias usage (only band names) (#755) --- docs/src/piccolo/query_types/select.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/src/piccolo/query_types/select.rst b/docs/src/piccolo/query_types/select.rst index f570c28ec..e55a11626 100644 --- a/docs/src/piccolo/query_types/select.rst +++ b/docs/src/piccolo/query_types/select.rst @@ -26,8 +26,7 @@ Or use an alias to make it shorter: >>> b = Band >>> await b.select(b.name) - [{'id': 1, 'name': 'Pythonistas', 'manager': 1, 'popularity': 1000}, - {'id': 2, 'name': 'Rustaceans', 'manager': 2, 'popularity': 500}] + [{'name': 'Rustaceans'}, {'name': 'Pythonistas'}] .. hint:: All of these examples also work synchronously using ``run_sync`` - From a61d915f42880dcc0a1245146d7cac2101200af0 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 31 Jan 2023 18:48:45 +0000 Subject: [PATCH 442/727] use `table_finder` in `piccolo app new` template (#756) --- .../apps/app/commands/templates/piccolo_app.py.jinja | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/piccolo/apps/app/commands/templates/piccolo_app.py.jinja b/piccolo/apps/app/commands/templates/piccolo_app.py.jinja index 9c86cecd7..4527161bc 100644 --- a/piccolo/apps/app/commands/templates/piccolo_app.py.jinja +++ b/piccolo/apps/app/commands/templates/piccolo_app.py.jinja @@ -5,7 +5,7 @@ the APP_CONFIG. import os -from piccolo.conf.apps import AppConfig +from piccolo.conf.apps import AppConfig, table_finder CURRENT_DIRECTORY = os.path.dirname(os.path.abspath(__file__)) @@ -13,8 +13,14 @@ CURRENT_DIRECTORY = os.path.dirname(os.path.abspath(__file__)) APP_CONFIG = AppConfig( app_name='{{ app_name }}', - migrations_folder_path=os.path.join(CURRENT_DIRECTORY, 'piccolo_migrations'), - table_classes=[], + migrations_folder_path=os.path.join( + CURRENT_DIRECTORY, + 'piccolo_migrations' + ), + table_classes=table_finder( + modules=["{{ app_name }}.tables"], + exclude_imported=True + ), migration_dependencies=[], commands=[] ) From 4e93ba31fc6e2096cbcec004f491faf2644bc2b7 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 21 Feb 2023 18:48:36 +0000 Subject: [PATCH 443/727] allow joins within update queries (#768) * prototype for update with joins * added `querystring_for_update` * added lots of tests --- piccolo/columns/combination.py | 44 +++++++++++++++++ piccolo/query/methods/update.py | 5 +- tests/table/test_update.py | 88 ++++++++++++++++++++++++++++++++- 3 files changed, 135 insertions(+), 2 deletions(-) diff --git a/piccolo/columns/combination.py b/piccolo/columns/combination.py index c263e5e51..e080cced2 100644 --- a/piccolo/columns/combination.py +++ b/piccolo/columns/combination.py @@ -44,6 +44,14 @@ def querystring(self) -> QueryString: self.second.querystring, ) + @property + def querystring_for_update(self) -> QueryString: + return QueryString( + "({} " + self.operator + " {})", + self.first.querystring_for_update, + self.second.querystring_for_update, + ) + def __str__(self): return self.querystring.__str__() @@ -122,6 +130,10 @@ def __init__(self, sql: str, *args: t.Any) -> None: """ self.querystring = QueryString(sql, *args) + @property + def querystring_for_update(self) -> QueryString: + return self.querystring + def __str__(self): return self.querystring.__str__() @@ -205,5 +217,37 @@ def querystring(self) -> QueryString: return QueryString(template, *args) + @property + def querystring_for_update(self) -> QueryString: + args: t.List[t.Any] = [] + if self.value != UNDEFINED: + args.append(self.value) + + if self.values != UNDEFINED: + args.append(self.values_querystring) + + column = self.column + + if column._meta.call_chain: + # Use a sub select to find the correct ID. + root_column = column._meta.call_chain[0] + sub_query = root_column._meta.table.select(root_column).where(self) + + column_name = column._meta.call_chain[0]._meta.name + return QueryString( + f"{column_name} IN ({{}})", + sub_query.querystrings[0], + ) + else: + template = self.operator.template.format( + name=self.column.get_where_string( + engine_type=self.column._meta.engine_type + ), + value="{}", + values="{}", + ) + + return QueryString(template, *args) + def __str__(self): return self.querystring.__str__() diff --git a/piccolo/query/methods/update.py b/piccolo/query/methods/update.py index b4c980edb..72d62307b 100644 --- a/piccolo/query/methods/update.py +++ b/piccolo/query/methods/update.py @@ -97,10 +97,13 @@ def default_querystrings(self) -> t.Sequence[QueryString]: ) if self.where_delegate._where: + # The JOIN syntax isn't allowed in SQL UPDATE queries, so we need + # to write the WHERE clause differently, using a sub select. + querystring = QueryString( "{} WHERE {}", querystring, - self.where_delegate._where.querystring, + self.where_delegate._where.querystring_for_update, ) if self.returning_delegate._returning: diff --git a/tests/table/test_update.py b/tests/table/test_update.py index 62c595870..366b54fce 100644 --- a/tests/table/test_update.py +++ b/tests/table/test_update.py @@ -24,7 +24,7 @@ is_running_sqlite, sqlite_only, ) -from tests.example_apps.music.tables import Band +from tests.example_apps.music.tables import Band, Manager class TestUpdate(DBTestCase): @@ -628,3 +628,89 @@ def test_update(self): ) self.assertIsInstance(updated_row["modified_on"], datetime.datetime) self.assertEqual(updated_row["name"], "test 2") + + +############################################################################### +# Test update with joins + + +class TestUpdateWithJoin(DBTestCase): + def test_join(self): + """ + Make sure updates work when the where clause needs a join. + """ + self.insert_rows() + Band.update({Band.name: "New name"}).where( + Band.manager.name == "Guido" + ).run_sync() + + self.assertEqual( + Band.count().where(Band.name == "New name").run_sync(), 1 + ) + + def test_multiple_matches(self): + """ + Make sure it works when the join has multiple matching values. + """ + self.insert_rows() + + # Create an additional band with the same manager. + manager = Manager.objects().get(Manager.name == "Guido").run_sync() + band = Band(name="Pythonistas 2", manager=manager) + band.save().run_sync() + + Band.update({Band.name: "New name"}).where( + Band.manager.name == "Guido" + ).run_sync() + + self.assertEqual( + Band.count().where(Band.name == "New name").run_sync(), 2 + ) + + def test_no_matches(self): + """ + Make sure it works when the join has no matching values. + """ + self.insert_rows() + + Band.update({Band.name: "New name"}).where( + Band.manager.name == "Mr Manager" + ).run_sync() + + self.assertEqual( + Band.count().where(Band.name == "New name").run_sync(), 0 + ) + + def test_and(self): + """ + Make sure it works when combined with other where clauses using AND. + """ + self.insert_rows() + + # Create an additional band with the same manager, and different + # popularity. + manager = Manager.objects().get(Manager.name == "Guido").run_sync() + band = Band(name="Pythonistas 2", manager=manager, popularity=10000) + band.save().run_sync() + + Band.update({Band.name: "New name"}).where( + Band.manager.name == "Guido", Band.popularity == 10000 + ).run_sync() + + self.assertEqual( + Band.count().where(Band.name == "New name").run_sync(), 1 + ) + + def test_or(self): + """ + Make sure it works when combined with other where clauses using OR. + """ + self.insert_rows() + + Band.update({Band.name: "New name"}).where( + (Band.manager.name == "Guido") | (Band.manager.name == "Graydon") + ).run_sync() + + self.assertEqual( + Band.count().where(Band.name == "New name").run_sync(), 2 + ) From 7e6012b8fe83f89ec6faf4a75be828e289815c3f Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 21 Feb 2023 19:37:20 +0000 Subject: [PATCH 444/727] bumped version (#769) --- CHANGES.rst | 20 ++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index aa6e7ee3e..a527d0479 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,26 @@ Changes ======= +0.106.0 +------- + +Joins now work within ``update`` queries. For example: + +.. code-block:: python + + await Band.update({ + Band.name: 'Amazing Band' + }).where( + Band.manager.name == 'Guido' + ) + +Other changes: + +* Improved the template used by ``piccolo app new`` when creating a new + Piccolo app (it now uses ``table_finder``). + +------------------------------------------------------------------------------- + 0.105.0 ------- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 8367cc383..6b2c80f7e 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.105.0" +__VERSION__ = "0.106.0" From a89e4766e977ef565327adc4c286e0d9b439f5e0 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Fri, 24 Feb 2023 17:10:39 +0100 Subject: [PATCH 445/727] update Starlite asgi template (#772) --- .../templates/app/_starlite_app.py.jinja | 31 +++++-------------- 1 file changed, 8 insertions(+), 23 deletions(-) diff --git a/piccolo/apps/asgi/commands/templates/app/_starlite_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_starlite_app.py.jinja index 8db51c61a..ea1343268 100644 --- a/piccolo/apps/asgi/commands/templates/app/_starlite_app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/_starlite_app.py.jinja @@ -2,26 +2,19 @@ import typing as t from piccolo.engine import engine_finder from piccolo_admin.endpoints import create_admin -from starlite import ( - MediaType, - Response, - Starlite, - StaticFilesConfig, - TemplateConfig, - asgi, - delete, - get, - patch, - post, -) -from starlite.plugins.piccolo_orm import PiccoloORMPlugin +from starlite import Starlite, asgi, delete, get, patch, post +from starlite.config.static_files import StaticFilesConfig +from starlite.config.template import TemplateConfig from starlite.contrib.jinja import JinjaTemplateEngine +from starlite.exceptions import NotFoundException +from starlite.plugins.piccolo_orm import PiccoloORMPlugin from starlite.types import Receive, Scope, Send from home.endpoints import home from home.piccolo_app import APP_CONFIG from home.tables import Task + # mounting Piccolo Admin @asgi("/admin/", is_mount=True) async def admin(scope: "Scope", receive: "Receive", send: "Send") -> None: @@ -45,11 +38,7 @@ async def create_task(data: Task) -> Task: async def update_task(task_id: int, data: Task) -> Task: task = await Task.objects().get(Task.id == task_id) if not task: - return Response( - content={}, - media_type=MediaType.JSON, - status_code=404, - ) + raise NotFoundException("Task does not exist") for key, value in data.to_dict().items(): task.id = task_id setattr(task, key, value) @@ -62,11 +51,7 @@ async def update_task(task_id: int, data: Task) -> Task: async def delete_task(task_id: int) -> None: task = await Task.objects().get(Task.id == task_id) if not task: - return Response( - content={}, - media_type=MediaType.JSON, - status_code=404, - ) + raise NotFoundException("Task does not exist") await task.remove() From 3a2aae743f1a54f5a52ace44342d6715c3a729ec Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 25 Feb 2023 17:27:05 +0000 Subject: [PATCH 446/727] Fix refresh typos (#773) * fix typo in PostgresEngine comment * fix typo in docstring of `test_refresh` * make sure `refresh` calls the classmethod --- piccolo/engine/postgres.py | 2 +- piccolo/query/methods/refresh.py | 2 +- tests/table/test_refresh.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/piccolo/engine/postgres.py b/piccolo/engine/postgres.py index 41ffb7731..cefa02092 100644 --- a/piccolo/engine/postgres.py +++ b/piccolo/engine/postgres.py @@ -322,7 +322,7 @@ async def prep_database(self): ########################################################################### # These typos existed in the codebase for a while, so leaving these proxy - # methods for now to ensure backwards compatility. + # methods for now to ensure backwards compatibility. async def start_connnection_pool(self, **kwargs) -> None: colored_warning( diff --git a/piccolo/query/methods/refresh.py b/piccolo/query/methods/refresh.py index cc7f3764f..0d0258a6a 100644 --- a/piccolo/query/methods/refresh.py +++ b/piccolo/query/methods/refresh.py @@ -72,7 +72,7 @@ async def run( raise ValueError("No columns to fetch.") updated_values = ( - await instance.select(*columns) + await instance.__class__.select(*columns) .where(pk_column == primary_key_value) .first() .run(node=node, in_pool=in_pool) diff --git a/tests/table/test_refresh.py b/tests/table/test_refresh.py index 22e01d8f7..69deb9a73 100644 --- a/tests/table/test_refresh.py +++ b/tests/table/test_refresh.py @@ -9,7 +9,7 @@ def setUp(self): def test_refresh(self): """ - Make sure ``refresh`` works, with not columns specified. + Make sure ``refresh`` works, with no columns specified. """ # Fetch an instance from the database. band: Band = Band.objects().get(Band.name == "Pythonistas").run_sync() From 3407ad379e2b5d412f04d41d2550d04a93780c5d Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 27 Feb 2023 19:01:52 +0000 Subject: [PATCH 447/727] Add `log_responses` to engine (#775) * initial prototype for `log_responses` * use `pprint` * colorise print statements * use `pprint.pprint` as `pprint.pp` not supported in Python 3.7 * reduce repetition * add missing slot * remove unused imports * add tests for query and response logging --- piccolo/engine/base.py | 23 ++++++++++++++-- piccolo/engine/cockroach.py | 2 ++ piccolo/engine/postgres.py | 37 +++++++++++++++++++------ piccolo/engine/sqlite.py | 52 ++++++++++++++++++++++++++---------- tests/engine/test_logging.py | 39 +++++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 24 deletions(-) create mode 100644 tests/engine/test_logging.py diff --git a/piccolo/engine/base.py b/piccolo/engine/base.py index f1b43ebc7..77de5ceb1 100644 --- a/piccolo/engine/base.py +++ b/piccolo/engine/base.py @@ -2,12 +2,13 @@ import contextvars import logging +import pprint import typing as t from abc import ABCMeta, abstractmethod from piccolo.querystring import QueryString from piccolo.utils.sync import run_sync -from piccolo.utils.warnings import Level, colored_warning +from piccolo.utils.warnings import Level, colored_string, colored_warning if t.TYPE_CHECKING: # pragma: no cover from piccolo.query.base import Query @@ -25,11 +26,12 @@ class Batch: class Engine(t.Generic[TransactionClass], metaclass=ABCMeta): - __slots__ = () + __slots__ = ("query_id",) def __init__(self): run_sync(self.check_version()) run_sync(self.prep_database()) + self.query_id = 0 @property @abstractmethod @@ -137,3 +139,20 @@ def transaction_exists(self) -> bool: """ return self.current_transaction.get() is not None + + ########################################################################### + # Logging queries and responses + + def get_query_id(self) -> int: + self.query_id += 1 + return self.query_id + + def print_query(self, query_id: int, query: str): + print(colored_string(f"\nQuery {query_id}:")) + print(query) + + def print_response(self, query_id: int, response: t.List): + print( + colored_string(f"\nQuery {query_id} response:", level=Level.high) + ) + pprint.pprint(response) diff --git a/piccolo/engine/cockroach.py b/piccolo/engine/cockroach.py index c12586255..6c5019531 100644 --- a/piccolo/engine/cockroach.py +++ b/piccolo/engine/cockroach.py @@ -24,12 +24,14 @@ def __init__( config: t.Dict[str, t.Any], extensions: t.Sequence[str] = (), log_queries: bool = False, + log_responses: bool = False, extra_nodes: t.Dict[str, CockroachEngine] = None, ) -> None: super().__init__( config=config, extensions=extensions, log_queries=log_queries, + log_responses=log_responses, extra_nodes=extra_nodes, ) diff --git a/piccolo/engine/postgres.py b/piccolo/engine/postgres.py index cefa02092..09c9b2477 100644 --- a/piccolo/engine/postgres.py +++ b/piccolo/engine/postgres.py @@ -209,6 +209,10 @@ class PostgresEngine(Engine[t.Optional[PostgresTransaction]]): If ``True``, all SQL and DDL statements are printed out before being run. Useful for debugging. + :param log_responses: + If ``True``, the raw response from each query is printed out. Useful + for debugging. + :param extra_nodes: If you have additional database nodes (e.g. read replicas) for the server, you can specify them here. It's a mapping of a memorable name @@ -240,6 +244,7 @@ class PostgresEngine(Engine[t.Optional[PostgresTransaction]]): "config", "extensions", "log_queries", + "log_responses", "extra_nodes", "pool", "current_transaction", @@ -253,6 +258,7 @@ def __init__( config: t.Dict[str, t.Any], extensions: t.Sequence[str] = ("uuid-ossp",), log_queries: bool = False, + log_responses: bool = False, extra_nodes: t.Mapping[str, PostgresEngine] = None, ) -> None: if extra_nodes is None: @@ -261,6 +267,7 @@ def __init__( self.config = config self.extensions = extensions self.log_queries = log_queries + self.log_responses = log_responses self.extra_nodes = extra_nodes self.pool: t.Optional[Pool] = None database_name = config.get("database", "Unknown") @@ -427,32 +434,46 @@ async def run_querystring( engine_type=self.engine_type ) + query_id = self.get_query_id() + if self.log_queries: - print(querystring) + self.print_query(query_id=query_id, query=querystring.__str__()) # If running inside a transaction: current_transaction = self.current_transaction.get() if current_transaction: - return await current_transaction.connection.fetch( + response = await current_transaction.connection.fetch( query, *query_args ) elif in_pool and self.pool: - return await self._run_in_pool(query, query_args) + response = await self._run_in_pool(query, query_args) else: - return await self._run_in_new_connection(query, query_args) + response = await self._run_in_new_connection(query, query_args) + + if self.log_responses: + self.print_response(query_id=query_id, response=response) + + return response async def run_ddl(self, ddl: str, in_pool: bool = True): + query_id = self.get_query_id() + if self.log_queries: - print(ddl) + self.print_query(query_id=query_id, query=ddl) # If running inside a transaction: current_transaction = self.current_transaction.get() if current_transaction: - return await current_transaction.connection.fetch(ddl) + response = await current_transaction.connection.fetch(ddl) elif in_pool and self.pool: - return await self._run_in_pool(ddl) + response = await self._run_in_pool(ddl) else: - return await self._run_in_new_connection(ddl) + response = await self._run_in_new_connection(ddl) + + if self.log_responses: + self.print_response(query_id=query_id, response=response) + + return response def atomic(self) -> Atomic: return Atomic(engine=self) diff --git a/piccolo/engine/sqlite.py b/piccolo/engine/sqlite.py index 441dff8c9..7fe026f3a 100644 --- a/piccolo/engine/sqlite.py +++ b/piccolo/engine/sqlite.py @@ -365,7 +365,12 @@ def dict_factory(cursor, row) -> t.Dict: class SQLiteEngine(Engine[t.Optional[SQLiteTransaction]]): - __slots__ = ("connection_kwargs", "current_transaction", "log_queries") + __slots__ = ( + "connection_kwargs", + "current_transaction", + "log_queries", + "log_responses", + ) engine_type = "sqlite" min_version_number = 3.25 @@ -374,6 +379,7 @@ def __init__( self, path: str = "piccolo.sqlite", log_queries: bool = False, + log_responses: bool = False, **connection_kwargs, ) -> None: """ @@ -383,6 +389,9 @@ def __init__( :param log_queries: If ``True``, all SQL and DDL statements are printed out before being run. Useful for debugging. + :param log_responses: + If ``True``, the raw response from each query is printed out. + Useful for debugging. :param connection_kwargs: These are passed directly to the database adapter. We recommend setting ``timeout`` if you expect your application to process a @@ -398,6 +407,7 @@ def __init__( } self.log_queries = log_queries + self.log_responses = log_responses self.connection_kwargs = { **default_connection_kwargs, **connection_kwargs, @@ -548,8 +558,10 @@ async def run_querystring( Connection pools aren't currently supported - the argument is there for consistency with other engines. """ + query_id = self.get_query_id() + if self.log_queries: - print(querystring) + self.print_query(query_id=query_id, query=querystring.__str__()) query, query_args = querystring.compile_string( engine_type=self.engine_type @@ -558,40 +570,52 @@ async def run_querystring( # If running inside a transaction: current_transaction = self.current_transaction.get() if current_transaction: - return await self._run_in_existing_connection( + response = await self._run_in_existing_connection( connection=current_transaction.connection, query=query, args=query_args, query_type=querystring.query_type, table=querystring.table, ) + else: + response = await self._run_in_new_connection( + query=query, + args=query_args, + query_type=querystring.query_type, + table=querystring.table, + ) - return await self._run_in_new_connection( - query=query, - args=query_args, - query_type=querystring.query_type, - table=querystring.table, - ) + if self.log_responses: + self.print_response(query_id=query_id, response=response) + + return response async def run_ddl(self, ddl: str, in_pool: bool = False): """ Connection pools aren't currently supported - the argument is there for consistency with other engines. """ + query_id = self.get_query_id() + if self.log_queries: - print(ddl) + self.print_query(query_id=query_id, query=ddl) # If running inside a transaction: current_transaction = self.current_transaction.get() if current_transaction: - return await self._run_in_existing_connection( + response = await self._run_in_existing_connection( connection=current_transaction.connection, query=ddl, ) + else: + response = await self._run_in_new_connection( + query=ddl, + ) - return await self._run_in_new_connection( - query=ddl, - ) + if self.log_responses: + self.print_response(query_id=query_id, response=response) + + return response def atomic( self, transaction_type: TransactionType = TransactionType.deferred diff --git a/tests/engine/test_logging.py b/tests/engine/test_logging.py new file mode 100644 index 000000000..2e6ec3d3a --- /dev/null +++ b/tests/engine/test_logging.py @@ -0,0 +1,39 @@ +from unittest.mock import patch + +from tests.base import DBTestCase +from tests.example_apps.music.tables import Manager + + +class TestLogging(DBTestCase): + def tearDown(self): + Manager._meta.db.log_queries = False + Manager._meta.db.log_responses = False + super().tearDown() + + def test_log_queries(self): + Manager._meta.db.log_queries = True + + with patch("piccolo.engine.base.Engine.print_query") as print_query: + Manager.select().run_sync() + print_query.assert_called_once() + + def test_log_responses(self): + Manager._meta.db.log_responses = True + + with patch( + "piccolo.engine.base.Engine.print_response" + ) as print_response: + Manager.select().run_sync() + print_response.assert_called_once() + + def test_log_queries_and_responses(self): + Manager._meta.db.log_queries = True + Manager._meta.db.log_responses = True + + with patch("piccolo.engine.base.Engine.print_query") as print_query: + with patch( + "piccolo.engine.base.Engine.print_response" + ) as print_response: + Manager.select().run_sync() + print_query.assert_called_once() + print_response.assert_called_once() From 021d782b706ec6ed33b88355477c31b5826d8d26 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 27 Feb 2023 20:38:48 +0000 Subject: [PATCH 448/727] bumped version (#776) --- CHANGES.rst | 22 ++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index a527d0479..bb4aa4b9a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,28 @@ Changes ======= +0.107.0 +------- + +Added the ``log_responses`` option to the database engines. This makes the +engine print out the raw response from the database for each query, which +is useful during debugging. + +.. code-block:: python + + # piccolo_conf.py + + DB = PostgresEngine( + config={'database': 'my_database'}, + log_queries=True, + log_responses=True + ) + +We also updated the Starlite ASGI template - it now uses the new import paths +(thanks to @sinisaos for this). + +------------------------------------------------------------------------------- + 0.106.0 ------- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 6b2c80f7e..564e661f9 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.106.0" +__VERSION__ = "0.107.0" From 72b85975ef4364cb5609c5c6af5240e4f4782f90 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 1 Mar 2023 14:54:19 +0000 Subject: [PATCH 449/727] Transaction savepoints (#778) * prototype for savepoints * allow the user to manually commit or rollback transactions * add docs for nested context managers * fix `test_nested_transaction` * make some new attributes private * added tests for savepoints * fix sqlite tests * one more test for using transaction in nested --- docs/src/piccolo/query_types/transactions.rst | 127 +++++++++++++++++- piccolo/engine/postgres.py | 118 +++++++++++++--- piccolo/engine/sqlite.py | 111 +++++++++++++-- tests/engine/test_nested_transaction.py | 17 ++- tests/engine/test_transaction.py | 64 +++++++++ 5 files changed, 400 insertions(+), 37 deletions(-) diff --git a/docs/src/piccolo/query_types/transactions.rst b/docs/src/piccolo/query_types/transactions.rst index 3db6c854f..803d1cfa3 100644 --- a/docs/src/piccolo/query_types/transactions.rst +++ b/docs/src/piccolo/query_types/transactions.rst @@ -8,9 +8,19 @@ Transactions allow multiple queries to be committed only once successful. This is useful for things like migrations, where you can't have it fail in an inbetween state. -.. note:: - In the examples below we use ``MyTable._meta.db`` to access the ``Engine``, - which is used to create transactions. +------------------------------------------------------------------------------- + +Accessing the ``Engine`` +------------------------ + +In the examples below we need to access the database ``Engine``. + +Each ``Table`` contains a reference to its ``Engine``, which is the easiest +way to access it. For example, with our ``Band`` table: + +.. code-block:: python + + DB = Band._meta.db ------------------------------------------------------------------------------- @@ -22,7 +32,7 @@ transaction before running it. .. code-block:: python - transaction = Band._meta.db.atomic() + transaction = DB.atomic() transaction.add(Manager.create_table()) transaction.add(Concert.create_table()) await transaction.run() @@ -40,14 +50,81 @@ async. .. code-block:: python - async with Band._meta.db.transaction(): + async with DB.transaction(): await Manager.create_table() await Concert.create_table() +Commit +~~~~~~ + +The transaction is automatically committed when you exit the context manager. + +.. code-block:: python + + async with DB.transaction(): + await query_1 + await query_2 + # Automatically committed if the code reaches here. + +You can manually commit it if you prefer: + +.. code-block:: python + + async with DB.transaction() as transaction: + await query_1 + await query_2 + await transaction.commit() + print('transaction committed!') + +Rollback +~~~~~~~~ + If an exception is raised within the body of the context manager, then the transaction is automatically rolled back. The exception is still propagated though. +Rather than raising an exception, if you want to rollback a transaction +manually you can do so as follows: + +.. code-block:: python + + async with DB.transaction() as transaction: + await Manager.create_table() + await Band.create_table() + await transaction.rollback() + +------------------------------------------------------------------------------- + +Nested transactions +------------------- + +Nested transactions aren't supported in Postgres, but we can achieve something +similar using `savepoints `_. + +Nested context managers +~~~~~~~~~~~~~~~~~~~~~~~ + +If you have nested context managers, for example: + +.. code-block:: python + + async with DB.transaction(): + async with DB.transaction(): + ... + +By default, the inner context manager does nothing, as we're already inside a +transaction. + +You can change this behaviour using ``allow_nested=False``, in which case a +``TransactionError`` is raised if you try creating a transaction when one +already exists. + +.. code-block:: python + + async with DB.transaction(): + async with DB.transaction(allow_nested=False): + # TransactionError('A transaction is already active.') + ``transaction_exists`` ~~~~~~~~~~~~~~~~~~~~~~ @@ -56,11 +133,49 @@ following: .. code-block:: python - >>> Band._meta.db.transaction_exists() + >>> DB.transaction_exists() True ------------------------------------------------------------------------------- +Savepoints +---------- + +Postgres supports savepoints, which is a way of partially rolling back a +transaction. + +.. code-block:: python + + async with DB.transaction() as transaction: + await Band.insert(Band(name='Pythonistas')) + + savepoint_1 = await transaction.savepoint() + + await Band.insert(Band(name='Terrible band')) + + # Oops, I made a mistake! + await savepoint_1.rollback_to() + +In the above example, the first query will be committed, but not the second. + +Named savepoints +~~~~~~~~~~~~~~~~ + +By default, we assign a name to the savepoint for you. But you can explicitly +give it a name: + +.. code-block:: python + + await transaction.savepoint('my_savepoint') + +This means you can rollback to this savepoint at any point just using the name: + +.. code-block:: python + + await transaction.rollback_to('my_savepoint') + +------------------------------------------------------------------------------- + Transaction types ----------------- diff --git a/piccolo/engine/postgres.py b/piccolo/engine/postgres.py index 09c9b2477..42a8d6bc2 100644 --- a/piccolo/engine/postgres.py +++ b/piccolo/engine/postgres.py @@ -123,6 +123,22 @@ def __await__(self): ############################################################################### +class Savepoint: + def __init__(self, name: str, transaction: PostgresTransaction): + self.name = name + self.transaction = transaction + + async def rollback_to(self): + await self.transaction.connection.execute( + f"ROLLBACK TO SAVEPOINT {self.name}" + ) + + async def release(self): + await self.transaction.connection.execute( + f"RELEASE SAVEPOINT {self.name}" + ) + + class PostgresTransaction: """ Used for wrapping queries in a transaction, using a context manager. @@ -136,37 +152,107 @@ class PostgresTransaction: """ - __slots__ = ("engine", "transaction", "context", "connection") + __slots__ = ( + "engine", + "transaction", + "context", + "connection", + "_savepoint_id", + "_parent", + "_committed", + "_rolled_back", + ) - def __init__(self, engine: PostgresEngine): + def __init__(self, engine: PostgresEngine, allow_nested: bool = True): + """ + :param allow_nested: + If ``True`` then if we try creating a new transaction when another + is already active, we treat this as a no-op:: + + async with DB.transaction(): + async with DB.transaction(): + pass + + If we want to disallow this behaviour, then setting + ``allow_nested=False`` will cause a ``TransactionError`` to be + raised. + + """ self.engine = engine - if self.engine.current_transaction.get(): - raise TransactionError( - "A transaction is already active - nested transactions aren't " - "currently supported." - ) + current_transaction = self.engine.current_transaction.get() - async def __aenter__(self): + self._savepoint_id = 0 + self._parent = None + self._committed = False + self._rolled_back = False + + if current_transaction: + if allow_nested: + self._parent = current_transaction + else: + raise TransactionError( + "A transaction is already active - nested transactions " + "aren't allowed." + ) + + async def __aenter__(self) -> PostgresTransaction: + if self._parent is not None: + return self._parent + + self.connection = await self.get_connection() + self.transaction = self.connection.transaction() + await self.begin() + self.context = self.engine.current_transaction.set(self) + return self + + async def get_connection(self): if self.engine.pool: - self.connection = await self.engine.pool.acquire() + return await self.engine.pool.acquire() else: - self.connection = await self.engine.get_new_connection() + return await self.engine.get_new_connection() - self.transaction = self.connection.transaction() + async def begin(self): await self.transaction.start() - self.context = self.engine.current_transaction.set(self) async def commit(self): await self.transaction.commit() + self._committed = True async def rollback(self): await self.transaction.rollback() + self._rolled_back = True + + async def rollback_to(self, savepoint_name: str): + """ + Used to rollback to a savepoint just using the name. + """ + await Savepoint(name=savepoint_name, transaction=self).rollback_to() + + ########################################################################### + + def get_savepoint_id(self) -> int: + self._savepoint_id += 1 + return self._savepoint_id + + async def savepoint(self, name: t.Optional[str] = None) -> Savepoint: + name = name or f"savepoint_{self.get_savepoint_id()}" + await self.connection.execute(f"SAVEPOINT {name}") + return Savepoint(name=name, transaction=self) + + ########################################################################### async def __aexit__(self, exception_type, exception, traceback): + if self._parent: + return exception is None + if exception: - await self.rollback() + # The user may have manually rolled it back. + if not self._rolled_back: + await self.rollback() else: - await self.commit() + # The user may have manually committed it. + if not self._committed and not self._rolled_back: + await self.commit() if self.engine.pool: await self.engine.pool.release(self.connection) @@ -478,5 +564,5 @@ async def run_ddl(self, ddl: str, in_pool: bool = True): def atomic(self) -> Atomic: return Atomic(engine=self) - def transaction(self) -> PostgresTransaction: - return PostgresTransaction(engine=self) + def transaction(self, allow_nested: bool = True) -> PostgresTransaction: + return PostgresTransaction(engine=self, allow_nested=allow_nested) diff --git a/piccolo/engine/sqlite.py b/piccolo/engine/sqlite.py index 7fe026f3a..9084baf71 100644 --- a/piccolo/engine/sqlite.py +++ b/piccolo/engine/sqlite.py @@ -303,6 +303,22 @@ def __await__(self): ############################################################################### +class Savepoint: + def __init__(self, name: str, transaction: SQLiteTransaction): + self.name = name + self.transaction = transaction + + async def rollback_to(self): + await self.transaction.connection.execute( + f"ROLLBACK TO SAVEPOINT {self.name}" + ) + + async def release(self): + await self.transaction.connection.execute( + f"RELEASE SAVEPOINT {self.name}" + ) + + class SQLiteTransaction: """ Used for wrapping queries in a transaction, using a context manager. @@ -316,12 +332,23 @@ class SQLiteTransaction: """ - __slots__ = ("engine", "context", "connection", "transaction_type") + __slots__ = ( + "engine", + "context", + "connection", + "transaction_type", + "allow_nested", + "_savepoint_id", + "_parent", + "_committed", + "_rolled_back", + ) def __init__( self, engine: SQLiteEngine, transaction_type: TransactionType = TransactionType.deferred, + allow_nested: bool = True, ): """ :param transaction_type: @@ -333,22 +360,76 @@ def __init__( """ self.engine = engine self.transaction_type = transaction_type - if self.engine.current_transaction.get(): - raise TransactionError( - "A transaction is already active - nested transactions aren't " - "currently supported." - ) + current_transaction = self.engine.current_transaction.get() - async def __aenter__(self): - self.connection = await self.engine.get_connection() - await self.connection.execute(f"BEGIN {self.transaction_type.value}") + self._savepoint_id = 0 + self._parent = None + self._committed = False + self._rolled_back = False + + if current_transaction: + if allow_nested: + self._parent = current_transaction + else: + raise TransactionError( + "A transaction is already active - nested transactions " + "aren't allowed." + ) + + async def __aenter__(self) -> SQLiteTransaction: + if self._parent is not None: + return self._parent + + self.connection = await self.get_connection() + await self.begin() self.context = self.engine.current_transaction.set(self) + return self + + async def get_connection(self): + return await self.engine.get_connection() + + async def begin(self): + await self.connection.execute(f"BEGIN {self.transaction_type.value}") + + async def commit(self): + await self.connection.execute("COMMIT") + self._committed = True + + async def rollback(self): + await self.connection.execute("ROLLBACK") + self._rolled_back = True + + async def rollback_to(self, savepoint_name: str): + """ + Used to rollback to a savepoint just using the name. + """ + await Savepoint(name=savepoint_name, transaction=self).rollback_to() + + ########################################################################### + + def get_savepoint_id(self) -> int: + self._savepoint_id += 1 + return self._savepoint_id + + async def savepoint(self, name: t.Optional[str] = None) -> Savepoint: + name = name or f"savepoint_{self.get_savepoint_id()}" + await self.connection.execute(f"SAVEPOINT {name}") + return Savepoint(name=name, transaction=self) + + ########################################################################### async def __aexit__(self, exception_type, exception, traceback): + if self._parent: + return exception is None + if exception: - await self.connection.execute("ROLLBACK") + # The user may have manually rolled it back. + if not self._rolled_back: + await self.rollback() else: - await self.connection.execute("COMMIT") + # The user may have manually committed it. + if not self._committed and not self._rolled_back: + await self.commit() await self.connection.close() self.engine.current_transaction.reset(self.context) @@ -623,11 +704,15 @@ def atomic( return Atomic(engine=self, transaction_type=transaction_type) def transaction( - self, transaction_type: TransactionType = TransactionType.deferred + self, + transaction_type: TransactionType = TransactionType.deferred, + allow_nested: bool = True, ) -> SQLiteTransaction: """ Create a new database transaction. See :class:`Transaction`. """ return SQLiteTransaction( - engine=self, transaction_type=transaction_type + engine=self, + transaction_type=transaction_type, + allow_nested=allow_nested, ) diff --git a/tests/engine/test_nested_transaction.py b/tests/engine/test_nested_transaction.py index dd1347b7d..23bee59a4 100644 --- a/tests/engine/test_nested_transaction.py +++ b/tests/engine/test_nested_transaction.py @@ -60,12 +60,25 @@ async def run_nested(self): """ Nested transactions currently aren't permitted in a connection. """ + # allow_nested=False with self.assertRaises(TransactionError): async with Manager._meta.db.transaction(): await Manager(name="Bob").save().run() - async with Manager._meta.db.transaction(): - await Manager(name="Dave").save().run() + async with Manager._meta.db.transaction(allow_nested=False): + pass + + # allow_nested=True + async with Manager._meta.db.transaction(): + async with Manager._meta.db.transaction(): + # Shouldn't raise an exception + pass + + # Utilise returned transaction. + async with Manager._meta.db.transaction(): + async with Manager._meta.db.transaction() as transaction: + await Manager(name="Dave").save().run() + await transaction.rollback() def test_nested(self): asyncio.run(self.run_nested()) diff --git a/tests/engine/test_transaction.py b/tests/engine/test_transaction.py index c56e802be..bf2489617 100644 --- a/tests/engine/test_transaction.py +++ b/tests/engine/test_transaction.py @@ -109,6 +109,34 @@ async def run_transaction(): self.assertTrue(Band.table_exists().run_sync()) self.assertTrue(Manager.table_exists().run_sync()) + def test_manual_commit(self): + """ + The context manager automatically commits changes, but we also + allow the user to do it manually. + """ + + async def run_transaction(): + async with Band._meta.db.transaction() as transaction: + await Manager.create_table() + await transaction.commit() + + asyncio.run(run_transaction()) + self.assertTrue(Manager.table_exists().run_sync()) + + def test_manual_rollback(self): + """ + The context manager will automatically rollback changes if an exception + is raised, but we also allow the user to do it manually. + """ + + async def run_transaction(): + async with Band._meta.db.transaction() as transaction: + await Manager.create_table() + await transaction.rollback() + + asyncio.run(run_transaction()) + self.assertFalse(Manager.table_exists().run_sync()) + @engines_only("postgres") def test_transaction_id(self): """ @@ -232,3 +260,39 @@ async def run_all(): .run_sync(), manager_names, ) + + +class TestSavepoint(TestCase): + def setUp(self): + Manager.create_table().run_sync() + + def tearDown(self): + Manager.alter().drop_table().run_sync() + + def test_savepoint(self): + async def run_test(): + async with Manager._meta.db.transaction() as transaction: + await Manager.insert(Manager(name="Manager 1")) + savepoint = await transaction.savepoint() + await Manager.insert(Manager(name="Manager 2")) + await savepoint.rollback_to() + + run_sync(run_test()) + + self.assertListEqual( + Manager.select(Manager.name).run_sync(), [{"name": "Manager 1"}] + ) + + def test_named_savepoint(self): + async def run_test(): + async with Manager._meta.db.transaction() as transaction: + await Manager.insert(Manager(name="Manager 1")) + await transaction.savepoint("my_savepoint") + await Manager.insert(Manager(name="Manager 2")) + await transaction.rollback_to("my_savepoint") + + run_sync(run_test()) + + self.assertListEqual( + Manager.select(Manager.name).run_sync(), [{"name": "Manager 1"}] + ) From ba633f6664df098ddc0845d337b15a11d4598a37 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 1 Mar 2023 15:19:22 +0000 Subject: [PATCH 450/727] bumped version (#779) --- CHANGES.rst | 35 +++++++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index bb4aa4b9a..4a1733a39 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,41 @@ Changes ======= +0.108.0 +------- + +Added support for savepoints within transactions. + +.. code-block:: python + + await DB.transaction() as transaction: + await Manager.objects().create(name="Great manager") + savepoint = await transaction.savepoint() + await Manager.objects().create(name="Great manager") + await savepoint.rollback_to() + # Only the first manager will be inserted. + +The behaviour of nested context managers has also been changed slightly. + +.. code-block:: python + + await DB.transaction() as transaction: + await DB.transaction() as transaction: + # This used to raise an exception + +We no longer raise an exception if there are nested transaction context +managers, instead the inner ones do nothing. + +If you want the existing behaviour: + +.. code-block:: python + + await DB.transaction() as transaction: + await DB.transactiona(allow_nested=False) as transaction: + # TransactionError! + +------------------------------------------------------------------------------- + 0.107.0 ------- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 564e661f9..000d559b7 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.107.0" +__VERSION__ = "0.108.0" From 356421134230818f5c1a70812a9913cb4e0abe9c Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 1 Mar 2023 15:24:21 +0000 Subject: [PATCH 451/727] fix typos in changelog --- CHANGES.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 4a1733a39..9121e2f2c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,7 +8,7 @@ Added support for savepoints within transactions. .. code-block:: python - await DB.transaction() as transaction: + async with DB.transaction() as transaction: await Manager.objects().create(name="Great manager") savepoint = await transaction.savepoint() await Manager.objects().create(name="Great manager") @@ -19,8 +19,8 @@ The behaviour of nested context managers has also been changed slightly. .. code-block:: python - await DB.transaction() as transaction: - await DB.transaction() as transaction: + async with DB.transaction(): + async with DB.transaction(): # This used to raise an exception We no longer raise an exception if there are nested transaction context @@ -30,8 +30,8 @@ If you want the existing behaviour: .. code-block:: python - await DB.transaction() as transaction: - await DB.transactiona(allow_nested=False) as transaction: + async with DB.transaction(): + async with DB.transactiona(allow_nested=False): # TransactionError! ------------------------------------------------------------------------------- From 7b5efdd3bc3e69b40f828547f04dae4189836954 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 5 Mar 2023 21:03:51 +0000 Subject: [PATCH 452/727] join on any column (#782) * join on any column * added docs --- docs/src/piccolo/api_reference/index.rst | 10 +++ docs/src/piccolo/query_types/index.rst | 10 +++ docs/src/piccolo/query_types/joins.rst | 37 ++++++++ docs/src/piccolo/query_types/objects.rst | 6 +- piccolo/columns/base.py | 45 ++++++++++ tests/table/test_join_on.py | 107 +++++++++++++++++++++++ 6 files changed, 214 insertions(+), 1 deletion(-) create mode 100644 docs/src/piccolo/query_types/joins.rst create mode 100644 tests/table/test_join_on.py diff --git a/docs/src/piccolo/api_reference/index.rst b/docs/src/piccolo/api_reference/index.rst index 1e1395174..439d9c0ec 100644 --- a/docs/src/piccolo/api_reference/index.rst +++ b/docs/src/piccolo/api_reference/index.rst @@ -11,6 +11,16 @@ Table ------------------------------------------------------------------------------- +Column +------ + +.. currentmodule:: piccolo.columns.base + +.. autoclass:: Column + :members: + +------------------------------------------------------------------------------- + Refresh ------- diff --git a/docs/src/piccolo/query_types/index.rst b/docs/src/piccolo/query_types/index.rst index 5c054f64e..ac013acf9 100644 --- a/docs/src/piccolo/query_types/index.rst +++ b/docs/src/piccolo/query_types/index.rst @@ -19,7 +19,17 @@ typical ORM. ./insert ./raw ./update + +------------------------------------------------------------------------------- + +Features +-------- + +.. toctree:: + :maxdepth: 1 + ./transactions + ./joins ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/query_types/joins.rst b/docs/src/piccolo/query_types/joins.rst new file mode 100644 index 000000000..d7727162e --- /dev/null +++ b/docs/src/piccolo/query_types/joins.rst @@ -0,0 +1,37 @@ +Joins +===== + +Joins are handled automatically by Piccolo. They work everywhere you'd expect +(select queries, where clauses, etc.). + +A `fluent interface `_ is used, +which lets you traverse foreign keys. + +Here's an example of a select query which uses joins (using the +:ref:`example schema `): + +.. code-block:: python + + # This gets the band's name, and the manager's name by joining to the + # manager table: + >>> await Band.select(Band.name, Band.manager.name) + +And a ``where`` clause which uses joins: + +.. code-block:: python + + # This automatically joins with the manager table to perform the where + # clause. It only returns the columns from the band table though by default. + >>> await Band.select().where(Band.manager.name == 'Guido') + +Left joins are used. + +join_on +------- + +Joins are usually performed using ``ForeignKey`` columns, though there may be +situations where you want to join using a column which isn't a ``ForeignKey``. + +You can do this using :meth:`join_on `. + +It's generally best to join on unique columns. diff --git a/docs/src/piccolo/query_types/objects.rst b/docs/src/piccolo/query_types/objects.rst index 456e6717c..0d8420222 100644 --- a/docs/src/piccolo/query_types/objects.rst +++ b/docs/src/piccolo/query_types/objects.rst @@ -78,7 +78,8 @@ We also have this shortcut which combines the above into a single line: Updating objects ---------------- -Objects have a ``save`` method, which is convenient for updating values: +Objects have a :meth:`save ` method, which is +convenient for updating values: .. code-block:: python @@ -278,6 +279,9 @@ has the latest data from the database, you can use the # And it has gotten stale, we can refresh it: await band.refresh() + # Or just refresh certain columns: + await band.refresh([Band.name]) + ------------------------------------------------------------------------------- Query clauses diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index adbfb6248..714c66de2 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -759,6 +759,51 @@ def as_alias(self, name: str) -> Column: column._alias = name return column + def join_on(self, column: Column) -> ForeignKey: + """ + Joins are typically performed via foreign key columns. For example, + here we get the band's name and the manager's name:: + + class Manager(Table): + name = Varchar() + + class Band(Table): + name = Varchar() + manager = ForeignKey(Manager) + + >>> await Band.select(Band.name, Band.manager.name) + + The ``join_on`` method lets you join tables even when foreign keys + don't exist, by joining on a column in another table. + + For example, here we want to get the manager's email, but no foreign + key exists:: + + class Manager(Table): + name = Varchar(unique=True) + email = Varchar() + + class Band(Table): + name = Varchar() + manager_name = Varchar() + + >>> await Band.select( + ... Band.name, + ... Band.manager_name.join_on(Manager.name).email + ... ) + + """ + from piccolo.columns.column_types import ForeignKey + + virtual_foreign_key = ForeignKey( + references=column._meta.table, target_column=column + ) + virtual_foreign_key._meta._name = self._meta.name + virtual_foreign_key._meta.call_chain = [*self._meta.call_chain] + virtual_foreign_key._meta._table = self._meta.table + virtual_foreign_key.set_proxy_columns() + return virtual_foreign_key + def get_default_value(self) -> t.Any: """ If the column has a default attribute, return it. If it's callable, diff --git a/tests/table/test_join_on.py b/tests/table/test_join_on.py new file mode 100644 index 000000000..7983b218a --- /dev/null +++ b/tests/table/test_join_on.py @@ -0,0 +1,107 @@ +from unittest import TestCase + +from piccolo.columns import Varchar +from piccolo.table import Table + + +class Manager(Table): + name = Varchar(unique=True) + email = Varchar(unique=True) + + +class Band(Table): + name = Varchar(unique=True) + manager_name = Varchar() + + +class Concert(Table): + title = Varchar() + band_name = Varchar() + + +class TestJoinOn(TestCase): + + tables = [Manager, Band, Concert] + + def setUp(self): + for table in self.tables: + table.create_table().run_sync() + + Manager.insert( + Manager(name="Guido", email="guido@example.com"), + Manager(name="Maz", email="maz@example.com"), + Manager(name="Graydon", email="graydon@example.com"), + ).run_sync() + + Band.insert( + Band(name="Pythonistas", manager_name="Guido"), + Band(name="Rustaceans", manager_name="Graydon"), + ).run_sync() + + Concert.insert( + Concert( + title="Rockfest", + band_name="Pythonistas", + ), + ).run_sync() + + def tearDown(self): + for table in self.tables: + table.alter().drop_table().run_sync() + + def test_join_on(self): + """ + Do a simple join between two tables. + """ + query = Band.select( + Band.name, + Band.manager_name, + Band.manager_name.join_on(Manager.name).email.as_alias( + "manager_email" + ), + ).order_by(Band.id) + + response = query.run_sync() + + self.assertListEqual( + response, + [ + { + "name": "Pythonistas", + "manager_name": "Guido", + "manager_email": "guido@example.com", + }, + { + "name": "Rustaceans", + "manager_name": "Graydon", + "manager_email": "graydon@example.com", + }, + ], + ) + + def test_deeper_join(self): + """ + Do a join between three tables. + """ + response = ( + Concert.select( + Concert.title, + Concert.band_name, + Concert.band_name.join_on(Band.name) + .manager_name.join_on(Manager.name) + .email.as_alias("manager_email"), + ) + .order_by(Concert.id) + .run_sync() + ) + + self.assertListEqual( + response, + [ + { + "title": "Rockfest", + "band_name": "Pythonistas", + "manager_email": "guido@example.com", + } + ], + ) From ff26a8805f5d4ddd906388bdfdb51f66bce93bc8 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 5 Mar 2023 21:50:25 +0000 Subject: [PATCH 453/727] v0.109.0 (#783) * fix rst warning * bumped version --- CHANGES.rst | 24 ++++++++++++++++++++++++ docs/src/piccolo/schema/column_types.rst | 7 +++++-- piccolo/__init__.py | 2 +- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 9121e2f2c..2c1fea5b4 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,30 @@ Changes ======= +0.109.0 +------- + +Joins are now possible without foreign keys using ``join_on``. + +For example: + +.. code-block:: python + + class Manager(Table): + name = Varchar(unique=True) + email = Varchar() + + class Band(Table): + name = Varchar() + manager_name = Varchar() + + >>> await Band.select( + ... Band.name, + ... Band.manager_name.join_on(Manager.name).email + ... ) + +------------------------------------------------------------------------------- + 0.108.0 ------- diff --git a/docs/src/piccolo/schema/column_types.rst b/docs/src/piccolo/schema/column_types.rst index 7f39f7c60..4b7fd1315 100644 --- a/docs/src/piccolo/schema/column_types.rst +++ b/docs/src/piccolo/schema/column_types.rst @@ -1,7 +1,5 @@ .. _ColumnTypes: -.. currentmodule:: piccolo.columns.column_types - ############ Column Types ############ @@ -13,7 +11,10 @@ Column Types Column ****** +.. currentmodule:: piccolo.columns.base + .. autoclass:: Column + :noindex: ------------------------------------------------------------------------------- @@ -21,6 +22,8 @@ Column Bytea ***** +.. currentmodule:: piccolo.columns.column_types + .. autoclass:: Bytea .. hint:: There is also a ``Blob`` column type, which is an alias for diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 000d559b7..ae23efe55 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.108.0" +__VERSION__ = "0.109.0" From e66714c590f350e2cda01b3a35d327cf40d66356 Mon Sep 17 00:00:00 2001 From: Yannis Burkhalter Date: Sun, 12 Mar 2023 11:49:40 +0100 Subject: [PATCH 454/727] Typo: "OnDelete.cascade" -> "OnUpdate.cascade" (#787) --- piccolo/columns/column_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 5adabe3d3..5dbb15e6e 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -1784,7 +1784,7 @@ class Band(Table): :param on_update: Determines what the database should do when a row has it's primary key - updated. If set to ``OnDelete.cascade``, any rows referencing the + updated. If set to ``OnUpdate.cascade``, any rows referencing the updated row will have their references updated to point to the new primary key. From 56669fab0cc6626ccc7be707b094a022b9a0ba36 Mon Sep 17 00:00:00 2001 From: Yannis Burkhalter Date: Sun, 12 Mar 2023 20:14:48 +0100 Subject: [PATCH 455/727] Typo: "import OnDelete" -> "import OnUpdate" (#788) --- piccolo/columns/column_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 5dbb15e6e..855e8cae0 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -1800,7 +1800,7 @@ class Band(Table): .. code-block:: python - from piccolo.columns import OnDelete + from piccolo.columns import OnUpdate class Band(Table): name = ForeignKey( From aa225bd9401db8ab4effb2ad970477743ebdaef6 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 11 Apr 2023 15:05:30 +0100 Subject: [PATCH 456/727] clarify Piccolo's support for sqlite (#803) --- .../piccolo/getting_started/database_support.rst | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/src/piccolo/getting_started/database_support.rst b/docs/src/piccolo/getting_started/database_support.rst index 3dc192888..8596b5fd9 100644 --- a/docs/src/piccolo/getting_started/database_support.rst +++ b/docs/src/piccolo/getting_started/database_support.rst @@ -4,10 +4,15 @@ Database Support ================ `Postgres `_ is the primary database which Piccolo -was designed for. +was designed for. It's robust, feature rich, and a great choice for most projects. -`CockroachDB `_ is in experimental beta. +`CockroachDB `_ is also supported. It's designed +to be scalable and fault tolerant, and is mostly compatible with Postgres. +There may be some minor features not supported, but it's OK to use. -Limited `SQLite `_ support is available, -mostly to enable tooling like the :ref:`playground `. Postgres is the only database we -recommend for use in production with Piccolo. +`SQLite `_ support was originally added to +enable tooling like the :ref:`playground `, but over time we've +added more and more support. Many people successfully use SQLite and Piccolo +together in production. The main missing feature is support for +:ref:`automatic database migrations ` due to SQLite's limited +support for ``ALTER TABLE`` ``DDL`` statements. From 19d5d52ba047a334b95bc60da2f3d58cfa20e30f Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 11 Apr 2023 15:57:58 +0100 Subject: [PATCH 457/727] ignore docs when deciding whether to run tests (#805) --- .github/workflows/tests.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 8e32e1ac5..4b7c7519a 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -3,8 +3,12 @@ name: Test Suite on: push: branches: ["master"] + paths-ignore: + - "docs/**" pull_request: branches: ["master"] + paths-ignore: + - "docs/**" jobs: linters: From 157d42b0c09ed60f22a15f6432634f380ed5086f Mon Sep 17 00:00:00 2001 From: sinisaos Date: Tue, 11 Apr 2023 16:59:01 +0200 Subject: [PATCH 458/727] update ASGI frameworks (#802) * update ASGI frameworks * update Litestar version --- README.md | 2 +- piccolo/apps/asgi/commands/new.py | 5 +- ...te_app.py.jinja => _litestar_app.py.jinja} | 16 +-- .../templates/app/_xpresso_app.py.jinja | 113 ------------------ .../asgi/commands/templates/app/app.py.jinja | 6 +- ....py.jinja => _litestar_endpoints.py.jinja} | 3 +- .../app/home/_xpresso_endpoints.py.jinja | 20 ---- .../templates/app/home/endpoints.py.jinja | 6 +- .../app/home/templates/home.html.jinja_raw | 7 +- 9 files changed, 18 insertions(+), 160 deletions(-) rename piccolo/apps/asgi/commands/templates/app/{_starlite_app.py.jinja => _litestar_app.py.jinja} (84%) delete mode 100644 piccolo/apps/asgi/commands/templates/app/_xpresso_app.py.jinja rename piccolo/apps/asgi/commands/templates/app/home/{_starlite_endpoints.py.jinja => _litestar_endpoints.py.jinja} (89%) delete mode 100644 piccolo/apps/asgi/commands/templates/app/home/_xpresso_endpoints.py.jinja diff --git a/README.md b/README.md index 75168c26a..82bfe5702 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ Let Piccolo scaffold you an ASGI web app, using Piccolo as the ORM: piccolo asgi new ``` -[Starlette](https://www.starlette.io/), [FastAPI](https://fastapi.tiangolo.com/), [BlackSheep](https://www.neoteroi.dev/blacksheep/), [Xpresso](https://xpresso-api.dev/) and [Starlite](https://starlite-api.github.io/starlite/) are currently supported. +[Starlette](https://www.starlette.io/), [FastAPI](https://fastapi.tiangolo.com/), [BlackSheep](https://www.neoteroi.dev/blacksheep/) and [Litestar](https://litestar.dev/) are currently supported. ## Are you a Django user? diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index 4e7d7efd0..aedabdf93 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -10,10 +10,9 @@ TEMPLATE_DIR = os.path.join(os.path.dirname(__file__), "templates/app/") SERVERS = ["uvicorn", "Hypercorn"] -ROUTERS = ["starlette", "fastapi", "blacksheep", "xpresso", "starlite"] +ROUTERS = ["starlette", "fastapi", "blacksheep", "litestar"] ROUTER_DEPENDENCIES = { - "starlite": ["starlite>=1.46.0"], - "xpresso": ["xpresso==0.43.0", "di==0.72.1"], + "litestar": ["litestar>=2.0.0a3"], } diff --git a/piccolo/apps/asgi/commands/templates/app/_starlite_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_litestar_app.py.jinja similarity index 84% rename from piccolo/apps/asgi/commands/templates/app/_starlite_app.py.jinja rename to piccolo/apps/asgi/commands/templates/app/_litestar_app.py.jinja index ea1343268..62e12c19e 100644 --- a/piccolo/apps/asgi/commands/templates/app/_starlite_app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/_litestar_app.py.jinja @@ -2,13 +2,13 @@ import typing as t from piccolo.engine import engine_finder from piccolo_admin.endpoints import create_admin -from starlite import Starlite, asgi, delete, get, patch, post -from starlite.config.static_files import StaticFilesConfig -from starlite.config.template import TemplateConfig -from starlite.contrib.jinja import JinjaTemplateEngine -from starlite.exceptions import NotFoundException -from starlite.plugins.piccolo_orm import PiccoloORMPlugin -from starlite.types import Receive, Scope, Send +from litestar import Litestar, asgi, delete, get, patch, post +from litestar.static_files import StaticFilesConfig +from litestar.template import TemplateConfig +from litestar.contrib.jinja import JinjaTemplateEngine +from litestar.contrib.piccolo_orm import PiccoloORMPlugin +from litestar.exceptions import NotFoundException +from litestar.types import Receive, Scope, Send from home.endpoints import home from home.piccolo_app import APP_CONFIG @@ -71,7 +71,7 @@ async def close_database_connection_pool(): print("Unable to connect to the database") -app = Starlite( +app = Litestar( route_handlers=[ admin, home, diff --git a/piccolo/apps/asgi/commands/templates/app/_xpresso_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_xpresso_app.py.jinja deleted file mode 100644 index 09010d199..000000000 --- a/piccolo/apps/asgi/commands/templates/app/_xpresso_app.py.jinja +++ /dev/null @@ -1,113 +0,0 @@ -import typing as t -from contextlib import asynccontextmanager - -from piccolo.engine import engine_finder -from piccolo.utils.pydantic import create_pydantic_model -from piccolo_admin.endpoints import create_admin -from starlette.staticfiles import StaticFiles -from xpresso import App, FromJson, FromPath, HTTPException, Operation, Path -from xpresso.routing.mount import Mount - -from home.endpoints import home -from home.piccolo_app import APP_CONFIG -from home.tables import Task - -TaskModelIn: t.Any = create_pydantic_model(table=Task, model_name="TaskModelIn") -TaskModelOut: t.Any = create_pydantic_model( - table=Task, include_default_columns=True, model_name="TaskModelOut" -) - - -async def tasks() -> t.List[TaskModelOut]: - return await Task.select().order_by(Task.id) - - -async def create_task(task_model: FromJson[TaskModelIn]) -> TaskModelOut: - task = Task(**task_model.dict()) - await task.save() - return task.to_dict() - - -async def update_task( - task_id: FromPath[int], task_model: FromJson[TaskModelIn] -) -> TaskModelOut: - task = await Task.objects().get(Task.id == task_id) - if not task: - raise HTTPException(status_code=404) - - for key, value in task_model.dict().items(): - setattr(task, key, value) - - await task.save() - - return task.to_dict() - - -async def delete_task(task_id: FromPath[int]): - task = await Task.objects().get(Task.id == task_id) - if not task: - raise HTTPException(status_code=404) - - await task.remove() - - return {} - - -@asynccontextmanager -async def lifespan(): - await open_database_connection_pool() - try: - yield - finally: - await close_database_connection_pool() - - -app = App( - routes=[ - Path( - "/", - get=Operation( - home, - include_in_schema=False, - ), - ), - Mount( - "/admin/", - create_admin( - tables=APP_CONFIG.table_classes, - # Required when running under HTTPS: - # allowed_hosts=['my_site.com'] - ), - ), - Path( - "/tasks/", - get=tasks, - post=create_task, - tags=["Task"], - ), - Path( - "/tasks/{task_id}/", - put=update_task, - delete=delete_task, - tags=["Task"], - ), - Mount("/static/", StaticFiles(directory="static")), - ], - lifespan=lifespan, -) - - -async def open_database_connection_pool(): - try: - engine = engine_finder() - await engine.start_connection_pool() - except Exception: - print("Unable to connect to the database") - - -async def close_database_connection_pool(): - try: - engine = engine_finder() - await engine.close_connection_pool() - except Exception: - print("Unable to connect to the database") diff --git a/piccolo/apps/asgi/commands/templates/app/app.py.jinja b/piccolo/apps/asgi/commands/templates/app/app.py.jinja index 47d673aad..2a8de80da 100644 --- a/piccolo/apps/asgi/commands/templates/app/app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/app.py.jinja @@ -4,8 +4,6 @@ {% include '_starlette_app.py.jinja' %} {% elif router == 'blacksheep' %} {% include '_blacksheep_app.py.jinja' %} -{% elif router == 'xpresso' %} - {% include '_xpresso_app.py.jinja' %} -{% elif router == 'starlite' %} - {% include '_starlite_app.py.jinja' %} +{% elif router == 'litestar' %} + {% include '_litestar_app.py.jinja' %} {% endif %} diff --git a/piccolo/apps/asgi/commands/templates/app/home/_starlite_endpoints.py.jinja b/piccolo/apps/asgi/commands/templates/app/home/_litestar_endpoints.py.jinja similarity index 89% rename from piccolo/apps/asgi/commands/templates/app/home/_starlite_endpoints.py.jinja rename to piccolo/apps/asgi/commands/templates/app/home/_litestar_endpoints.py.jinja index 9ced8951c..b33586da5 100644 --- a/piccolo/apps/asgi/commands/templates/app/home/_starlite_endpoints.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/home/_litestar_endpoints.py.jinja @@ -1,7 +1,7 @@ import os import jinja2 -from starlite import MediaType, Request, Response, get +from litestar import MediaType, Request, Response, get ENVIRONMENT = jinja2.Environment( loader=jinja2.FileSystemLoader( @@ -19,3 +19,4 @@ def home(request: Request) -> Response: media_type=MediaType.HTML, status_code=200, ) + diff --git a/piccolo/apps/asgi/commands/templates/app/home/_xpresso_endpoints.py.jinja b/piccolo/apps/asgi/commands/templates/app/home/_xpresso_endpoints.py.jinja deleted file mode 100644 index 2068880ac..000000000 --- a/piccolo/apps/asgi/commands/templates/app/home/_xpresso_endpoints.py.jinja +++ /dev/null @@ -1,20 +0,0 @@ -import os - -import jinja2 -from xpresso.responses import HTMLResponse - -ENVIRONMENT = jinja2.Environment( - loader=jinja2.FileSystemLoader( - searchpath=os.path.join(os.path.dirname(__file__), "templates") - ) -) - - -async def home(): - template = ENVIRONMENT.get_template("home.html.jinja") - - content = template.render( - title="Piccolo + ASGI", - ) - - return HTMLResponse(content) diff --git a/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja b/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja index c864994a9..fc5011b24 100644 --- a/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja @@ -2,8 +2,6 @@ {% include '_starlette_endpoints.py.jinja' %} {% elif router == 'blacksheep' %} {% include '_blacksheep_endpoints.py.jinja' %} -{% elif router == 'xpresso' %} - {% include '_xpresso_endpoints.py.jinja' %} -{% elif router == 'starlite' %} - {% include '_starlite_endpoints.py.jinja' %} +{% elif router == 'litestar' %} + {% include '_litestar_endpoints.py.jinja' %} {% endif %} diff --git a/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw b/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw index cbc65de87..ab175a6b0 100644 --- a/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw +++ b/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw @@ -51,12 +51,7 @@
  • Admin
  • Swagger API
    • -

    Xpresso

    - -

    Starlite

    +

    Litestar

    • Admin
    • Swagger API
      • From bd62f7a9ed7ce4f2f2dca919d7788755aeafd31c Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 11 Apr 2023 21:59:21 +0100 Subject: [PATCH 459/727] update ASGI framework names --- docs/src/index.rst | 2 +- docs/src/piccolo/asgi/index.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/src/index.rst b/docs/src/index.rst index 928664c75..de6045240 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -54,7 +54,7 @@ Give me an ASGI web app! piccolo asgi new -FastAPI, Starlette, BlackSheep, and Xpresso are currently supported, with more +FastAPI, Starlette, BlackSheep, and Litestar are currently supported, with more coming soon. ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/asgi/index.rst b/docs/src/piccolo/asgi/index.rst index 7f88a64d7..67bf87027 100644 --- a/docs/src/piccolo/asgi/index.rst +++ b/docs/src/piccolo/asgi/index.rst @@ -21,8 +21,8 @@ Routing frameworks ****************** Currently, `Starlette `_, `FastAPI `_, -`BlackSheep `_, `Xpresso `_ and -`Starlite `_ are supported. +`BlackSheep `_, and +`Litestar `_ are supported. Other great ASGI routing frameworks exist, and may be supported in the future (`Quart `_ , From 0df93d2ac70111ab092d8608916ff21a59c00100 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 12 Apr 2023 23:32:59 +0100 Subject: [PATCH 460/727] Streamline docs (#806) * add `piccolo_api` to intersphinx * move deployment docs under tutorials I don't think it needs to be top level * remove supported databases file - was duplicated information * add history section to `what is piccolo` page Gives more context as to why Piccolo exists * add fastapi tutorial --- docs/src/conf.py | 5 +- docs/src/index.rst | 1 - docs/src/piccolo/features/index.rst | 1 - .../piccolo/features/supported_databases.rst | 20 ------- .../getting_started/database_support.rst | 7 +++ .../getting_started/setup_postgres.rst | 8 --- .../getting_started/what_is_piccolo.rst | 25 +++++++++ .../index.rst => tutorials/deployment.rst} | 4 +- docs/src/piccolo/tutorials/fastapi.rst | 52 ++++++++++++++++++ docs/src/piccolo/tutorials/fastapi_src/app.py | 54 +++++++++++++++++++ docs/src/piccolo/tutorials/index.rst | 2 + 11 files changed, 146 insertions(+), 33 deletions(-) delete mode 100644 docs/src/piccolo/features/supported_databases.rst rename docs/src/piccolo/{deployment/index.rst => tutorials/deployment.rst} (98%) create mode 100644 docs/src/piccolo/tutorials/fastapi.rst create mode 100644 docs/src/piccolo/tutorials/fastapi_src/app.py diff --git a/docs/src/conf.py b/docs/src/conf.py index 4339368b9..8cb4ebee8 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -48,7 +48,10 @@ # -- Intersphinx ------------------------------------------------------------- -intersphinx_mapping = {"python": ("https://docs.python.org/3", None)} +intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), + "piccolo_api": ("https://piccolo-api.readthedocs.io/en/latest/", None), +} extensions += ["sphinx.ext.intersphinx"] # -- Autodoc ----------------------------------------------------------------- diff --git a/docs/src/index.rst b/docs/src/index.rst index de6045240..73d0cf5e6 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -21,7 +21,6 @@ batteries included. piccolo/testing/index piccolo/features/index piccolo/playground/index - piccolo/deployment/index piccolo/ecosystem/index piccolo/tutorials/index piccolo/contributing/index diff --git a/docs/src/piccolo/features/index.rst b/docs/src/piccolo/features/index.rst index c6ff9ebcc..0be80ebef 100644 --- a/docs/src/piccolo/features/index.rst +++ b/docs/src/piccolo/features/index.rst @@ -5,6 +5,5 @@ Features :maxdepth: 1 ./types_and_tab_completion - ./supported_databases ./security ./syntax diff --git a/docs/src/piccolo/features/supported_databases.rst b/docs/src/piccolo/features/supported_databases.rst deleted file mode 100644 index 4ad199ec8..000000000 --- a/docs/src/piccolo/features/supported_databases.rst +++ /dev/null @@ -1,20 +0,0 @@ -Supported Databases -=================== - -Postgres --------- -Postgres is the primary focus for Piccolo, and is what we expect most people -will be using in production. - -------------------------------------------------------------------------------- - -Cockroach DB ------------- -Cockroach support is in experimental beta. - -------------------------------------------------------------------------------- - -SQLite ------- -SQLite support is not as complete as Postgres, but it is available - mostly -because it's easy to setup. diff --git a/docs/src/piccolo/getting_started/database_support.rst b/docs/src/piccolo/getting_started/database_support.rst index 8596b5fd9..1106cd350 100644 --- a/docs/src/piccolo/getting_started/database_support.rst +++ b/docs/src/piccolo/getting_started/database_support.rst @@ -16,3 +16,10 @@ added more and more support. Many people successfully use SQLite and Piccolo together in production. The main missing feature is support for :ref:`automatic database migrations ` due to SQLite's limited support for ``ALTER TABLE`` ``DDL`` statements. + +What about other databases? +--------------------------- + +Our focus is on providing great support for a limited number of databases +(especially Postgres), however it's likely that we'll support more databases in +the future. diff --git a/docs/src/piccolo/getting_started/setup_postgres.rst b/docs/src/piccolo/getting_started/setup_postgres.rst index 9092ea40f..b95ba521b 100644 --- a/docs/src/piccolo/getting_started/setup_postgres.rst +++ b/docs/src/piccolo/getting_started/setup_postgres.rst @@ -80,11 +80,3 @@ Postgres version **************** Piccolo is tested on most major Postgres versions (see the `GitHub Actions file `_). - -------------------------------------------------------------------------------- - -What about other databases? -*************************** - -At the moment the focus is on providing the best Postgres experience possible, -along with some SQLite support. Other databases may be supported in the future. diff --git a/docs/src/piccolo/getting_started/what_is_piccolo.rst b/docs/src/piccolo/getting_started/what_is_piccolo.rst index e187c9946..5e2cf8db5 100644 --- a/docs/src/piccolo/getting_started/what_is_piccolo.rst +++ b/docs/src/piccolo/getting_started/what_is_piccolo.rst @@ -12,3 +12,28 @@ Some of it's stand out features are: * Batteries included - a :ref:`User model and authentication `, :ref:`migrations `, an :ref:`admin `, and more. * Templates for creating your own :ref:`ASGI web app `. + +History +------- + +Piccolo was created while working at a design agency, where almost all projects +being undertaken were API driven (often with high traffic), and required +web sockets. The author was naturally interested in the possibilities of :mod:`asyncio`. +Piccolo is built from the ground up with asyncio in mind. Likewise, Piccolo +makes extensive use of :mod:`type annotations `, another innovation in +Python around the time Piccolo was started. + +A really important thing when working at a design agency is having a **great +admin interface**. A huge amount of effort has gone into +`Piccolo Admin `_ +to make something you'd be proud to give to a client. + +A lot of batteries are included because Piccolo is a pragmatic framework +focused on delivering quality, functional apps to customers. This is why we have +templating tools like ``piccolo asgi new`` for getting a web app started +quickly, automatic database migrations for making iteration fast, and lots of +authentication middleware and endpoints for rapidly +`building APIs `_ out of the box. + +Piccolo has been used extensively by the author on professional projects, for +a range of corporate and startup clients. diff --git a/docs/src/piccolo/deployment/index.rst b/docs/src/piccolo/tutorials/deployment.rst similarity index 98% rename from docs/src/piccolo/deployment/index.rst rename to docs/src/piccolo/tutorials/deployment.rst index 26475385b..2a1b25b68 100644 --- a/docs/src/piccolo/deployment/index.rst +++ b/docs/src/piccolo/tutorials/deployment.rst @@ -1,5 +1,5 @@ -Deployment -========== +Deploying using Docker +====================== Docker ------ diff --git a/docs/src/piccolo/tutorials/fastapi.rst b/docs/src/piccolo/tutorials/fastapi.rst new file mode 100644 index 000000000..b46ee2fad --- /dev/null +++ b/docs/src/piccolo/tutorials/fastapi.rst @@ -0,0 +1,52 @@ +FastAPI +======= + +`FastAPI `_ is a popular ASGI web framework. The +purpose of this tutorial is to give some hints on how to get started with +Piccolo and FastAPI. + +Piccolo and FastAPI are a great match, and are commonly used together. + +Creating a new project +---------------------- + +Using the ``piccolo asgi new`` command, Piccolo will scaffold a new FastAPI for +you - simple! + +Pydantic models +--------------- + +FastAPI uses `Pydantic `_ for serialising and +deserialising data. + +Piccolo provides :func:`create_pydantic_model ` +which created Pydantic models for you based on your Piccolo tables. + +Of course, you can also just define your Pydantic models by hand. + +Transactions +------------ + +Using FastAPI's dependency injection system, we can easily wrap each endpoint +in a transaction. + +.. literalinclude:: fastapi_src/app.py + :emphasize-lines: 19-21,36 + +FastAPI dependencies can be declared at the endpoint, ``APIRouter``, or even +app level. + +``FastAPIWrapper`` +------------------ + +Piccolo API has a powerful utility called +:class:`FastAPIWrapper ` which +generates REST endpoints based on your Piccolo tables, and adds them to FastAPI's +Swagger docs. It's a very productive way of building an API. + +Authentication +-------------- + +`Piccolo API `_ ships with +`authentication middleware `_ +which is compatible with `FastAPI middleware `_. diff --git a/docs/src/piccolo/tutorials/fastapi_src/app.py b/docs/src/piccolo/tutorials/fastapi_src/app.py new file mode 100644 index 000000000..56525287f --- /dev/null +++ b/docs/src/piccolo/tutorials/fastapi_src/app.py @@ -0,0 +1,54 @@ +from fastapi import Depends, FastAPI +from pydantic import BaseModel + +from piccolo.columns.column_types import Varchar +from piccolo.engine.sqlite import SQLiteEngine +from piccolo.table import Table + +DB = SQLiteEngine() + + +class Band(Table, db=DB): + """ + You would usually import this from tables.py + """ + + name = Varchar() + + +async def transaction(): + async with DB.transaction() as transaction: + yield transaction + + +app = FastAPI() + + +@app.get("/bands/", dependencies=[Depends(transaction)]) +async def get_bands(): + return await Band.select() + + +class CreateBandModel(BaseModel): + name: str + + +@app.post("/bands/", dependencies=[Depends(transaction)]) +async def create_band(model: CreateBandModel): + await Band({Band.name: model.name}).save() + + # If an exception is raised then the transaction is rolled back. + raise Exception("Oops") + + +async def main(): + await Band.create_table(if_not_exists=True) + + +if __name__ == "__main__": + import asyncio + + import uvicorn + + asyncio.run(main()) + uvicorn.run(app) diff --git a/docs/src/piccolo/tutorials/index.rst b/docs/src/piccolo/tutorials/index.rst index c4228b2f1..5c9d66989 100644 --- a/docs/src/piccolo/tutorials/index.rst +++ b/docs/src/piccolo/tutorials/index.rst @@ -9,3 +9,5 @@ help you solve common problems: ./migrate_existing_project ./using_sqlite_and_asyncio_effectively + ./deployment + ./fastapi From 05048ab9d58611795924565965ea6170e265a323 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 12 Apr 2023 23:45:15 +0100 Subject: [PATCH 461/727] fix typos in fastapi docs --- docs/src/piccolo/tutorials/fastapi.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/src/piccolo/tutorials/fastapi.rst b/docs/src/piccolo/tutorials/fastapi.rst index b46ee2fad..0e0c91a61 100644 --- a/docs/src/piccolo/tutorials/fastapi.rst +++ b/docs/src/piccolo/tutorials/fastapi.rst @@ -10,8 +10,8 @@ Piccolo and FastAPI are a great match, and are commonly used together. Creating a new project ---------------------- -Using the ``piccolo asgi new`` command, Piccolo will scaffold a new FastAPI for -you - simple! +Using the ``piccolo asgi new`` command, Piccolo will scaffold a new FastAPI app +for you - simple! Pydantic models --------------- @@ -20,7 +20,7 @@ FastAPI uses `Pydantic `_ for serialising and deserialising data. Piccolo provides :func:`create_pydantic_model ` -which created Pydantic models for you based on your Piccolo tables. +which creates Pydantic models for you based on your Piccolo tables. Of course, you can also just define your Pydantic models by hand. From f3ae2d5284e4a313e1659f2da398646f84add54e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 20 Apr 2023 18:12:50 +0100 Subject: [PATCH 462/727] use generic types for `ModelBuilder` (#811) --- piccolo/testing/model_builder.py | 16 +++++++++------- tests/type_checking.py | 6 ++++++ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/piccolo/testing/model_builder.py b/piccolo/testing/model_builder.py index a4d6d9410..3058422b8 100644 --- a/piccolo/testing/model_builder.py +++ b/piccolo/testing/model_builder.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json import typing as t from datetime import date, datetime, time, timedelta @@ -5,7 +7,7 @@ from uuid import UUID from piccolo.columns import Array, Column -from piccolo.table import Table +from piccolo.custom_types import TableInstance from piccolo.testing.random_builder import RandomBuilder from piccolo.utils.sync import run_sync @@ -27,11 +29,11 @@ class ModelBuilder: @classmethod async def build( cls, - table_class: t.Type[Table], + table_class: t.Type[TableInstance], defaults: t.Dict[t.Union[Column, str], t.Any] = None, persist: bool = True, minimal: bool = False, - ) -> Table: + ) -> TableInstance: """ Build a ``Table`` instance with random data and save async. If the ``Table`` has any foreign keys, then the related rows are also @@ -78,11 +80,11 @@ async def build( @classmethod def build_sync( cls, - table_class: t.Type[Table], + table_class: t.Type[TableInstance], defaults: t.Dict[t.Union[Column, str], t.Any] = None, persist: bool = True, minimal: bool = False, - ) -> Table: + ) -> TableInstance: """ A sync wrapper around :meth:`build`. """ @@ -98,11 +100,11 @@ def build_sync( @classmethod async def _build( cls, - table_class: t.Type[Table], + table_class: t.Type[TableInstance], defaults: t.Dict[t.Union[Column, str], t.Any] = None, minimal: bool = False, persist: bool = True, - ) -> Table: + ) -> TableInstance: model = table_class(_ignore_missing=True) defaults = {} if not defaults else defaults diff --git a/tests/type_checking.py b/tests/type_checking.py index 5c7dffbc9..11f2cef6f 100644 --- a/tests/type_checking.py +++ b/tests/type_checking.py @@ -9,6 +9,8 @@ from typing_extensions import assert_type +from piccolo.testing.model_builder import ModelBuilder + from .example_apps.music.tables import Band, Manager if t.TYPE_CHECKING: @@ -89,3 +91,7 @@ async def insert() -> None: Band.insert(Band()) # This is an error: Band.insert(Manager()) # type: ignore + + async def model_builder() -> None: + assert_type(await ModelBuilder.build(Band), Band) + assert_type(ModelBuilder.build_sync(Band), Band) From 9157ac156d4a62d0a67b1cbcfe2a5909a6e3e4fc Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 26 Apr 2023 10:29:38 +0100 Subject: [PATCH 463/727] initial implementation for `DISTINCT ON` (#813) * initial implementation for `DISTINCT ON` * fix broken tests * use `default_factory` * fix typo `coackroach` -> `cockroach` * add a test for `on` allowed values * make `drop_tables` more efficient for Postgres and Cockroach * use `Album` table for tests * insert more discount code data in playground * add more data for `RecordingStudio` to playground * added validation * add docs * add docstring to ``DistinctOnError`` --- docs/src/piccolo/query_clauses/distinct.rst | 60 +++++++ .../piccolo/query_clauses/distinct/albums.csv | 5 + piccolo/apps/playground/commands/run.py | 32 +++- piccolo/query/methods/select.py | 35 ++-- piccolo/query/mixins.py | 80 ++++++++- tests/conftest.py | 21 ++- tests/table/test_select.py | 161 +++++++++++++++++- 7 files changed, 368 insertions(+), 26 deletions(-) create mode 100644 docs/src/piccolo/query_clauses/distinct/albums.csv diff --git a/docs/src/piccolo/query_clauses/distinct.rst b/docs/src/piccolo/query_clauses/distinct.rst index e8d101125..b04705d54 100644 --- a/docs/src/piccolo/query_clauses/distinct.rst +++ b/docs/src/piccolo/query_clauses/distinct.rst @@ -13,3 +13,63 @@ You can use ``distinct`` clauses with the following queries: [{'title': 'Pythonistas'}] This is equivalent to ``SELECT DISTINCT name FROM band`` in SQL. + +on +-- + +Using the ``on`` parameter we can create ``DISTINCT ON`` queries. + +.. note:: Postgres and CockroachDB only. For more info, see the `Postgres docs `_. + +If we have the following table: + +.. code-block:: python + + class Album(Table): + band = Varchar() + title = Varchar() + release_date = Date() + +With this data in the database: + +.. csv-table:: Albums + :file: ./distinct/albums.csv + :header-rows: 1 + +To get the latest album for each band, we can do so with a query like this: + +.. code-block:: python + + >>> await Album.select().distinct( + ... on=[Album.band] + ... ).order_by( + ... Album.band + ... ).order_by( + ... Album.release_date, + ... ascending=False + ... ) + + [ + { + 'id': 2, + 'band': 'Pythonistas', + 'title': 'Py album 2022', + 'release_date': '2022-12-01' + }, + { + 'id': 4, + 'band': 'Rustaceans', + 'title': 'Rusty album 2022', + 'release_date': '2022-12-01' + }, + ] + +The first column specified in ``on`` must match the first column specified in +``order_by``, otherwise a :class:`DistinctOnError ` will be raised. + +Source +~~~~~~ + +.. currentmodule:: piccolo.query.mixins + +.. autoclass:: DistinctOnError diff --git a/docs/src/piccolo/query_clauses/distinct/albums.csv b/docs/src/piccolo/query_clauses/distinct/albums.csv new file mode 100644 index 000000000..c259f1154 --- /dev/null +++ b/docs/src/piccolo/query_clauses/distinct/albums.csv @@ -0,0 +1,5 @@ +id,band,title,release_date +1,Pythonistas,Py album 2021,2021-12-01 +2,Pythonistas,Py album 2022,2022-12-01 +3,Rustaceans,Rusty album 2021,2021-12-01 +4,Rustaceans,Rusty album 2022,2022-12-01 diff --git a/piccolo/apps/playground/commands/run.py b/piccolo/apps/playground/commands/run.py index 092b79c20..32f840d51 100644 --- a/piccolo/apps/playground/commands/run.py +++ b/piccolo/apps/playground/commands/run.py @@ -136,8 +136,7 @@ def populate(): """ for _table in reversed(TABLES): try: - if _table.table_exists().run_sync(): - _table.alter().drop_table().run_sync() + _table.alter().drop_table(if_exists=True).run_sync() except Exception as e: print(e) @@ -180,13 +179,30 @@ def populate(): ticket = Ticket(concert=concert.id, price=Decimal("50.0")) ticket.save().run_sync() - discount_code = DiscountCode(code=uuid.uuid4()) - discount_code.save().run_sync() + DiscountCode.insert( + *[DiscountCode({DiscountCode.code: uuid.uuid4()}) for _ in range(5)] + ).run_sync() - recording_studio = RecordingStudio( - name="Abbey Road", facilities={"restaurant": True, "mixing_desk": True} - ) - recording_studio.save().run_sync() + RecordingStudio.insert( + RecordingStudio( + { + RecordingStudio.name: "Abbey Road", + RecordingStudio.facilities: { + "restaurant": True, + "mixing_desk": True, + }, + } + ), + RecordingStudio( + { + RecordingStudio.name: "Electric Lady", + RecordingStudio.facilities: { + "restaurant": False, + "mixing_desk": True, + }, + }, + ), + ).run_sync() def run( diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 62a5a1759..f94e173dc 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -353,8 +353,18 @@ def columns(self: Self, *columns: t.Union[Selectable, str]) -> Self: self.columns_delegate.columns(*_columns) return self - def distinct(self: Self) -> Self: - self.distinct_delegate.distinct() + def distinct( + self: Self, *, on: t.Optional[t.Sequence[Column]] = None + ) -> Self: + if on is not None and self.engine_type not in ( + "postgres", + "cockroach", + ): + raise ValueError( + "Only Postgres and Cockroach supports DISTINCT ON" + ) + + self.distinct_delegate.distinct(enabled=True, on=on) return self def group_by(self: Self, *columns: t.Union[Column, str]) -> Self: @@ -722,17 +732,22 @@ def default_querystrings(self) -> t.Sequence[QueryString]: ####################################################################### - select = ( - "SELECT DISTINCT" if self.distinct_delegate._distinct else "SELECT" - ) - query = f"{select} {columns_str} FROM {self.table._meta.tablename}" + args: t.List[t.Any] = [] - for join in joins: - query += f" {join}" + query = "SELECT" - ####################################################################### + distinct = self.distinct_delegate._distinct + if distinct: + if distinct.on: + distinct.validate_on(self.order_by_delegate._order_by) - args: t.List[t.Any] = [] + query += "{}" + args.append(distinct.querystring) + + query += f" {columns_str} FROM {self.table._meta.tablename}" + + for join in joins: + query += f" {join}" if self.as_of_delegate._as_of: query += "{}" diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index 397f84f99..109b48629 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import collections.abc import itertools import typing as t from dataclasses import dataclass, field @@ -18,6 +19,70 @@ from piccolo.table import Table # noqa +class DistinctOnError(ValueError): + """ + Raised when ``DISTINCT ON`` queries are malformed. + """ + + pass + + +@dataclass +class Distinct: + __slots__ = ("enabled", "on") + + enabled: bool + on: t.Optional[t.Sequence[Column]] + + @property + def querystring(self) -> QueryString: + if self.enabled: + if self.on: + column_names = ", ".join( + i._meta.get_full_name(with_alias=False) for i in self.on + ) + return QueryString(f" DISTINCT ON ({column_names})") + else: + return QueryString(" DISTINCT") + else: + return QueryString(" ALL") + + def validate_on(self, order_by: OrderBy): + """ + When using the `on` argument, the first column must match the first + order by column. + + :raises DistinctOnError: + If the columns don't match. + + """ + validated = True + + try: + first_order_column = order_by.order_by_items[0].columns[0] + except IndexError: + validated = False + else: + if not self.on: + validated = False + elif isinstance(first_order_column, Column) and not self.on[ + 0 + ]._equals(first_order_column): + validated = False + + if not validated: + raise DistinctOnError( + "The first `order_by` column must match the first column " + "passed to `on`." + ) + + def __str__(self) -> str: + return self.querystring.__str__() + + def copy(self) -> Distinct: + return self.__class__(enabled=self.enabled, on=self.on) + + @dataclass class Limit: __slots__ = ("number",) @@ -259,10 +324,19 @@ def as_of(self, interval: str = "-1s"): @dataclass class DistinctDelegate: - _distinct: bool = False + _distinct: Distinct = field( + default_factory=lambda: Distinct(enabled=False, on=None) + ) + + def distinct( + self, enabled: bool, on: t.Optional[t.Sequence[Column]] = None + ): + if on and not isinstance(on, collections.abc.Sequence): + # Check a sequence is passed in, otherwise the user will get some + # unuseful errors later on. + raise ValueError("`on` must be a sequence of `Column` instances") - def distinct(self): - self._distinct = True + self._distinct = Distinct(enabled=enabled, on=on) @dataclass diff --git a/tests/conftest.py b/tests/conftest.py index 72c20d275..dd71cba21 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,7 +6,7 @@ async def drop_tables(): - for table in [ + tables = [ "ticket", "concert", "venue", @@ -21,13 +21,26 @@ async def drop_tables(): "instrument", "mega_table", "small_table", - ]: - await ENGINE._run_in_new_connection(f"DROP TABLE IF EXISTS {table}") + ] + assert ENGINE + + if ENGINE.engine_type == "sqlite": + # SQLite doesn't allow us to drop more than one table at a time. + for table in tables: + await ENGINE._run_in_new_connection( + f"DROP TABLE IF EXISTS {table}" + ) + else: + table_str = ", ".join(tables) + await ENGINE._run_in_new_connection( + f"DROP TABLE IF EXISTS {table_str} CASCADE" + ) def pytest_sessionstart(session): """ - Make sure all the tables have been dropped. + Make sure all the tables have been dropped, just in case a previous test + run was aborted part of the way through. https://docs.pytest.org/en/latest/reference.html#_pytest.hookspec.pytest_configure """ diff --git a/tests/table/test_select.py b/tests/table/test_select.py index b8aad0c23..2ce429c2d 100644 --- a/tests/table/test_select.py +++ b/tests/table/test_select.py @@ -1,12 +1,15 @@ +import datetime from unittest import TestCase import pytest from piccolo.apps.user.tables import BaseUser +from piccolo.columns import Date, Varchar from piccolo.columns.combination import WhereRaw from piccolo.query import OrderByRaw from piccolo.query.methods.select import Avg, Count, Max, Min, SelectRaw, Sum -from piccolo.table import create_db_tables_sync, drop_db_tables_sync +from piccolo.query.mixins import DistinctOnError +from piccolo.table import Table, create_db_tables_sync, drop_db_tables_sync from tests.base import ( DBTestCase, engine_is, @@ -521,6 +524,27 @@ def test_distinct(self): response = query.run_sync() self.assertEqual(response, [{"name": "Pythonistas"}]) + def test_distinct_on(self): + """ + Make sure the distinct clause works, with the ``on`` param. + """ + self.insert_rows() + self.insert_rows() + + query = Band.select(Band.name).where(Band.name == "Pythonistas") + self.assertNotIn("DISTINCT", query.__str__()) + + response = query.run_sync() + self.assertEqual( + response, [{"name": "Pythonistas"}, {"name": "Pythonistas"}] + ) + + query = query.distinct() + self.assertIn("DISTINCT", query.__str__()) + + response = query.run_sync() + self.assertEqual(response, [{"name": "Pythonistas"}]) + def test_count_group_by(self): """ Test grouping and counting all rows. @@ -1237,3 +1261,138 @@ def test_order_by_raw(self): {"name": "Rustaceans"}, ], ) + + +class Album(Table): + band = Varchar() + title = Varchar() + release_date = Date() + + +class TestDistinctOn(TestCase): + def setUp(self): + Album.create_table().run_sync() + + def tearDown(self): + Album.alter().drop_table().run_sync() + + @engines_only("postgres", "cockroach") + def test_distinct_on(self): + """ + Make sure the ``distinct`` method can be used to create a + ``DISTINCT ON`` clause. + """ + Album.insert( + Album( + { + Album.band: "Pythonistas", + Album.title: "P1", + Album.release_date: datetime.date( + year=2022, month=1, day=1 + ), + } + ), + Album( + { + Album.band: "Pythonistas", + Album.title: "P2", + Album.release_date: datetime.date( + year=2023, month=1, day=1 + ), + } + ), + Album( + { + Album.band: "Rustaceans", + Album.title: "R1", + Album.release_date: datetime.date( + year=2022, month=1, day=1 + ), + } + ), + Album( + { + Album.band: "Rustaceans", + Album.title: "R2", + Album.release_date: datetime.date( + year=2023, month=1, day=1 + ), + } + ), + Album( + { + Album.band: "C-Sharps", + Album.title: "C1", + Album.release_date: datetime.date( + year=2022, month=1, day=1 + ), + } + ), + Album( + { + Album.band: "C-Sharps", + Album.title: "C2", + Album.release_date: datetime.date( + year=2023, month=1, day=1 + ), + } + ), + ).run_sync() + + # Get the most recent album for each band. + query = ( + Album.select(Album.band, Album.title) + .distinct(on=[Album.band]) + .order_by(Album.band) + .order_by(Album.release_date, ascending=False) + ) + self.assertIn("DISTINCT ON", query.__str__()) + response = query.run_sync() + + self.assertEqual( + response, + [ + {"band": "C-Sharps", "title": "C2"}, + {"band": "Pythonistas", "title": "P2"}, + {"band": "Rustaceans", "title": "R2"}, + ], + ) + + @engines_only("sqlite") + def test_distinct_on_sqlite(self): + """ + SQLite doesn't support ``DISTINCT ON``, so a ``ValueError`` should be + raised. + """ + with self.assertRaises(ValueError) as manager: + Album.select().distinct(on=[Album.band]) + + self.assertEqual( + manager.exception.__str__(), + "Only Postgres and Cockroach supports DISTINCT ON", + ) + + @engines_only("postgres", "cockroach") + def test_distinct_on_error(self): + """ + If we pass in something other than a sequence of columns, it should + raise a ValueError. + """ + with self.assertRaises(ValueError) as manager: + Album.select().distinct(on=Album.band) + + self.assertEqual( + manager.exception.__str__(), + "`on` must be a sequence of `Column` instances", + ) + + @engines_only("postgres", "cockroach") + def test_distinct_on_order_by_error(self): + """ + The first column passed to `order_by` must match the first column + passed to `on`, otherwise an exception is raised. + """ + with self.assertRaises(DistinctOnError): + Album.select().distinct(on=[Album.band]).order_by( + Album.release_date + ).run_sync() From 98417309a798e8c985081999e253a7582f3fa65c Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 26 Apr 2023 10:43:21 +0100 Subject: [PATCH 464/727] bumped version --- CHANGES.rst | 40 ++++++++++++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2c1fea5b4..b7d869036 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,46 @@ Changes ======= +0.110.0 +------- + +ASGI frameworks +~~~~~~~~~~~~~~~ + +The ASGI frameworks in ``piccolo asgi new`` have been updated. ``starlite`` has +been renamed to ``litestar``. Thanks to @sinisaos for this. + +ModelBuilder +~~~~~~~~~~~~ + +Generic types are now used in ``ModelBuilder``. + +.. code-block:: python + + # mypy knows this is a `Band` instance: + band = await ModelBuilder.build(Band) + +``DISTINCT ON`` +~~~~~~~~~~~~~~~ + +Added support for ``DISTINCT ON`` queries. For example, here we fetch the most +recent album for each band: + +.. code-block:: python + + >>> await Album.select().distinct( + ... on=[Album.band] + ... ).order_by( + ... Album.band + ... ).order_by( + ... Album.release_date, + ... ascending=False + ... ) + +Thanks to @sinisaos and @williamflaherty for their help with this. + +------------------------------------------------------------------------------- + 0.109.0 ------- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index ae23efe55..3fdc931bf 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.109.0" +__VERSION__ = "0.110.0" From 63af9c4e10bcc436039a132d90c8f1b9de4a50dd Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 3 May 2023 16:51:00 +0100 Subject: [PATCH 465/727] On conflict clause / upserts (#816) * initial commit * fix tests * version pin litestar * use typing extensions for Literal * make suggested change * undo * Update piccolo/query/mixins.py Co-authored-by: sinisaos * Update piccolo/query/mixins.py Co-authored-by: sinisaos * add one extra comma * first attempt at docs * add `NotImplementedError` for unsupported methods * fix typo in sqlite version number * fix tests * get tests running for sqlite * add test for `do nothing` * add test for violating non target constraint * remove old comment * allow multiple on conflict clauses * `target` -> `targets` It accepts a list, so targets makes more sense. * add docstring for `test_do_nothing` * add tests for multiple ON CONFLICT clauses * add docs for multiple ``on_conflict`` clauses * add docs for using `all_columns` * fix typo in test name * add test for using `all_columns` * add test for using an enum to specify the action * add a test to make sure `DO UPDATE` with no values raises an exception * rename `targets` back to `target` * integrate @sinisaos tests * move `on_conflict` to its own page * refactor the `where` clause --------- Co-authored-by: sinisaos --- docs/src/piccolo/query_clauses/as_of.rst | 4 +- docs/src/piccolo/query_clauses/index.rst | 1 + .../src/piccolo/query_clauses/on_conflict.rst | 229 ++++++++++ .../query_clauses/on_conflict/bands.csv | 2 + docs/src/piccolo/query_types/insert.rst | 46 +- piccolo/apps/asgi/commands/new.py | 2 +- piccolo/query/methods/insert.py | 66 ++- piccolo/query/methods/objects.py | 2 + piccolo/query/methods/select.py | 12 +- piccolo/query/mixins.py | 172 +++++++- tests/table/test_insert.py | 398 +++++++++++++++++- tests/table/test_select.py | 4 +- 12 files changed, 900 insertions(+), 38 deletions(-) create mode 100644 docs/src/piccolo/query_clauses/on_conflict.rst create mode 100644 docs/src/piccolo/query_clauses/on_conflict/bands.csv diff --git a/docs/src/piccolo/query_clauses/as_of.rst b/docs/src/piccolo/query_clauses/as_of.rst index 2d8407e42..97527033b 100644 --- a/docs/src/piccolo/query_clauses/as_of.rst +++ b/docs/src/piccolo/query_clauses/as_of.rst @@ -3,6 +3,8 @@ as_of ===== +.. note:: Cockroach only. + You can use ``as_of`` clause with the following queries: * :ref:`Select` @@ -21,5 +23,3 @@ This generates an ``AS OF SYSTEM TIME`` clause. See `documentation `_. This is very useful for performance, as it will reduce transaction contention across a cluster. - -Currently only supported on Cockroach Engine. diff --git a/docs/src/piccolo/query_clauses/index.rst b/docs/src/piccolo/query_clauses/index.rst index abb3fa18b..f9167ff63 100644 --- a/docs/src/piccolo/query_clauses/index.rst +++ b/docs/src/piccolo/query_clauses/index.rst @@ -26,6 +26,7 @@ by modifying the return values. ./freeze ./group_by ./offset + ./on_conflict ./output ./returning diff --git a/docs/src/piccolo/query_clauses/on_conflict.rst b/docs/src/piccolo/query_clauses/on_conflict.rst new file mode 100644 index 000000000..9ae8444e3 --- /dev/null +++ b/docs/src/piccolo/query_clauses/on_conflict.rst @@ -0,0 +1,229 @@ +.. _on_conflict: + +on_conflict +=========== + +.. hint:: This is an advanced topic, and first time learners of Piccolo + can skip if they want. + +You can use the ``on_conflict`` clause with the following queries: + +* :ref:`Insert` + +Introduction +------------ + +When inserting rows into a table, if a unique constraint fails on one or more +of the rows, then the insertion fails. + +Using the ``on_conflict`` clause, we can instead tell the database to ignore +the error (using ``DO NOTHING``), or to update the row (using ``DO UPDATE``). + +This is sometimes called an **upsert** (update if it already exists else insert). + +Example data +------------ + +If we have the following table: + +.. code-block:: python + + class Band(Table): + name = Varchar(unique=True) + popularity = Integer() + +With this data: + +.. csv-table:: + :file: ./on_conflict/bands.csv + +Let's try inserting another row with the same ``name``, and we'll get an error: + +.. code-block:: python + + >>> await Band.insert( + ... Band(name="Pythonistas", popularity=1200) + ... ) + Unique constraint error! + +``DO NOTHING`` +-------------- + +To ignore the error: + +.. code-block:: python + + >>> await Band.insert( + ... Band(name="Pythonistas", popularity=1200) + ... ).on_conflict( + ... action="DO NOTHING" + ... ) + +If we fetch the data from the database, we'll see that it hasn't changed: + +.. code-block:: python + + >>> await Band.select().where(Band.name == "Pythonistas").first() + {'id': 1, 'name': 'Pythonistas', 'popularity': 1000} + + +``DO UPDATE`` +------------- + +Instead, if we want to update the ``popularity``: + +.. code-block:: python + + >>> await Band.insert( + ... Band(name="Pythonistas", popularity=1200) + ... ).on_conflict( + ... action="DO UPDATE", + ... values=[Band.popularity] + ... ) + +If we fetch the data from the database, we'll see that it was updated: + +.. code-block:: python + + >>> await Band.select().where(Band.name == "Pythonistas").first() + {'id': 1, 'name': 'Pythonistas', 'popularity': 1200} + +``target`` +---------- + +Using the ``target`` argument, we can specify which constraint we're concerned +with. By specifying ``target=Band.name`` we're only concerned with the unique +constraint for the ``band`` column. If you omit the ``target`` argument, then +it works for all constraints on the table. + +.. code-block:: python + :emphasize-lines: 5 + + >>> await Band.insert( + ... Band(name="Pythonistas", popularity=1200) + ... ).on_conflict( + ... action="DO NOTHING", + ... target=Band.name + ... ) + +If you want to target a composite unique constraint, you can do so by passing +in a tuple of columns: + +.. code-block:: python + :emphasize-lines: 5 + + >>> await Band.insert( + ... Band(name="Pythonistas", popularity=1200) + ... ).on_conflict( + ... action="DO NOTHING", + ... target=(Band.name, Band.popularity) + ... ) + +You can also specify the name of a constraint using a string: + +.. code-block:: python + :emphasize-lines: 5 + + >>> await Band.insert( + ... Band(name="Pythonistas", popularity=1200) + ... ).on_conflict( + ... action="DO NOTHING", + ... target='some_constraint' + ... ) + +``values`` +---------- + +This lets us specify which values to update when a conflict occurs. + +By specifying a :class:`Column `, this means that +the new value for that column will be used: + +.. code-block:: python + :emphasize-lines: 6 + + # The new popularity will be 1200. + >>> await Band.insert( + ... Band(name="Pythonistas", popularity=1200) + ... ).on_conflict( + ... action="DO UPDATE", + ... values=[Band.popularity] + ... ) + +Instead, we can specify a custom value using a tuple: + +.. code-block:: python + :emphasize-lines: 6 + + # The new popularity will be 1111. + >>> await Band.insert( + ... Band(name="Pythonistas", popularity=1200) + ... ).on_conflict( + ... action="DO UPDATE", + ... values=[(Band.popularity, 1111)] + ... ) + +If we want to update all of the values, we can use :meth:`all_columns`. + +.. code-block:: python + :emphasize-lines: 5 + + >>> await Band.insert( + ... Band(id=1, name="Pythonistas", popularity=1200) + ... ).on_conflict( + ... action="DO UPDATE", + ... values=Band.all_columns() + ... ) + +``where`` +--------- + +This can be used with ``DO UPDATE``. It gives us more control over whether the +update should be made: + +.. code-block:: python + :emphasize-lines: 6 + + >>> await Band.insert( + ... Band(id=1, name="Pythonistas", popularity=1200) + ... ).on_conflict( + ... action="DO UPDATE", + ... values=[Band.popularity], + ... where=Band.popularity < 1000 + ... ) + +Multiple ``on_conflict`` clauses +-------------------------------- + +SQLite allows you to specify multiple ``ON CONFLICT`` clauses, but Postgres and +Cockroach don't. + +.. code-block:: python + + >>> await Band.insert( + ... Band(name="Pythonistas", popularity=1200) + ... ).on_conflict( + ... action="DO UPDATE", + ... ... + ... ).on_conflict( + ... action="DO NOTHING", + ... ... + ... ) + +Learn more +---------- + +* `Postgres docs `_ +* `Cockroach docs `_ +* `SQLite docs `_ + +Source +------ + +.. currentmodule:: piccolo.query.methods.insert + +.. automethod:: Insert.on_conflict + +.. autoclass:: OnConflictAction + :members: + :undoc-members: diff --git a/docs/src/piccolo/query_clauses/on_conflict/bands.csv b/docs/src/piccolo/query_clauses/on_conflict/bands.csv new file mode 100644 index 000000000..d796928a1 --- /dev/null +++ b/docs/src/piccolo/query_clauses/on_conflict/bands.csv @@ -0,0 +1,2 @@ +id,name,popularity +1,Pythonistas,1000 diff --git a/docs/src/piccolo/query_types/insert.rst b/docs/src/piccolo/query_types/insert.rst index eda460de1..f8d1de007 100644 --- a/docs/src/piccolo/query_types/insert.rst +++ b/docs/src/piccolo/query_types/insert.rst @@ -3,33 +3,47 @@ Insert ====== -This is used to insert rows into the table. - -.. code-block:: python - - >>> await Band.insert(Band(name="Pythonistas")) - [{'id': 3}] - -We can insert multiple rows in one go: +This is used to bulk insert rows into the table: .. code-block:: python await Band.insert( + Band(name="Pythonistas") Band(name="Darts"), Band(name="Gophers") ) ------------------------------------------------------------------------------- -add ---- +``add`` +------- -You can also compose it as follows: +If we later decide to insert additional rows, we can use the ``add`` method: .. code-block:: python - await Band.insert().add( - Band(name="Darts") - ).add( - Band(name="Gophers") - ) + query = Band.insert(Band(name="Pythonistas")) + + if other_bands: + query = query.add( + Band(name="Darts"), + Band(name="Gophers") + ) + + await query + +------------------------------------------------------------------------------- + +Query clauses +------------- + +on_conflict +~~~~~~~~~~~ + +See :ref:`On_Conflict`. + + +returning +~~~~~~~~~ + +See :ref:`Returning`. diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index aedabdf93..f4b0e16e3 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -12,7 +12,7 @@ SERVERS = ["uvicorn", "Hypercorn"] ROUTERS = ["starlette", "fastapi", "blacksheep", "litestar"] ROUTER_DEPENDENCIES = { - "litestar": ["litestar>=2.0.0a3"], + "litestar": ["litestar==2.0.0a3"], } diff --git a/piccolo/query/methods/insert.py b/piccolo/query/methods/insert.py index 9f31f445a..283bdde0b 100644 --- a/piccolo/query/methods/insert.py +++ b/piccolo/query/methods/insert.py @@ -2,9 +2,16 @@ import typing as t -from piccolo.custom_types import TableInstance +from typing_extensions import Literal + +from piccolo.custom_types import Combinable, TableInstance from piccolo.query.base import Query -from piccolo.query.mixins import AddDelegate, ReturningDelegate +from piccolo.query.mixins import ( + AddDelegate, + OnConflictAction, + OnConflictDelegate, + ReturningDelegate, +) from piccolo.querystring import QueryString if t.TYPE_CHECKING: # pragma: no cover @@ -15,7 +22,7 @@ class Insert( t.Generic[TableInstance], Query[TableInstance, t.List[t.Dict[str, t.Any]]] ): - __slots__ = ("add_delegate", "returning_delegate") + __slots__ = ("add_delegate", "on_conflict_delegate", "returning_delegate") def __init__( self, table: t.Type[TableInstance], *instances: TableInstance, **kwargs @@ -23,6 +30,7 @@ def __init__( super().__init__(table, **kwargs) self.add_delegate = AddDelegate() self.returning_delegate = ReturningDelegate() + self.on_conflict_delegate = OnConflictDelegate() self.add(*instances) ########################################################################### @@ -36,6 +44,43 @@ def returning(self: Self, *columns: Column) -> Self: self.returning_delegate.returning(columns) return self + def on_conflict( + self: Self, + target: t.Optional[t.Union[str, Column, t.Tuple[Column, ...]]] = None, + action: t.Union[ + OnConflictAction, Literal["DO NOTHING", "DO UPDATE"] + ] = OnConflictAction.do_nothing, + values: t.Optional[ + t.Sequence[t.Union[Column, t.Tuple[Column, t.Any]]] + ] = None, + where: t.Optional[Combinable] = None, + ) -> Self: + if ( + self.engine_type == "sqlite" + and self.table._meta.db.get_version_sync() < 3.24 + ): + raise NotImplementedError( + "SQLite versions lower than 3.24 don't support ON CONFLICT" + ) + + if ( + self.engine_type in ("postgres", "cockroach") + and len(self.on_conflict_delegate._on_conflict.on_conflict_items) + == 1 + ): + raise NotImplementedError( + "Postgres and Cockroach only support a single ON CONFLICT " + "clause." + ) + + self.on_conflict_delegate.on_conflict( + target=target, + action=action, + values=values, + where=where, + ) + return self + ########################################################################### def _raw_response_callback(self, results): @@ -70,16 +115,27 @@ def default_querystrings(self) -> t.Sequence[QueryString]: engine_type = self.engine_type + on_conflict = self.on_conflict_delegate._on_conflict + if on_conflict.on_conflict_items: + querystring = QueryString( + "{}{}", + querystring, + on_conflict.querystring, + query_type="insert", + table=self.table, + ) + if engine_type in ("postgres", "cockroach") or ( engine_type == "sqlite" and self.table._meta.db.get_version_sync() >= 3.35 ): - if self.returning_delegate._returning: + returning = self.returning_delegate._returning + if returning: return [ QueryString( "{}{}", querystring, - self.returning_delegate._returning.querystring, + returning.querystring, query_type="insert", table=self.table, ) diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index db9e43ccc..6892fc95c 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -230,6 +230,8 @@ def callback( return self def as_of(self, interval: str = "-1s") -> Objects: + if self.engine_type != "cockroach": + raise NotImplementedError("Only CockroachDB supports AS OF") self.as_of_delegate.as_of(interval) return self diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index f94e173dc..4e1949f55 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -356,13 +356,8 @@ def columns(self: Self, *columns: t.Union[Selectable, str]) -> Self: def distinct( self: Self, *, on: t.Optional[t.Sequence[Column]] = None ) -> Self: - if on is not None and self.engine_type not in ( - "postgres", - "cockroach", - ): - raise ValueError( - "Only Postgres and Cockroach supports DISTINCT ON" - ) + if on is not None and self.engine_type == "sqlite": + raise NotImplementedError("SQLite doesn't support DISTINCT ON") self.distinct_delegate.distinct(enabled=True, on=on) return self @@ -377,6 +372,9 @@ def group_by(self: Self, *columns: t.Union[Column, str]) -> Self: return self def as_of(self: Self, interval: str = "-1s") -> Self: + if self.engine_type != "cockroach": + raise NotImplementedError("Only CockroachDB supports AS OF") + self.as_of_delegate.as_of(interval) return self diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index 109b48629..43d126436 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -7,6 +7,8 @@ from dataclasses import dataclass, field from enum import Enum, auto +from typing_extensions import Literal + from piccolo.columns import And, Column, Or, Where from piccolo.columns.column_types import ForeignKey from piccolo.custom_types import Combinable @@ -581,9 +583,10 @@ class OffsetDelegate: Typically used in conjunction with order_by and limit. - Example usage: + Example usage:: + + .offset(100) - .offset(100) """ _offset: t.Optional[Offset] = None @@ -613,12 +616,173 @@ def __str__(self): @dataclass class GroupByDelegate: """ - Used to group results - needed when doing aggregation. + Used to group results - needed when doing aggregation:: + + .group_by(Band.name) - .group_by(Band.name) """ _group_by: t.Optional[GroupBy] = None def group_by(self, *columns: Column): self._group_by = GroupBy(columns=columns) + + +class OnConflictAction(str, Enum): + """ + Specify which action to take on conflict. + """ + + do_nothing = "DO NOTHING" + do_update = "DO UPDATE" + + +@dataclass +class OnConflictItem: + target: t.Optional[t.Union[str, Column, t.Tuple[Column, ...]]] = None + action: t.Optional[OnConflictAction] = None + values: t.Optional[ + t.Sequence[t.Union[Column, t.Tuple[Column, t.Any]]] + ] = None + where: t.Optional[Combinable] = None + + @property + def target_string(self) -> str: + target = self.target + assert target + + def to_string(value) -> str: + if isinstance(value, Column): + return f'"{value._meta.db_column_name}"' + else: + raise ValueError("OnConflict.target isn't a valid type") + + if isinstance(target, str): + return f'ON CONSTRAINT "{target}"' + elif isinstance(target, Column): + return f"({to_string(target)})" + elif isinstance(target, tuple): + columns_str = ", ".join([to_string(i) for i in target]) + return f"({columns_str})" + else: + raise ValueError("OnConflict.target isn't a valid type") + + @property + def action_string(self) -> QueryString: + action = self.action + if isinstance(action, OnConflictAction): + if action == OnConflictAction.do_nothing: + return QueryString(OnConflictAction.do_nothing.value) + elif action == OnConflictAction.do_update: + values = [] + query = f"{OnConflictAction.do_update.value} SET" + + if not self.values: + raise ValueError("No values specified for `on conflict`") + + for value in self.values: + if isinstance(value, Column): + column_name = value._meta.db_column_name + query += f' "{column_name}"=EXCLUDED."{column_name}",' + elif isinstance(value, tuple): + column = value[0] + value_ = value[1] + if isinstance(column, Column): + column_name = column._meta.db_column_name + else: + raise ValueError("Unsupported column type") + + query += f' "{column_name}"={{}},' + values.append(value_) + + return QueryString(query.rstrip(","), *values) + + raise ValueError("OnConflict.action isn't a valid type") + + @property + def querystring(self) -> QueryString: + query = " ON CONFLICT" + values = [] + + if self.target: + query += f" {self.target_string}" + + if self.action: + query += " {}" + values.append(self.action_string) + + if self.where: + query += " WHERE {}" + values.append(self.where.querystring) + + return QueryString(query, *values) + + def __str__(self) -> str: + return self.querystring.__str__() + + +@dataclass +class OnConflict: + """ + Multiple `ON CONFLICT` statements are allowed - which is why we have this + parent class. + """ + + on_conflict_items: t.List[OnConflictItem] = field(default_factory=list) + + @property + def querystring(self) -> QueryString: + query = "".join("{}" for i in self.on_conflict_items) + return QueryString( + query, *[i.querystring for i in self.on_conflict_items] + ) + + def __str__(self) -> str: + return self.querystring.__str__() + + +@dataclass +class OnConflictDelegate: + """ + Used with insert queries to specify what to do when a query fails due to + a constraint:: + + .on_conflict(action='DO NOTHING') + + .on_conflict(action='DO UPDATE', values=[Band.popularity]) + + .on_conflict(action='DO UPDATE', values=[(Band.popularity, 1)]) + + """ + + _on_conflict: OnConflict = field(default_factory=OnConflict) + + def on_conflict( + self, + target: t.Optional[t.Union[str, Column, t.Tuple[Column, ...]]] = None, + action: t.Union[ + OnConflictAction, Literal["DO NOTHING", "DO UPDATE"] + ] = OnConflictAction.do_nothing, + values: t.Optional[ + t.Sequence[t.Union[Column, t.Tuple[Column, t.Any]]] + ] = None, + where: t.Optional[Combinable] = None, + ): + action_: OnConflictAction + if isinstance(action, OnConflictAction): + action_ = action + elif isinstance(action, str): + action_ = OnConflictAction(action.upper()) + else: + raise ValueError("Unrecognised `on conflict` action.") + + if where and action_ == OnConflictAction.do_nothing: + raise ValueError( + "The `where` option can only be used with DO NOTHING." + ) + + self._on_conflict.on_conflict_items.append( + OnConflictItem( + target=target, action=action_, values=values, where=where + ) + ) diff --git a/tests/table/test_insert.py b/tests/table/test_insert.py index 474497f25..1c5fab732 100644 --- a/tests/table/test_insert.py +++ b/tests/table/test_insert.py @@ -1,8 +1,22 @@ +import sqlite3 +from unittest import TestCase + import pytest -from tests.base import DBTestCase, engine_version_lt, is_running_sqlite +from piccolo.columns import Integer, Varchar +from piccolo.query.methods.insert import OnConflictAction +from piccolo.table import Table +from piccolo.utils.lazy_loader import LazyLoader +from tests.base import ( + DBTestCase, + engine_version_lt, + engines_only, + is_running_sqlite, +) from tests.example_apps.music.tables import Band, Manager +asyncpg = LazyLoader("asyncpg", globals(), "asyncpg") + class TestInsert(DBTestCase): def test_insert(self): @@ -76,3 +90,385 @@ def test_insert_returning_alias(self): ) self.assertListEqual(response, [{"manager_name": "Maz"}]) + + +@pytest.mark.skipif( + is_running_sqlite() and engine_version_lt(3.24), + reason="SQLite version not supported", +) +class TestOnConflict(TestCase): + class Band(Table): + name = Varchar(unique=True) + popularity = Integer() + + def setUp(self) -> None: + Band = self.Band + Band.create_table().run_sync() + self.band = Band({Band.name: "Pythonistas", Band.popularity: 1000}) + self.band.save().run_sync() + + def tearDown(self) -> None: + Band = self.Band + Band.alter().drop_table().run_sync() + + def test_do_update(self): + """ + Make sure that `DO UPDATE` works. + """ + Band = self.Band + + new_popularity = self.band.popularity + 1000 + + Band.insert( + Band(name=self.band.name, popularity=new_popularity) + ).on_conflict( + target=Band.name, + action="DO UPDATE", + values=[Band.popularity], + ).run_sync() + + self.assertListEqual( + Band.select().run_sync(), + [ + { + "id": self.band.id, + "name": self.band.name, + "popularity": new_popularity, # changed + } + ], + ) + + def test_do_update_tuple_values(self): + """ + Make sure we can use tuples in ``values``. + """ + Band = self.Band + + new_popularity = self.band.popularity + 1000 + new_name = "Rustaceans" + + Band.insert( + Band( + id=self.band.id, + name=new_name, + popularity=new_popularity, + ) + ).on_conflict( + action="DO UPDATE", + target=Band.id, + values=[ + (Band.name, new_name), + (Band.popularity, new_popularity + 2000), + ], + ).run_sync() + + self.assertListEqual( + Band.select().run_sync(), + [ + { + "id": self.band.id, + "name": new_name, + "popularity": new_popularity + 2000, + } + ], + ) + + def test_do_update_no_values(self): + """ + Make sure that `DO UPDATE` with no `values` raises an exception. + """ + Band = self.Band + + new_popularity = self.band.popularity + 1000 + + with self.assertRaises(ValueError) as manager: + Band.insert( + Band(name=self.band.name, popularity=new_popularity) + ).on_conflict( + target=Band.name, + action="DO UPDATE", + ).run_sync() + + self.assertEqual( + manager.exception.__str__(), + "No values specified for `on conflict`", + ) + + @engines_only("postgres", "cockroach") + def test_target_tuple(self): + """ + Make sure that a composite unique constraint can be used as a target. + + We only run it on Postgres and Cockroach because we use ALTER TABLE + to add a contraint, which SQLite doesn't support. + """ + Band = self.Band + + # Add a composite unique constraint: + Band.raw( + "ALTER TABLE band ADD CONSTRAINT id_name_unique UNIQUE (id, name)" + ).run_sync() + + Band.insert( + Band( + id=self.band.id, + name=self.band.name, + popularity=self.band.popularity, + ) + ).on_conflict( + target=(Band.id, Band.name), + action="DO NOTHING", + ).run_sync() + + @engines_only("postgres", "cockroach") + def test_target_string(self): + """ + Make sure we can explicitly specify the name of target constraint using + a string. + + We just test this on Postgres for now, as we have to get the constraint + name from the database. + """ + Band = self.Band + + constraint_name = Band.raw( + """ + SELECT constraint_name + FROM information_schema.constraint_column_usage + WHERE column_name = 'name' + AND table_name = 'band'; + """ + ).run_sync()[0]["constraint_name"] + + query = Band.insert(Band(name=self.band.name)).on_conflict( + target=constraint_name, + action="DO NOTHING", + ) + self.assertIn(f'ON CONSTRAINT "{constraint_name}"', query.__str__()) + query.run_sync() + + def test_violate_non_target(self): + """ + Make sure that if we specify a target constraint, but violate a + different constraint, then we still get the error. + """ + Band = self.Band + + new_popularity = self.band.popularity + 1000 + + with self.assertRaises(Exception) as manager: + Band.insert( + Band(name=self.band.name, popularity=new_popularity) + ).on_conflict( + target=Band.id, # Target the primary key instead. + action="DO UPDATE", + values=[Band.popularity], + ).run_sync() + + if self.Band._meta.db.engine_type in ("postgres", "cockroach"): + self.assertIsInstance( + manager.exception, asyncpg.exceptions.UniqueViolationError + ) + elif self.Band._meta.db.engine_type == "sqlite": + self.assertIsInstance(manager.exception, sqlite3.IntegrityError) + + def test_where(self): + """ + Make sure we can pass in a `where` argument. + """ + Band = self.Band + + new_popularity = self.band.popularity + 1000 + + query = Band.insert( + Band(name=self.band.name, popularity=new_popularity) + ).on_conflict( + target=Band.name, + action="DO UPDATE", + values=[Band.popularity], + where=Band.popularity < self.band.popularity, + ) + + self.assertIn( + f'WHERE "band"."popularity" < {self.band.popularity}', + query.__str__(), + ) + + query.run_sync() + + def test_do_nothing_where(self): + """ + Make sure an error is raised if `where` is used with `DO NOTHING`. + """ + Band = self.Band + + with self.assertRaises(ValueError) as manager: + Band.insert(Band()).on_conflict( + action="DO NOTHING", + where=Band.popularity < self.band.popularity, + ) + + self.assertEqual( + manager.exception.__str__(), + "The `where` option can only be used with DO NOTHING.", + ) + + def test_do_nothing(self): + """ + Make sure that `DO NOTHING` works. + """ + Band = self.Band + + new_popularity = self.band.popularity + 1000 + + Band.insert( + Band(name="Pythonistas", popularity=new_popularity) + ).on_conflict(action="DO NOTHING").run_sync() + + self.assertListEqual( + Band.select().run_sync(), + [ + { + "id": self.band.id, + "name": self.band.name, + "popularity": self.band.popularity, + } + ], + ) + + @engines_only("sqlite") + def test_multiple_do_update(self): + """ + Make sure multiple `ON CONFLICT` clauses work for SQLite. + """ + Band = self.Band + + new_popularity = self.band.popularity + 1000 + + # Conflicting with name - should update. + Band.insert( + Band(name="Pythonistas", popularity=new_popularity) + ).on_conflict(action="DO NOTHING", target=Band.id).on_conflict( + action="DO UPDATE", target=Band.name, values=[Band.popularity] + ).run_sync() + + self.assertListEqual( + Band.select().run_sync(), + [ + { + "id": self.band.id, + "name": self.band.name, + "popularity": new_popularity, # changed + } + ], + ) + + @engines_only("sqlite") + def test_multiple_do_nothing(self): + """ + Make sure multiple `ON CONFLICT` clauses work for SQLite. + """ + Band = self.Band + + new_popularity = self.band.popularity + 1000 + + # Conflicting with ID - should be ignored. + Band.insert( + Band( + id=self.band.id, + name="Pythonistas", + popularity=new_popularity, + ) + ).on_conflict(action="DO NOTHING", target=Band.id).on_conflict( + action="DO UPDATE", + target=Band.name, + values=[Band.popularity], + ).run_sync() + + self.assertListEqual( + Band.select().run_sync(), + [ + { + "id": self.band.id, + "name": self.band.name, + "popularity": self.band.popularity, + } + ], + ) + + @engines_only("postgres", "cockroach") + def test_mutiple_error(self): + """ + Postgres and Cockroach don't support multiple `ON CONFLICT` clauses. + """ + with self.assertRaises(NotImplementedError) as manager: + Band = self.Band + + Band.insert(Band()).on_conflict(action="DO NOTHING").on_conflict( + action="DO UPDATE", + ).run_sync() + + assert manager.exception.__str__() == ( + "Postgres and Cockroach only support a single ON CONFLICT clause." + ) + + def test_all_columns(self): + """ + We can use ``all_columns`` instead of specifying the ``values`` + manually. + """ + Band = self.Band + + new_popularity = self.band.popularity + 1000 + new_name = "Rustaceans" + + # Conflicting with ID - should be ignored. + q = Band.insert( + Band( + id=self.band.id, + name=new_name, + popularity=new_popularity, + ) + ).on_conflict( + action="DO UPDATE", + target=Band.id, + values=Band.all_columns(), + ) + q.run_sync() + + self.assertListEqual( + Band.select().run_sync(), + [ + { + "id": self.band.id, + "name": new_name, + "popularity": new_popularity, + } + ], + ) + + def test_enum(self): + """ + A string literal can be passed in, or an enum, to determine the action. + Make sure that the enum works. + """ + Band = self.Band + + Band.insert( + Band( + id=self.band.id, + name=self.band.name, + popularity=self.band.popularity, + ) + ).on_conflict(action=OnConflictAction.do_nothing).run_sync() + + self.assertListEqual( + Band.select().run_sync(), + [ + { + "id": self.band.id, + "name": self.band.name, + "popularity": self.band.popularity, + } + ], + ) diff --git a/tests/table/test_select.py b/tests/table/test_select.py index 2ce429c2d..892972aec 100644 --- a/tests/table/test_select.py +++ b/tests/table/test_select.py @@ -1364,12 +1364,12 @@ def test_distinct_on_sqlite(self): SQLite doesn't support ``DISTINCT ON``, so a ``ValueError`` should be raised. """ - with self.assertRaises(ValueError) as manager: + with self.assertRaises(NotImplementedError) as manager: Album.select().distinct(on=[Album.band]) self.assertEqual( manager.exception.__str__(), - "Only Postgres and Cockroach supports DISTINCT ON", + "SQLite doesn't support DISTINCT ON", ) @engines_only("postgres", "cockroach") From 8edabba31ef3564d1ede7f5c406a39e9a07721db Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 3 May 2023 17:32:06 +0100 Subject: [PATCH 466/727] Create .readthedocs.yaml (#819) --- .readthedocs.yaml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .readthedocs.yaml diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000..2e91ea548 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,21 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +sphinx: + configuration: docs/src/conf.py + +formats: + - pdf + - epub + +python: + install: + - requirements: requirements/readthedocs-requirements.txt From 2786348bc0f7eabad99486197b0789c52f3d7c25 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 3 May 2023 22:56:51 +0100 Subject: [PATCH 467/727] bumped version --- CHANGES.rst | 34 ++++++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index b7d869036..c9a9315c6 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,40 @@ Changes ======= +0.111.0 +------- + +Added the ``on_conflict`` clause for ``insert`` queries. This enables **upserts**. + +For example, here we insert some bands, and if they already exist then do +nothing: + +.. code-block:: python + + await Band.insert( + Band(name='Pythonistas'), + Band(name='Rustaceans'), + Band(name='C-Sharps'), + ).on_conflict(action='DO NOTHING') + +Here we insert some albums, and if they already exist then we update the price: + +.. code-block:: python + + await Album.insert( + Album(title='OK Computer', price=10.49), + Album(title='Kid A', price=9.99), + Album(title='The Bends', price=9.49), + ).on_conflict( + action='DO UPDATE', + target=DiscountCode.title, + values=[DiscountCode.price] + ) + +Thanks to @sinisaos for helping with this. + +------------------------------------------------------------------------------- + 0.110.0 ------- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 3fdc931bf..6ab050680 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.110.0" +__VERSION__ = "0.111.0" From e58d45aa6995e45e2e39be143b34e62ffc8d43ae Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 3 May 2023 23:09:16 +0100 Subject: [PATCH 468/727] fix typos in changelog --- CHANGES.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index c9a9315c6..d2d2315d6 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -27,8 +27,8 @@ Here we insert some albums, and if they already exist then we update the price: Album(title='The Bends', price=9.49), ).on_conflict( action='DO UPDATE', - target=DiscountCode.title, - values=[DiscountCode.price] + target=Album.title, + values=[Album.price] ) Thanks to @sinisaos for helping with this. From 68a56be92792fe1dd1f53812bf651e17b8386a9d Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 11 May 2023 21:47:27 +0100 Subject: [PATCH 469/727] fix `ModelBuilder` with `Decimal` / `Numeric` columns with no `digits` arg (#824) --- piccolo/testing/model_builder.py | 2 +- tests/testing/test_model_builder.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/piccolo/testing/model_builder.py b/piccolo/testing/model_builder.py index 3058422b8..c2bb38007 100644 --- a/piccolo/testing/model_builder.py +++ b/piccolo/testing/model_builder.py @@ -152,7 +152,7 @@ def _randomize_attribute(cls, column: Column) -> t.Any: """ random_value: t.Any if column.value_type == Decimal: - precision, scale = column._meta.params["digits"] + precision, scale = column._meta.params["digits"] or (4, 2) random_value = RandomBuilder.next_float( maximum=10 ** (precision - scale), scale=scale ) diff --git a/tests/testing/test_model_builder.py b/tests/testing/test_model_builder.py index 6e52289f8..fe6ee77a6 100644 --- a/tests/testing/test_model_builder.py +++ b/tests/testing/test_model_builder.py @@ -1,7 +1,7 @@ import asyncio import unittest -from piccolo.columns import Array, Integer, Real, Varchar +from piccolo.columns import Array, Decimal, Integer, Numeric, Real, Varchar from piccolo.table import Table from piccolo.testing.model_builder import ModelBuilder from tests.base import engines_skip @@ -23,6 +23,13 @@ class TableWithArrayField(Table): floats = Array(Real()) +class TableWithDecimal(Table): + numeric = Numeric() + numeric_with_digits = Numeric(digits=(4, 2)) + decimal = Decimal() + decimal_with_digits = Decimal(digits=(4, 2)) + + # Cockroach Bug: Can turn ON when resolved: https://github.com/cockroachdb/cockroach/issues/71908 # noqa: E501 @engines_skip("cockroach") class TestModelBuilder(unittest.TestCase): @@ -39,12 +46,14 @@ def setUpClass(cls): Concert, Ticket, TableWithArrayField, + TableWithDecimal, ): table_class.create_table().run_sync() @classmethod def tearDownClass(cls) -> None: for table_class in ( + TableWithDecimal, TableWithArrayField, Ticket, Concert, @@ -66,6 +75,7 @@ async def build_model(model): asyncio.run(build_model(Poster)) asyncio.run(build_model(RecordingStudio)) asyncio.run(build_model(TableWithArrayField)) + asyncio.run(build_model(TableWithDecimal)) def test_model_builder_sync(self): ModelBuilder.build_sync(Manager) @@ -73,6 +83,7 @@ def test_model_builder_sync(self): ModelBuilder.build_sync(Poster) ModelBuilder.build_sync(RecordingStudio) ModelBuilder.build_sync(TableWithArrayField) + ModelBuilder.build_sync(TableWithDecimal) def test_model_builder_with_choices(self): shirt = ModelBuilder.build_sync(Shirt) From 6a350dd8e0f8972fe76db45d185c919a250fae31 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 16 May 2023 16:37:54 +0100 Subject: [PATCH 470/727] bumped version --- CHANGES.rst | 7 +++++++ piccolo/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index d2d2315d6..750e1b0ae 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,13 @@ Changes ======= +0.111.1 +------- + +Fixing a bug with ``ModelBuilder`` and ``Decimal`` / ``Numeric`` columns. + +------------------------------------------------------------------------------- + 0.111.0 ------- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 6ab050680..c84038cb9 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.111.0" +__VERSION__ = "0.111.1" From 7fc82af9c5634a53a660d16026c17914ec3864c3 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 22 May 2023 22:57:05 +0100 Subject: [PATCH 471/727] Postgres schema support (#569) * prototype for postgres schema support * using new `schema` arg in `generate` * added `TestCreateWithSchema` * make sure `table_exists` works with Postgres schemas * refactor `table_exists` test * add tests for foreign keys / joins * fix black error * fix tests * make postgres and cockroach querystrings the same * make Cockroach and Postgres share create index DDL * be able to change the schema for a table * add docs for `set_schema` * add support for schema to migrations * added `SchemaManager` * make sure `DropTable` is schema aware * try and create a schema before the table * skip schema tests for sqlite * use `SchemaManager` in existing tests * fix assertions * fix typo in docstring * added `test_change_schema` * added docs * tweak docs * added `SchemaManager` to the docs * tweak docs * add a test for auto migrations * fix test * added `auto_create_schema` option to create table * add comment about not dropping schemas * improve docs * add schema rename support to `SchemaManager` * test running migrations backwards * add test for `schema` arg of `Table` metaclass * add more detail to comment about why we don't drop schemas * make sure schema changes work with `--preview` * add tests for previewing migration * use `SchemaManager` in `TestCreateWithSchema` * add a test for dropping a table in a schema * improve alter statement docstrings * remove unused `StringifiedCallable` * make sure we don't create the schema if it's 'public' * remove print statement * clean up * remove unused import * skip `TestTableExistsSchema` for sqlite --- docs/src/piccolo/api_reference/index.rst | 10 + docs/src/piccolo/query_types/alter.rst | 23 ++ docs/src/piccolo/schema/advanced.rst | 56 +++- .../apps/migrations/auto/diffable_table.py | 24 +- .../apps/migrations/auto/migration_manager.py | 78 ++++- piccolo/apps/migrations/auto/operations.py | 8 + piccolo/apps/migrations/auto/schema_differ.py | 197 +++++++++-- .../apps/migrations/auto/schema_snapshot.py | 6 + piccolo/apps/migrations/commands/backwards.py | 25 +- piccolo/apps/migrations/commands/new.py | 1 + piccolo/apps/schema/commands/generate.py | 6 +- piccolo/columns/base.py | 7 +- piccolo/query/methods/alter.py | 124 +++++-- piccolo/query/methods/create.py | 45 ++- piccolo/query/methods/create_index.py | 16 +- piccolo/query/methods/delete.py | 2 +- piccolo/query/methods/indexes.py | 10 +- piccolo/query/methods/insert.py | 2 +- piccolo/query/methods/select.py | 12 +- piccolo/query/methods/table_exists.py | 27 +- piccolo/query/methods/update.py | 2 +- piccolo/schema.py | 308 ++++++++++++++++++ piccolo/table.py | 44 ++- .../auto/integration/test_migrations.py | 127 +++++++- .../migrations/auto/test_migration_manager.py | 39 ++- .../migrations/auto/test_schema_differ.py | 47 ++- tests/apps/schema/commands/test_generate.py | 40 +-- tests/base.py | 8 +- tests/columns/foreign_key/test_schema.py | 103 ++++++ tests/conf/test_apps.py | 36 +- tests/example_apps/music/piccolo_app.py | 24 +- tests/table/test_alter.py | 60 ++++ tests/table/test_create.py | 51 +++ tests/table/test_drop_db_tables.py | 16 +- tests/table/test_metaclass.py | 33 ++ tests/table/test_raw.py | 4 +- tests/table/test_table_exists.py | 28 +- tests/test_schema.py | 184 +++++++++++ 38 files changed, 1602 insertions(+), 231 deletions(-) create mode 100644 piccolo/schema.py create mode 100644 tests/columns/foreign_key/test_schema.py create mode 100644 tests/test_schema.py diff --git a/docs/src/piccolo/api_reference/index.rst b/docs/src/piccolo/api_reference/index.rst index 439d9c0ec..ce8804a2f 100644 --- a/docs/src/piccolo/api_reference/index.rst +++ b/docs/src/piccolo/api_reference/index.rst @@ -11,6 +11,16 @@ Table ------------------------------------------------------------------------------- +SchemaManager +------------- + +.. currentmodule:: piccolo.schema + +.. autoclass:: SchemaManager + :members: + +------------------------------------------------------------------------------- + Column ------ diff --git a/docs/src/piccolo/query_types/alter.rst b/docs/src/piccolo/query_types/alter.rst index 2542641f6..c7fe0fcbc 100644 --- a/docs/src/piccolo/query_types/alter.rst +++ b/docs/src/piccolo/query_types/alter.rst @@ -86,6 +86,29 @@ Set whether a column is nullable or not. ------------------------------------------------------------------------------- +set_schema +---------- + +Used to change the `schema `_ +which a table belongs to. + +.. code-block:: python + + await Band.alter().set_schema('schema_1') + +Schemas are a way of organising the tables within a database. Only Postgres and +Cockroach support schemas. :ref:`Learn more here `. + +After changing a table's schema, you need to update your ``Table`` accordingly, +otherwise subsequent queries will fail, as they'll be trying to find the table +in the old schema. + +.. code-block:: python + + Band._meta.schema = 'schema_1' + +------------------------------------------------------------------------------- + set_unique ---------- diff --git a/docs/src/piccolo/schema/advanced.rst b/docs/src/piccolo/schema/advanced.rst index 0a7147c4d..0e4f3db0f 100644 --- a/docs/src/piccolo/schema/advanced.rst +++ b/docs/src/piccolo/schema/advanced.rst @@ -3,6 +3,60 @@ Advanced ======== +.. _Schemas: + +Schemas +------- + +Postgres and CoackroachDB have a concept called **schemas**. + +It's a way of grouping the tables in a database. To learn more: + +* `Postgres docs `_ +* `CockroachDB docs `_ + +To specify a table's schema, do the following: + +.. code-block:: python + + class Band(Table, schema="music"): + ... + + # The table will be created in the `music` schema. + # The music schema will also be created if it doesn't already exist. + >>> await Band.create_table() + +If the ``schema`` argument isn't specified, then the table is created in the +``public`` schema. + +Migration support +~~~~~~~~~~~~~~~~~ + +Schemas are fully supported in :ref:`database migrations `. +For example, if we change the ``schema`` argument: + +.. code-block:: python + + class Band(Table, schema="music_2"): + ... + +Then create an automatic migration and run it, then the table will be moved to +the new schema: + +.. code-block:: bash + + >>> piccolo migrations new my_app --auto + >>> piccolo migrations forwards my_app + +``SchemaManager`` +~~~~~~~~~~~~~~~~~ + +The :class:`SchemaManager ` class is used +internally by Piccolo to interact with schemas. You may find it useful if you +want to write a script to interact with schemas (create / delete / list etc). + +------------------------------------------------------------------------------- + Readable -------- @@ -321,4 +375,4 @@ And later we can retrieve the value of the attribute: .. code-block:: python >>> MyTable.my_col.custom_attr - 'foo' \ No newline at end of file + 'foo' diff --git a/piccolo/apps/migrations/auto/diffable_table.py b/piccolo/apps/migrations/auto/diffable_table.py index 18a7c2809..b4457cc4d 100644 --- a/piccolo/apps/migrations/auto/diffable_table.py +++ b/piccolo/apps/migrations/auto/diffable_table.py @@ -16,19 +16,24 @@ from piccolo.table import Table, create_table_class -def compare_dicts(dict_1, dict_2) -> t.Dict[str, t.Any]: +def compare_dicts( + dict_1: t.Dict[str, t.Any], dict_2: t.Dict[str, t.Any] +) -> t.Dict[str, t.Any]: """ Returns a new dictionary which only contains key, value pairs which are in the first dictionary and not the second. - For example: - dict_1 = {'a': 1, 'b': 2} - dict_2 = {'a': 1} - returns {'b': 2} + For example:: - dict_1 = {'a': 2, 'b': 2} - dict_2 = {'a': 1} - returns {'a': 2, 'b': 2} + >>> dict_1 = {'a': 1, 'b': 2} + >>> dict_2 = {'a': 1} + >>> compare_dicts(dict_1, dict_2) + {'b': 2} + + >>> dict_1 = {'a': 2, 'b': 2} + >>> dict_2 = {'a': 1} + >>> compare_dicts(dict_1, dict_2) + {'a': 2, 'b': 2} """ output = {} @@ -90,6 +95,7 @@ class DiffableTable: class_name: str tablename: str + schema: t.Optional[str] = None columns: t.List[Column] = field(default_factory=list) previous_class_name: t.Optional[str] = None @@ -212,7 +218,7 @@ def to_table_class(self) -> t.Type[Table]: """ return create_table_class( class_name=self.class_name, - class_kwargs={"tablename": self.tablename}, + class_kwargs={"tablename": self.tablename, "schema": self.schema}, class_members={ column._meta.name: column for column in self.columns }, diff --git a/piccolo/apps/migrations/auto/migration_manager.py b/piccolo/apps/migrations/auto/migration_manager.py index ed5cf0905..2bfde6ff5 100644 --- a/piccolo/apps/migrations/auto/migration_manager.py +++ b/piccolo/apps/migrations/auto/migration_manager.py @@ -7,6 +7,7 @@ from piccolo.apps.migrations.auto.diffable_table import DiffableTable from piccolo.apps.migrations.auto.operations import ( AlterColumn, + ChangeTableSchema, DropColumn, RenameColumn, RenameTable, @@ -135,6 +136,9 @@ class MigrationManager: add_tables: t.List[DiffableTable] = field(default_factory=list) drop_tables: t.List[DiffableTable] = field(default_factory=list) rename_tables: t.List[RenameTable] = field(default_factory=list) + change_table_schemas: t.List[ChangeTableSchema] = field( + default_factory=list + ) add_columns: AddColumnCollection = field( default_factory=AddColumnCollection ) @@ -156,6 +160,7 @@ def add_table( self, class_name: str, tablename: str, + schema: str = None, columns: t.Optional[t.List[Column]] = None, ): if not columns: @@ -163,13 +168,36 @@ def add_table( self.add_tables.append( DiffableTable( - class_name=class_name, tablename=tablename, columns=columns + class_name=class_name, + tablename=tablename, + columns=columns, + schema=schema, ) ) - def drop_table(self, class_name: str, tablename: str): + def drop_table( + self, class_name: str, tablename: str, schema: t.Optional[str] = None + ): self.drop_tables.append( - DiffableTable(class_name=class_name, tablename=tablename) + DiffableTable( + class_name=class_name, tablename=tablename, schema=schema + ) + ) + + def change_table_schema( + self, + class_name: str, + tablename: str, + new_schema: t.Optional[str] = None, + old_schema: t.Optional[str] = None, + ): + self.change_table_schemas.append( + ChangeTableSchema( + class_name=class_name, + tablename=tablename, + new_schema=new_schema, + old_schema=old_schema, + ) ) def rename_table( @@ -757,6 +785,49 @@ async def _run_add_columns(self, backwards=False): _Table.create_index([add_column.column]) ) + async def _run_change_table_schema(self, backwards=False): + from piccolo.schema import SchemaManager + + schema_manager = SchemaManager() + + for change_table_schema in self.change_table_schemas: + if backwards: + # Note, we don't try dropping any schemas we may have created. + # It's dangerous to do so, just in case the user manually + # added tables etc to the scheme, and we delete them. + + if change_table_schema.old_schema not in (None, "public"): + await self._run_query( + schema_manager.create_schema( + schema_name=change_table_schema.old_schema, + if_not_exists=True, + ) + ) + await self._run_query( + schema_manager.move_table( + table_name=change_table_schema.tablename, + new_schema=change_table_schema.old_schema or "public", + current_schema=change_table_schema.new_schema, + ) + ) + + else: + if change_table_schema.new_schema not in (None, "public"): + await self._run_query( + schema_manager.create_schema( + schema_name=change_table_schema.new_schema, + if_not_exists=True, + ) + ) + + await self._run_query( + schema_manager.move_table( + table_name=change_table_schema.tablename, + new_schema=change_table_schema.new_schema, + current_schema=change_table_schema.old_schema, + ) + ) + async def run(self, backwards=False): direction = "backwards" if backwards else "forwards" if self.preview: @@ -783,6 +854,7 @@ async def run(self, backwards=False): raw() await self._run_add_tables(backwards=backwards) + await self._run_change_table_schema(backwards=backwards) await self._run_rename_tables(backwards=backwards) await self._run_add_columns(backwards=backwards) await self._run_drop_columns(backwards=backwards) diff --git a/piccolo/apps/migrations/auto/operations.py b/piccolo/apps/migrations/auto/operations.py index 5903e1000..2a5192b3b 100644 --- a/piccolo/apps/migrations/auto/operations.py +++ b/piccolo/apps/migrations/auto/operations.py @@ -12,6 +12,14 @@ class RenameTable: new_tablename: str +@dataclass +class ChangeTableSchema: + class_name: str + tablename: str + old_schema: t.Optional[str] + new_schema: t.Optional[str] + + @dataclass class RenameColumn: table_class_name: str diff --git a/piccolo/apps/migrations/auto/schema_differ.py b/piccolo/apps/migrations/auto/schema_differ.py index 5e8ca076c..5ea83ab6e 100644 --- a/piccolo/apps/migrations/auto/schema_differ.py +++ b/piccolo/apps/migrations/auto/schema_differ.py @@ -1,5 +1,6 @@ from __future__ import annotations +import inspect import typing as t from copy import deepcopy from dataclasses import dataclass, field @@ -8,7 +9,12 @@ DiffableTable, TableDelta, ) -from piccolo.apps.migrations.auto.operations import RenameColumn, RenameTable +from piccolo.apps.migrations.auto.migration_manager import MigrationManager +from piccolo.apps.migrations.auto.operations import ( + ChangeTableSchema, + RenameColumn, + RenameTable, +) from piccolo.apps.migrations.auto.serialisation import ( Definition, Import, @@ -43,6 +49,14 @@ def renamed_from(self, new_class_name: str) -> t.Optional[str]: return rename[0].old_class_name if rename else None +@dataclass +class ChangeTableSchemaCollection: + collection: t.List[ChangeTableSchema] = field(default_factory=list) + + def append(self, change_table_schema: ChangeTableSchema): + self.collection.append(change_table_schema) + + @dataclass class RenameColumnCollection: rename_columns: t.List[RenameColumn] = field(default_factory=list) @@ -70,10 +84,20 @@ def new_column_names(self): @dataclass class AlterStatements: - statements: t.List[str] + statements: t.List[str] = field(default_factory=list) extra_imports: t.List[Import] = field(default_factory=list) extra_definitions: t.List[Definition] = field(default_factory=list) + def extend(self, alter_statements: AlterStatements): + self.statements.extend(alter_statements.statements) + self.extra_imports.extend(alter_statements.extra_imports) + self.extra_definitions.extend(alter_statements.extra_definitions) + return self + + +def optional_str_repr(value: t.Optional[str]) -> str: + return f"'{value}'" if value else "None" + @dataclass class SchemaDiffer: @@ -98,6 +122,9 @@ def __post_init__(self): self.schema_snapshot_map: t.Dict[str, DiffableTable] = { i.class_name: i for i in self.schema_snapshot } + self.table_schema_changes_collection = ( + self.check_table_schema_changes() + ) self.rename_tables_collection = self.check_rename_tables() self.rename_columns_collection = self.check_renamed_columns() @@ -175,6 +202,28 @@ def check_rename_tables(self) -> RenameTableCollection: return collection + def check_table_schema_changes(self) -> ChangeTableSchemaCollection: + collection = ChangeTableSchemaCollection() + + for table in self.schema: + snapshot_table = self.schema_snapshot_map.get( + table.class_name, None + ) + if not snapshot_table: + continue + + if table.schema != snapshot_table.schema: + collection.append( + ChangeTableSchema( + class_name=table.class_name, + tablename=table.tablename, + new_schema=table.schema, + old_schema=snapshot_table.schema, + ) + ) + + return collection + def check_renamed_columns(self) -> RenameColumnCollection: """ Work out whether any of the columns were renamed. @@ -231,6 +280,54 @@ def check_renamed_columns(self) -> RenameColumnCollection: ########################################################################### + def _stringify_func( + self, + func: t.Callable, + params: t.Dict[str, t.Any], + prefix: t.Optional[str] = None, + ) -> AlterStatements: + """ + Generates a string representing how to call the given function with the + give params. For example:: + + def my_callable(arg_1: str, arg_2: str): + ... + + >>> _stringify_func( + ... my_callable, + ... {"arg_1": "a", "arg_2": "b"} + ... ).statements + ['my_callable(arg_1="a", arg_2="b")'] + + """ + signature = inspect.signature(func) + + if "self" in signature.parameters.keys(): + params["self"] = None + + serialised_params = serialise_params(params) + + func_name = func.__name__ + + # This will raise an exception is we're missing parameters, which helps + # with debugging: + bound = signature.bind(**serialised_params.params) + bound.apply_defaults() + + args = bound.arguments + if "self" in args: + args.pop("self") + + args_str = ", ".join(f"{i}={repr(j)}" for i, j in args.items()) + + return AlterStatements( + statements=[f"{prefix or ''}{func_name}({args_str})"], + extra_definitions=serialised_params.extra_definitions, + extra_imports=serialised_params.extra_imports, + ) + + ########################################################################### + @property def create_tables(self) -> AlterStatements: new_tables: t.List[DiffableTable] = list( @@ -245,12 +342,22 @@ def create_tables(self) -> AlterStatements: not in self.rename_tables_collection.new_class_names ] - return AlterStatements( - statements=[ - f"manager.add_table('{i.class_name}', tablename='{i.tablename}')" # noqa: E501 - for i in new_tables - ] - ) + alter_statements = AlterStatements() + + for i in new_tables: + alter_statements.extend( + self._stringify_func( + func=MigrationManager.add_table, + params={ + "class_name": i.class_name, + "tablename": i.tablename, + "schema": i.schema, + }, + prefix="manager.", + ) + ) + + return alter_statements @property def drop_tables(self) -> AlterStatements: @@ -266,21 +373,52 @@ def drop_tables(self) -> AlterStatements: not in self.rename_tables_collection.old_class_names ] - return AlterStatements( - statements=[ - f"manager.drop_table(class_name='{i.class_name}', tablename='{i.tablename}')" # noqa: E501 - for i in drop_tables - ] - ) + alter_statements = AlterStatements() + + for i in drop_tables: + alter_statements.extend( + self._stringify_func( + func=MigrationManager.drop_table, + params={ + "class_name": i.class_name, + "tablename": i.tablename, + "schema": i.schema, + }, + prefix="manager.", + ) + ) + + return alter_statements @property def rename_tables(self) -> AlterStatements: - return AlterStatements( - statements=[ - f"manager.rename_table(old_class_name='{renamed_table.old_class_name}', old_tablename='{renamed_table.old_tablename}', new_class_name='{renamed_table.new_class_name}', new_tablename='{renamed_table.new_tablename}')" # noqa - for renamed_table in self.rename_tables_collection.rename_tables # noqa: E501 - ] - ) + alter_statements = AlterStatements() + + for i in self.rename_tables_collection.rename_tables: + alter_statements.extend( + self._stringify_func( + func=MigrationManager.rename_table, + params=i.__dict__, + prefix="manager.", + ) + ) + + return alter_statements + + @property + def change_table_schemas(self) -> AlterStatements: + alter_statements = AlterStatements() + + for i in self.table_schema_changes_collection.collection: + alter_statements.extend( + self._stringify_func( + func=MigrationManager.change_table_schema, + params=i.__dict__, + prefix="manager.", + ) + ) + + return alter_statements ########################################################################### @@ -442,12 +580,18 @@ def add_columns(self) -> AlterStatements: @property def rename_columns(self) -> AlterStatements: - return AlterStatements( - statements=[ - f"manager.rename_column(table_class_name='{i.table_class_name}', tablename='{i.tablename}', old_column_name='{i.old_column_name}', new_column_name='{i.new_column_name}', old_db_column_name='{i.old_db_column_name}', new_db_column_name='{i.new_db_column_name}')" # noqa: E501 - for i in self.rename_columns_collection.rename_columns - ] - ) + alter_statements = AlterStatements() + + for i in self.rename_columns_collection.rename_columns: + alter_statements.extend( + self._stringify_func( + func=MigrationManager.rename_column, + params=i.__dict__, + prefix="manager.", + ) + ) + + return alter_statements ########################################################################### @@ -506,6 +650,7 @@ def get_alter_statements(self) -> t.List[AlterStatements]: "Created tables": self.create_tables, "Dropped tables": self.drop_tables, "Renamed tables": self.rename_tables, + "Tables which changed schema": self.change_table_schemas, "Created table columns": self.new_table_columns, "Dropped columns": self.drop_columns, "Columns added to existing tables": self.add_columns, diff --git a/piccolo/apps/migrations/auto/schema_snapshot.py b/piccolo/apps/migrations/auto/schema_snapshot.py index 9543576b5..45963b717 100644 --- a/piccolo/apps/migrations/auto/schema_snapshot.py +++ b/piccolo/apps/migrations/auto/schema_snapshot.py @@ -50,6 +50,12 @@ def get_snapshot(self) -> t.List[DiffableTable]: table.tablename = rename_table.new_tablename break + for change_table_schema in manager.change_table_schemas: + for table in tables: + if table.tablename == change_table_schema.tablename: + table.schema = change_table_schema.new_schema + break + for table in tables: add_columns = manager.add_columns.columns_for_table_class_name( table.class_name diff --git a/piccolo/apps/migrations/commands/backwards.py b/piccolo/apps/migrations/commands/backwards.py index f3dc6fd33..464612011 100644 --- a/piccolo/apps/migrations/commands/backwards.py +++ b/piccolo/apps/migrations/commands/backwards.py @@ -2,6 +2,7 @@ import os import sys +import typing as t from piccolo.apps.migrations.auto.migration_manager import MigrationManager from piccolo.apps.migrations.commands.base import ( @@ -9,6 +10,7 @@ MigrationResult, ) from piccolo.apps.migrations.tables import Migration +from piccolo.conf.apps import AppConfig, MigrationModule class BackwardsMigrationManager(BaseMigrationManager): @@ -27,20 +29,10 @@ def __init__( self.preview = preview super().__init__() - async def run(self) -> MigrationResult: - await self.create_migration_table() - - app_modules = self.get_app_modules() - - migration_modules = {} - - for app_module in app_modules: - app_config = getattr(app_module, "APP_CONFIG") - if app_config.app_name == self.app_name: - migration_modules = self.get_migration_modules( - app_config.migrations_folder_path - ) - break + async def run_migrations_backwards(self, app_config: AppConfig): + migration_modules: t.Dict[ + str, MigrationModule + ] = self.get_migration_modules(app_config.migrations_folder_path) ran_migration_ids = await Migration.get_migrations_which_ran( app_name=self.app_name @@ -112,6 +104,11 @@ async def run(self) -> MigrationResult: print(message, file=sys.stderr) return MigrationResult(success=False, message=message) + async def run(self) -> MigrationResult: + await self.create_migration_table() + app_config = self.get_app_config(self.app_name) + return await self.run_migrations_backwards(app_config=app_config) + async def run_backwards( app_name: str, diff --git a/piccolo/apps/migrations/commands/new.py b/piccolo/apps/migrations/commands/new.py index ec3812139..a9a23cc0f 100644 --- a/piccolo/apps/migrations/commands/new.py +++ b/piccolo/apps/migrations/commands/new.py @@ -190,6 +190,7 @@ async def get_alter_statements( class_name=i.__name__, tablename=i._meta.tablename, columns=i._meta.non_default_columns, + schema=i._meta.schema, ) for i in app_config.table_classes ] diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index 1eb30f390..3cbe729e4 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -666,10 +666,6 @@ async def get_foreign_key_reference( return ConstraintTable() -def get_table_name(name: str, schema: str) -> str: - return name if schema == "public" else f"{schema}.{name}" - - async def create_table_class_from_db( table_class: t.Type[Table], tablename: str, @@ -812,7 +808,7 @@ async def create_table_class_from_db( table = create_table_class( class_name=_snake_to_camel(tablename), - class_kwargs={"tablename": get_table_name(tablename, schema_name)}, + class_kwargs={"tablename": tablename, "schema": schema_name}, class_members=columns, ) output_schema.tables.append(table) diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 714c66de2..8c5b84064 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -909,12 +909,13 @@ def ddl(self) -> str: if not self._meta.null: query += " NOT NULL" - foreign_key_meta: t.Optional[ForeignKeyMeta] = getattr( - self, "_foreign_key_meta", None + foreign_key_meta = t.cast( + t.Optional[ForeignKeyMeta], + getattr(self, "_foreign_key_meta", None), ) if foreign_key_meta: references = foreign_key_meta.resolved_references - tablename = references._meta.tablename + tablename = references._meta.get_formatted_tablename() on_delete = foreign_key_meta.on_delete.value on_update = foreign_key_meta.on_update.value target_column_name = ( diff --git a/piccolo/query/methods/alter.py b/piccolo/query/methods/alter.py index c51caa14d..040b2f883 100644 --- a/piccolo/query/methods/alter.py +++ b/piccolo/query/methods/alter.py @@ -239,9 +239,20 @@ def ddl(self) -> str: ) +@dataclass +class SetSchema(AlterStatement): + __slots__ = ("schema_name",) + + schema_name: str + + @property + def ddl(self) -> str: + return f'SET SCHEMA "{self.schema_name}"' + + @dataclass class DropTable: - tablename: str + table: t.Type[Table] cascade: bool if_exists: bool @@ -252,7 +263,7 @@ def ddl(self) -> str: if self.if_exists: query += " IF EXISTS" - query += f" {self.tablename}" + query += f" {self.table._meta.get_formatted_tablename()}" if self.cascade: query += " CASCADE" @@ -275,6 +286,7 @@ class Alter(DDL): "_set_digits", "_set_length", "_set_null", + "_set_schema", "_set_unique", ) @@ -293,11 +305,15 @@ def __init__(self, table: t.Type[Table], **kwargs): self._set_digits: t.List[SetDigits] = [] self._set_length: t.List[SetLength] = [] self._set_null: t.List[SetNull] = [] + self._set_schema: t.List[SetSchema] = [] self._set_unique: t.List[SetUnique] = [] def add_column(self: Self, name: str, column: Column) -> Self: """ - Band.alter().add_column(‘members’, Integer()) + Add a column to the table:: + + >>> await Band.alter().add_column('members', Integer()) + """ column._meta._table = self.table column._meta._name = name @@ -311,14 +327,20 @@ def add_column(self: Self, name: str, column: Column) -> Self: def drop_column(self, column: t.Union[str, Column]) -> Alter: """ - Band.alter().drop_column(Band.popularity) + Drop a column from the table:: + + >>> await Band.alter().drop_column(Band.popularity) + """ self._drop.append(DropColumn(column)) return self def drop_default(self, column: t.Union[str, Column]) -> Alter: """ - Band.alter().drop_default(Band.popularity) + Drop the default from a column:: + + >>> await Band.alter().drop_default(Band.popularity) + """ self._drop_default.append(DropDefault(column=column)) return self @@ -327,10 +349,13 @@ def drop_table( self, cascade: bool = False, if_exists: bool = False ) -> Alter: """ - Band.alter().drop_table() + Drop the table:: + + >>> await Band.alter().drop_table() + """ self._drop_table = DropTable( - tablename=self.table._meta.tablename, + table=self.table, cascade=cascade, if_exists=if_exists, ) @@ -338,7 +363,10 @@ def drop_table( def rename_table(self, new_name: str) -> Alter: """ - Band.alter().rename_table('musical_group') + Rename the table:: + + >>> await Band.alter().rename_table('musical_group') + """ # We override the existing one rather than appending. self._rename_table = [RenameTable(new_name=new_name)] @@ -348,8 +376,14 @@ def rename_column( self, column: t.Union[str, Column], new_name: str ) -> Alter: """ - Band.alter().rename_column(Band.popularity, ‘rating’) - Band.alter().rename_column('popularity', ‘rating’) + Rename a column on the table:: + + # Specify the column with a `Column` instance: + >>> await Band.alter().rename_column(Band.popularity, 'rating') + + # Or by name: + >>> await Band.alter().rename_column('popularity', 'rating') + """ self._rename_columns.append(RenameColumn(column, new_name)) return self @@ -361,7 +395,16 @@ def set_column_type( using_expression: t.Optional[str] = None, ) -> Alter: """ - Change the type of a column. + Change the type of a column:: + + >>> await Band.alter().set_column_type(Band.popularity, BigInt()) + + :param using_expression: + When changing a column's type, the database doesn't always know how + to convert the existing data in that column to the new type. You + can provide a hint to the database on what to do. For example + ``'name::integer'``. + """ self._set_column_type.append( SetColumnType( @@ -374,9 +417,10 @@ def set_column_type( def set_default(self, column: Column, value: t.Any) -> Alter: """ - Set the default for a column. + Set the default for a column:: + + >>> await Band.alter().set_default(Band.popularity, 0) - Band.alter().set_default(Band.popularity, 0) """ self._set_default.append(SetDefault(column=column, value=value)) return self @@ -385,8 +429,14 @@ def set_null( self, column: t.Union[str, Column], boolean: bool = True ) -> Alter: """ - Band.alter().set_null(Band.name, True) - Band.alter().set_null('name', True) + Change a column to be nullable or not:: + + # Specify the column using a `Column` instance: + >>> await Band.alter().set_null(Band.name, True) + + # Or using a string: + >>> await Band.alter().set_null('name', True) + """ self._set_null.append(SetNull(column, boolean)) return self @@ -395,8 +445,14 @@ def set_unique( self, column: t.Union[str, Column], boolean: bool = True ) -> Alter: """ - Band.alter().set_unique(Band.name, True) - Band.alter().set_unique('name', True) + Make a column unique or not:: + + # Specify the column using a `Column` instance: + >>> await Band.alter().set_unique(Band.name, True) + + # Or using a string: + >>> await Band.alter().set_unique('name', True) + """ self._set_unique.append(SetUnique(column, boolean)) return self @@ -405,9 +461,10 @@ def set_length(self, column: t.Union[str, Varchar], length: int) -> Alter: """ Change the max length of a varchar column. Unfortunately, this isn't supported by SQLite, but SQLite also doesn't enforce any length limits - on varchar columns anyway. + on varchar columns anyway:: + + >>> await Band.alter().set_length('name', 512) - Band.alter().set_length('name', 512) """ if self.engine_type == "sqlite": colored_warning( @@ -454,13 +511,14 @@ def add_foreign_key_constraint( referenced_column_name: str = "id", ) -> Alter: """ - This will add a new foreign key constraint. + Add a new foreign key constraint:: + + >>> await Band.alter().add_foreign_key_constraint( + ... Band.manager, + ... referenced_table_name='manager', + ... on_delete=OnDelete.cascade + ... ) - Band.alter().add_foreign_key_constraint( - Band.manager, - referenced_table_name='manager', - on_delete=OnDelete.cascade - ) """ constraint_name = self._get_constraint_name(column=column) column_name = AlterColumnStatement(column=column).column_name @@ -483,7 +541,7 @@ def set_digits( digits: t.Optional[t.Tuple[int, int]], ) -> Alter: """ - Alter the precision and scale for a Numeric column. + Alter the precision and scale for a ``Numeric`` column. """ column_type = ( column.__class__.__name__.upper() @@ -499,12 +557,23 @@ def set_digits( ) return self + def set_schema(self, schema_name: str) -> Alter: + """ + Move the table to a different schema. + + :param schema_name: + The schema to move the table to. + + """ + self._set_schema.append(SetSchema(schema_name=schema_name)) + return self + @property def default_ddl(self) -> t.Sequence[str]: if self._drop_table is not None: return [self._drop_table.ddl] - query = f"ALTER TABLE {self.table._meta.tablename}" + query = f"ALTER TABLE {self.table._meta.get_formatted_tablename()}" alterations = [ i.ddl @@ -520,6 +589,7 @@ def default_ddl(self) -> t.Sequence[str]: self._set_length, self._set_default, self._set_digits, + self._set_schema, ) ] diff --git a/piccolo/query/methods/create.py b/piccolo/query/methods/create.py index 94292de51..68cccf6b2 100644 --- a/piccolo/query/methods/create.py +++ b/piccolo/query/methods/create.py @@ -14,21 +14,55 @@ class Create(DDL): Creates a database table. """ - __slots__ = ("if_not_exists", "only_default_columns") + __slots__ = ("if_not_exists", "only_default_columns", "auto_create_schema") def __init__( self, table: t.Type[Table], if_not_exists: bool = False, only_default_columns: bool = False, + auto_create_schema: bool = True, **kwargs, ): + """ + :param table: + The table to create. + :param if_not_exists: + If ``True``, no error will be raised if this table already exists. + :param only_default_columns: + If ``True``, just the basic table and default primary key are + created, rather than all columns. Not typically needed. + :param auto_create_schema: + If the table belongs to a database schema, then make sure the + schema exists before creating the table. + + """ super().__init__(table, **kwargs) self.if_not_exists = if_not_exists self.only_default_columns = only_default_columns + self.auto_create_schema = auto_create_schema @property def default_ddl(self) -> t.Sequence[str]: + ddl: t.List[str] = [] + + schema_name = self.table._meta.schema + if ( + self.auto_create_schema + and schema_name is not None + and schema_name != "public" + and self.engine_type != "sqlite" + ): + from piccolo.schema import CreateSchema + + ddl.append( + CreateSchema( + schema_name=schema_name, + if_not_exists=True, + db=self.table._meta.db, + ).ddl + ) + prefix = "CREATE TABLE" if self.if_not_exists: prefix += " IF NOT EXISTS" @@ -38,14 +72,13 @@ def default_ddl(self) -> t.Sequence[str]: else: columns = self.table._meta.columns - base = f"{prefix} {self.table._meta.tablename}" + base = f"{prefix} {self.table._meta.get_formatted_tablename()}" columns_sql = ", ".join(i.ddl for i in columns) - create_table_ddl = f"{base} ({columns_sql})" + ddl.append(f"{base} ({columns_sql})") - create_indexes: t.List[str] = [] for column in columns: if column._meta.index is True: - create_indexes.extend( + ddl.extend( CreateIndex( table=self.table, columns=[column], @@ -54,4 +87,4 @@ def default_ddl(self) -> t.Sequence[str]: ).ddl ) - return [create_table_ddl] + create_indexes + return ddl diff --git a/piccolo/query/methods/create_index.py b/piccolo/query/methods/create_index.py index 197a6dbda..b10e0c203 100644 --- a/piccolo/query/methods/create_index.py +++ b/piccolo/query/methods/create_index.py @@ -42,7 +42,7 @@ def prefix(self) -> str: def postgres_ddl(self) -> t.Sequence[str]: column_names = self.column_names index_name = self.table._get_index_name(column_names) - tablename = self.table._meta.tablename + tablename = self.table._meta.get_formatted_tablename() method_name = self.method.value column_names_str = ", ".join([f'"{i}"' for i in self.column_names]) return [ @@ -54,23 +54,13 @@ def postgres_ddl(self) -> t.Sequence[str]: @property def cockroach_ddl(self) -> t.Sequence[str]: - column_names = self.column_names - index_name = self.table._get_index_name(column_names) - tablename = self.table._meta.tablename - method_name = self.method.value - column_names_str = ", ".join([f'"{i}"' for i in self.column_names]) - return [ - ( - f"{self.prefix} {index_name} ON {tablename} USING " - f"{method_name} ({column_names_str})" - ) - ] + return self.postgres_ddl @property def sqlite_ddl(self) -> t.Sequence[str]: column_names = self.column_names index_name = self.table._get_index_name(column_names) - tablename = self.table._meta.tablename + tablename = self.table._meta.get_formatted_tablename() method_name = self.method.value if method_name != "btree": diff --git a/piccolo/query/methods/delete.py b/piccolo/query/methods/delete.py index 121375c23..bc0746063 100644 --- a/piccolo/query/methods/delete.py +++ b/piccolo/query/methods/delete.py @@ -53,7 +53,7 @@ def _validate(self): @property def default_querystrings(self) -> t.Sequence[QueryString]: - query = f"DELETE FROM {self.table._meta.tablename}" + query = f"DELETE FROM {self.table._meta.get_formatted_tablename()}" querystring = QueryString(query) diff --git a/piccolo/query/methods/indexes.py b/piccolo/query/methods/indexes.py index 1edbdf00b..14ab8e552 100644 --- a/piccolo/query/methods/indexes.py +++ b/piccolo/query/methods/indexes.py @@ -17,19 +17,13 @@ def postgres_querystrings(self) -> t.Sequence[QueryString]: QueryString( "SELECT indexname AS name FROM pg_indexes " "WHERE tablename = {}", - self.table._meta.tablename, + self.table._meta.get_formatted_tablename(quoted=False), ) ] @property def cockroach_querystrings(self) -> t.Sequence[QueryString]: - return [ - QueryString( - "SELECT indexname AS name FROM pg_indexes " - "WHERE tablename = {}", - self.table._meta.tablename, - ) - ] + return self.postgres_querystrings @property def sqlite_querystrings(self) -> t.Sequence[QueryString]: diff --git a/piccolo/query/methods/insert.py b/piccolo/query/methods/insert.py index 283bdde0b..2963d397a 100644 --- a/piccolo/query/methods/insert.py +++ b/piccolo/query/methods/insert.py @@ -100,7 +100,7 @@ def _raw_response_callback(self, results): @property def default_querystrings(self) -> t.Sequence[QueryString]: - base = f'INSERT INTO "{self.table._meta.tablename}"' + base = f"INSERT INTO {self.table._meta.get_formatted_tablename()}" columns = ",".join( f'"{i._meta.db_column_name}"' for i in self.table._meta.columns ) diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 4e1949f55..5720380e5 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -659,10 +659,12 @@ def _get_joins(self, columns: t.Sequence[Selectable]) -> t.List[str]: index - 1 ]._meta.table_alias else: - left_tablename = key._meta.table._meta.tablename + left_tablename = ( + key._meta.table._meta.get_formatted_tablename() + ) # noqa: E501 right_tablename = ( - key._foreign_key_meta.resolved_references._meta.tablename + key._foreign_key_meta.resolved_references._meta.get_formatted_tablename() # noqa: E501 ) pk_name = column._meta.call_chain[ @@ -670,9 +672,9 @@ def _get_joins(self, columns: t.Sequence[Selectable]) -> t.List[str]: ]._foreign_key_meta.resolved_target_column._meta.name _joins.append( - f'LEFT JOIN "{right_tablename}" "{table_alias}"' + f'LEFT JOIN {right_tablename} "{table_alias}"' " ON " - f'("{left_tablename}"."{key._meta.name}" = "{table_alias}"."{pk_name}")' # noqa: E501 + f'({left_tablename}."{key._meta.name}" = "{table_alias}"."{pk_name}")' # noqa: E501 ) joins.extend(_joins) @@ -742,7 +744,7 @@ def default_querystrings(self) -> t.Sequence[QueryString]: query += "{}" args.append(distinct.querystring) - query += f" {columns_str} FROM {self.table._meta.tablename}" + query += f" {columns_str} FROM {self.table._meta.get_formatted_tablename()}" # noqa: E501 for join in joins: query += f" {join}" diff --git a/piccolo/query/methods/table_exists.py b/piccolo/query/methods/table_exists.py index 81332faf3..dc801bb59 100644 --- a/piccolo/query/methods/table_exists.py +++ b/piccolo/query/methods/table_exists.py @@ -19,24 +19,27 @@ def sqlite_querystrings(self) -> t.Sequence[QueryString]: return [ QueryString( "SELECT EXISTS(SELECT * FROM sqlite_master WHERE " - f"name = '{self.table._meta.tablename}') AS 'exists'" + "name = {}) AS 'exists'", + self.table._meta.tablename, ) ] @property def postgres_querystrings(self) -> t.Sequence[QueryString]: - return [ - QueryString( - "SELECT EXISTS(SELECT * FROM information_schema.tables WHERE " - f"table_name = '{self.table._meta.tablename}')" + subquery = QueryString( + "SELECT * FROM information_schema.tables WHERE table_name = {}", + self.table._meta.tablename, + ) + + if self.table._meta.schema: + subquery = QueryString( + "{} AND table_schema = {}", subquery, self.table._meta.schema ) - ] + + query = QueryString("SELECT EXISTS({})", subquery) + + return [query] @property def cockroach_querystrings(self) -> t.Sequence[QueryString]: - return [ - QueryString( - "SELECT EXISTS(SELECT * FROM information_schema.tables WHERE " - f"table_name = '{self.table._meta.tablename}')" - ) - ] + return self.postgres_querystrings diff --git a/piccolo/query/methods/update.py b/piccolo/query/methods/update.py index 72d62307b..0d914b858 100644 --- a/piccolo/query/methods/update.py +++ b/piccolo/query/methods/update.py @@ -90,7 +90,7 @@ def default_querystrings(self) -> t.Sequence[QueryString]: for col, _ in self.values_delegate._values.items() ) - query = f"UPDATE {self.table._meta.tablename} SET {columns_str}" + query = f"UPDATE {self.table._meta.get_formatted_tablename()} SET {columns_str}" # noqa: E501 querystring = QueryString( query, *self.values_delegate.get_sql_values() diff --git a/piccolo/schema.py b/piccolo/schema.py new file mode 100644 index 000000000..7899238d5 --- /dev/null +++ b/piccolo/schema.py @@ -0,0 +1,308 @@ +from __future__ import annotations + +import abc +import typing as t + +from piccolo.engine.base import Engine +from piccolo.engine.finder import engine_finder +from piccolo.querystring import QueryString +from piccolo.utils.sync import run_sync + + +class SchemaDDLBase(abc.ABC): + + db: Engine + + @abc.abstractproperty + def ddl(self) -> str: + pass + + def __await__(self): + return self.run().__await__() + + async def run(self, in_pool=True): + return await self.db.run_ddl(self.ddl, in_pool=in_pool) + + def run_sync(self, *args, **kwargs): + return run_sync(self.run(*args, **kwargs)) + + def __str__(self) -> str: + return self.ddl.__str__() + + +class CreateSchema(SchemaDDLBase): + def __init__( + self, + schema_name: str, + *, + if_not_exists: bool, + db: Engine, + ): + self.schema_name = schema_name + self.if_not_exists = if_not_exists + self.db = db + + async def run(self, *args, **kwargs): + if self.schema_name == "public" or self.schema_name is None: + return + + return await super().run(self, *args, **kwargs) + + @property + def ddl(self) -> str: + query = "CREATE SCHEMA" + if self.if_not_exists: + query += " IF NOT EXISTS" + query += f' "{self.schema_name}"' + + return query + + +class DropSchema(SchemaDDLBase): + def __init__( + self, + schema_name: str, + *, + if_exists: bool, + cascade: bool, + db: Engine, + ): + self.schema_name = schema_name + self.if_exists = if_exists + self.cascade = cascade + self.db = db + + @property + def ddl(self) -> str: + query = "DROP SCHEMA" + if self.if_exists: + query += " IF EXISTS" + query += f' "{self.schema_name}"' + + if self.cascade: + query += " CASCADE" + + return query + + +class RenameSchema(SchemaDDLBase): + def __init__( + self, + schema_name: str, + new_schema_name: str, + db: Engine, + ): + self.schema_name = schema_name + self.new_schema_name = new_schema_name + self.db = db + + @property + def ddl(self): + return ( + f'ALTER SCHEMA "{self.schema_name}" ' + f'RENAME TO "{self.new_schema_name}"' + ) + + +class MoveTable(SchemaDDLBase): + def __init__( + self, + table_name: str, + new_schema: str, + db: Engine, + current_schema: t.Optional[str] = None, + ): + self.table_name = table_name + self.current_schema = current_schema + self.new_schema = new_schema + self.db = db + + @property + def ddl(self): + table_name = f'"{self.table_name}"' + if self.current_schema: + table_name = f'"{self.current_schema}".{table_name}' + + return f'ALTER TABLE {table_name} SET SCHEMA "{self.new_schema}"' + + +class ListTables: + def __init__(self, db: Engine, schema_name: str): + self.db = db + self.schema_name = schema_name + + async def run(self): + response = await self.db.run_querystring( + QueryString( + """ + SELECT table_name + FROM information_schema.tables + WHERE table_schema = {} + """, + self.schema_name, + ) + ) + return [i["table_name"] for i in response] + + def run_sync(self): + return run_sync(self.run()) + + def __await__(self): + return self.run().__await__() + + +class ListSchemas: + def __init__(self, db: Engine): + self.db = db + + async def run(self): + response = await self.db.run_querystring( + QueryString("SELECT schema_name FROM information_schema.schemata") + ) + return [i["schema_name"] for i in response] + + def run_sync(self): + return run_sync(self.run()) + + def __await__(self): + return self.run().__await__() + + +class SchemaManager: + def __init__(self, db: t.Optional[Engine] = None): + """ + A useful utility class for interacting with schemas. + + :param db: + Used to execute the database queries. If not specified, we try and + import it from ``piccolo_conf.py``. + """ + db = db or engine_finder() + + if not db: + raise ValueError("The DB can't be found.") + + self.db = db + + def create_schema( + self, schema_name: str, *, if_not_exists: bool = True + ) -> CreateSchema: + """ + Creates the specified schema:: + + >>> await SchemaManager().create_schema(schema_name="music") + + :param schema_name: + The name of the schema to create. + :param if_not_exists: + No error will be raised if the schema already exists. + + """ + return CreateSchema( + schema_name=schema_name, + if_not_exists=if_not_exists, + db=self.db, + ) + + def drop_schema( + self, + schema_name: str, + *, + if_exists: bool = True, + cascade: bool = False, + ) -> DropSchema: + """ + Drops the specified schema:: + + >>> await SchemaManager().drop_schema(schema_name="music") + + :param schema_name: + The name of the schema to drop. + :param if_exists: + No error will be raised if the schema doesn't exist. + :param cascade: + If ``True`` then it will automatically drop the tables within the + schema. + + """ + return DropSchema( + schema_name=schema_name, + if_exists=if_exists, + cascade=cascade, + db=self.db, + ) + + def rename_schema( + self, schema_name: str, new_schema_name: str + ) -> RenameSchema: + """ + Rename the schema:: + + >>> await SchemaManager().rename_schema( + ... schema_name="music", + ... new_schema_name="music_info" + ... ) + + :param schema_name: + The current name of the schema. + :param new_schema_name: + What to rename the schema to. + + """ + return RenameSchema( + schema_name=schema_name, + new_schema_name=new_schema_name, + db=self.db, + ) + + def move_table( + self, + table_name: str, + new_schema: str, + current_schema: t.Optional[str] = None, + ) -> MoveTable: + """ + Moves a table to a different schema:: + + >>> await SchemaManager().move_schema( + ... table_name='my_table', + ... new_schema='schema_1' + ... ) + + :param table_name: + The name of the table to move. + :param new_schema: + The name of the scheam you want to move the table too. + :current_schema: + If not specified, ``'public'`` is assumed. + + """ + return MoveTable( + table_name=table_name, + new_schema=new_schema, + current_schema=current_schema, + db=self.db, + ) + + def list_tables(self, schema_name: str) -> ListTables: + """ + Returns the name of each table in the given schema:: + + >>> await SchemaManager().list_tables(schema_name="music") + ['band', 'manager'] + + :param schema_name: + List the tables in this schema. + + """ + return ListTables(db=self.db, schema_name=schema_name) + + def list_schemas(self) -> ListSchemas: + """ + Returns the name of each schema in the database:: + + >>> await SchemaManager().list_schemas() + ['public', 'schema_1'] + + """ + return ListSchemas(db=self.db) diff --git a/piccolo/table.py b/piccolo/table.py index 2982180bd..91cd21ce3 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -4,6 +4,7 @@ import itertools import types import typing as t +import warnings from dataclasses import dataclass, field from piccolo.columns import Column @@ -82,12 +83,36 @@ class TableMeta: help_text: t.Optional[str] = None _db: t.Optional[Engine] = None m2m_relationships: t.List[M2M] = field(default_factory=list) + schema: t.Optional[str] = None # Records reverse foreign key relationships - i.e. when the current table # is the target of a foreign key. Used by external libraries such as # Piccolo API. _foreign_key_references: t.List[ForeignKey] = field(default_factory=list) + def get_formatted_tablename( + self, include_schema: bool = True, quoted: bool = True + ) -> str: + """ + Returns the tablename, in the desired format. + + :param include_schema: + If ``True``, the Postgres schema is included. For example, + 'my_schema.my_table'. + :param quote: + If ``True``, the name is wrapped in double quotes. For example, + '"my_schema"."my_table"'. + + """ + components = [self.tablename] + if include_schema and self.schema: + components.insert(0, self.schema) + + if quoted: + return ".".join(f'"{i}"' for i in components) + else: + return ".".join(components) + @property def foreign_key_references(self) -> t.List[ForeignKey]: foreign_keys: t.List[ForeignKey] = list(self._foreign_key_references) @@ -197,6 +222,7 @@ def __init_subclass__( db: t.Optional[Engine] = None, tags: t.List[str] = None, help_text: t.Optional[str] = None, + schema: t.Optional[str] = None, ): # sourcery no-metrics """ Automatically populate the _meta, which includes the tablename, and @@ -215,12 +241,21 @@ def __init_subclass__( A user friendly description of what the table is used for. It isn't used in the database, but will be used by tools such a Piccolo Admin for tooltips. + :param schema: + The Postgres schema to use for this table. """ if tags is None: tags = [] tablename = tablename or _camel_to_snake(cls.__name__) + if "." in tablename: + warnings.warn( + "There's a '.' in the tablename - please use the `schema` " + "argument instead." + ) + schema, tablename = tablename.split(".", maxsplit=1) + if tablename in PROTECTED_TABLENAMES: raise ValueError( f"{tablename} is a protected name, please give your table a " @@ -310,6 +345,7 @@ def __init_subclass__( help_text=help_text, _db=db, m2m_relationships=m2m_relationships, + schema=schema, ) for foreign_key_column in foreign_key_columns: @@ -936,7 +972,7 @@ def raw(cls, sql: str, *args: t.Any) -> Raw: .. code-block:: python - await Band.raw("select * from band where name = {}", 'Pythonistas') + await Band.raw("SELECT * FROM band WHERE name = {}", 'Pythonistas') """ return Raw(table=cls, querystring=QueryString(sql, *args)) @@ -1004,7 +1040,10 @@ def delete(cls, force=False) -> Delete: @classmethod def create_table( - cls, if_not_exists=False, only_default_columns=False + cls, + if_not_exists=False, + only_default_columns=False, + auto_create_schema: bool = True, ) -> Create: """ Create table, along with all columns. @@ -1018,6 +1057,7 @@ def create_table( table=cls, if_not_exists=if_not_exists, only_default_columns=only_default_columns, + auto_create_schema=auto_create_schema, ) @classmethod diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index 23387c192..2c0925a22 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -11,6 +11,9 @@ import uuid from unittest.mock import MagicMock, patch +from piccolo.apps.migrations.commands.backwards import ( + BackwardsMigrationManager, +) from piccolo.apps.migrations.commands.forwards import ForwardsMigrationManager from piccolo.apps.migrations.commands.new import ( _create_migrations_folder, @@ -46,6 +49,7 @@ from piccolo.columns.m2m import M2M from piccolo.columns.reference import LazyTableReference from piccolo.conf.apps import AppConfig +from piccolo.schema import SchemaManager from piccolo.table import Table, create_table_class, drop_db_tables_sync from piccolo.utils.sync import run_sync from tests.base import DBTestCase, engines_only, engines_skip @@ -99,10 +103,19 @@ def array_default_varchar(): class MigrationTestCase(DBTestCase): - def run_migrations(self, app_config: AppConfig): - manager = ForwardsMigrationManager(app_name=app_config.app_name) - run_sync(manager.create_migration_table()) - run_sync(manager.run_migrations(app_config=app_config)) + def _run_migrations(self, app_config: AppConfig): + forwards_manager = ForwardsMigrationManager( + app_name=app_config.app_name + ) + run_sync(forwards_manager.create_migration_table()) + run_sync(forwards_manager.run_migrations(app_config=app_config)) + + def _get_migrations_folder_path(self) -> str: + temp_directory_path = tempfile.gettempdir() + migrations_folder_path = os.path.join( + temp_directory_path, "piccolo_migrations" + ) + return migrations_folder_path def _test_migrations( self, @@ -114,7 +127,7 @@ def _test_migrations( :param table_snapshots: A list of lists. Each sub list represents a snapshot of the table - state. Migrations will be created and run based each snapshot. + state. Migrations will be created and run based on each snapshot. :param test_function: After the migrations are run, this function is called. It is passed a ``RowMeta`` instance which can be used to check the column was @@ -122,10 +135,7 @@ def _test_migrations( test passes, otherwise ``False``. """ - temp_directory_path = tempfile.gettempdir() - migrations_folder_path = os.path.join( - temp_directory_path, "piccolo_migrations" - ) + migrations_folder_path = self._get_migrations_folder_path() if os.path.exists(migrations_folder_path): shutil.rmtree(migrations_folder_path) @@ -146,7 +156,7 @@ def _test_migrations( ) ) self.assertTrue(os.path.exists(meta.migration_path)) - self.run_migrations(app_config=app_config) + self._run_migrations(app_config=app_config) # It's kind of absurd sleeping for 1 microsecond, but it guarantees # the migration IDs will be unique, and just in case computers @@ -168,6 +178,37 @@ def _test_migrations( msg=f"Meta is incorrect: {row_meta}", ) + def _run_backwards(self, migration_id: str): + """ + After running :meth:`_test_migrations`, if you call `_run_backwards` + then the migrations can be reversed. + + :param migration_id: + Which migration to reverse to. Can be: + + * A migration ID. + * A number, like ``1``, then it will reverse the most recent + migration. + * ``'all'`` then all of the migrations will be reversed. + + """ + migrations_folder_path = self._get_migrations_folder_path() + + app_config = AppConfig( + app_name="test_app", + migrations_folder_path=migrations_folder_path, + table_classes=[], + ) + + backwards_manager = BackwardsMigrationManager( + app_name=app_config.app_name, + migration_id=migration_id, + auto_agree=True, + ) + run_sync( + backwards_manager.run_migrations_backwards(app_config=app_config) + ) + @engines_only("postgres", "cockroach") class TestMigrations(MigrationTestCase): @@ -1027,3 +1068,69 @@ def test_target_column(self): """ ) self.assertTrue(response[0]["exists"]) + + +############################################################################### + + +@engines_only("postgres", "cockroach") +class TestSchemas(MigrationTestCase): + new_schema = "schema_1" + + def setUp(self) -> None: + self.schema_manager = SchemaManager() + + def tearDown(self) -> None: + self.schema_manager.drop_schema( + self.new_schema, if_exists=True, cascade=True + ).run_sync() + Migration.alter().drop_table(if_exists=True).run_sync() + + def test_schemas(self): + """ + Make sure migrations still work when a foreign key references a column + other than the primary key. + """ + manager_1 = create_table_class(class_name="Manager") + manager_2 = create_table_class( + class_name="Manager", class_kwargs={"schema": self.new_schema} + ) + + self._test_migrations( + table_snapshots=[ + [manager_1], + [manager_2], + ], + ) + + # The schema should automaticaly be created. + self.assertIn( + manager_2._meta.schema, + self.schema_manager.list_schemas().run_sync(), + ) + + # Make sure that the table is in the new schema. + self.assertListEqual( + self.schema_manager.list_tables( + schema_name=self.new_schema + ).run_sync(), + ["manager"], + ) + + ####################################################################### + + # Reverse the last migration, which should move the table back to the + # public schema. + self._run_backwards(migration_id="1") + + self.assertIn( + "manager", + self.schema_manager.list_tables(schema_name="public").run_sync(), + ) + + # We don't delete the schema we created as it's risky, just in case + # other tables etc were manually added to it. + self.assertIn( + self.new_schema, + self.schema_manager.list_schemas().run_sync(), + ) diff --git a/tests/apps/migrations/auto/test_migration_manager.py b/tests/apps/migrations/auto/test_migration_manager.py index bfb771452..8d58bb8ce 100644 --- a/tests/apps/migrations/auto/test_migration_manager.py +++ b/tests/apps/migrations/auto/test_migration_manager.py @@ -145,7 +145,7 @@ def test_rename_column(self): asyncio.run(manager.run()) self.assertEqual( fake_out.getvalue(), - """ - [preview forwards]... \n ALTER TABLE band RENAME COLUMN "name" TO "title";\n""", # noqa: E501 + """ - [preview forwards]... \n ALTER TABLE "band" RENAME COLUMN "name" TO "title";\n""", # noqa: E501 ) response = self.run_sync("SELECT * FROM band;") self.assertTrue("title" not in response[0].keys()) @@ -240,8 +240,8 @@ def test_add_table(self, get_app_config: MagicMock): self.assertEqual( response, [{"id": id[0]["id"], "name": "Bob Jones"}] ) - # Reverse + # Reverse get_app_config.return_value = AppConfig( app_name="music", migrations_folder_path="" ) @@ -257,12 +257,12 @@ def test_add_table(self, get_app_config: MagicMock): if engine_is("postgres"): self.assertEqual( fake_out.getvalue(), - """ - [preview forwards]... \n CREATE TABLE musician ("id" SERIAL PRIMARY KEY NOT NULL, "name" VARCHAR(255) NOT NULL DEFAULT '');\n""", # noqa: E501 + """ - [preview forwards]... \n CREATE TABLE "musician" ("id" SERIAL PRIMARY KEY NOT NULL, "name" VARCHAR(255) NOT NULL DEFAULT '');\n""", # noqa: E501 ) if engine_is("cockroach"): self.assertEqual( fake_out.getvalue(), - """ - [preview forwards]... \n CREATE TABLE musician ("id" INTEGER PRIMARY KEY NOT NULL DEFAULT unique_rowid(), "name" VARCHAR(255) NOT NULL DEFAULT '');\n""", # noqa: E501 + """ - [preview forwards]... \n CREATE TABLE "musician" ("id" INTEGER PRIMARY KEY NOT NULL DEFAULT unique_rowid(), "name" VARCHAR(255) NOT NULL DEFAULT '');\n""", # noqa: E501 ) self.assertEqual(self.table_exists("musician"), False) @@ -292,7 +292,7 @@ def test_add_column(self): if engine_is("postgres"): self.run_sync( - "INSERT INTO manager VALUES (default, 'Dave', 'dave@me.com');" + "INSERT INTO \"manager\" VALUES (default, 'Dave', 'dave@me.com');" # noqa: E501 ) response = self.run_sync("SELECT * FROM manager;") self.assertEqual( @@ -326,7 +326,7 @@ def test_add_column(self): asyncio.run(manager.run()) self.assertEqual( fake_out.getvalue(), - """ - [preview forwards]... \n ALTER TABLE manager ADD COLUMN "email" VARCHAR(100) UNIQUE DEFAULT '';\n""", # noqa: E501 + """ - [preview forwards]... \n ALTER TABLE "manager" ADD COLUMN "email" VARCHAR(100) UNIQUE DEFAULT '';\n""", # noqa: E501 ) response = self.run_sync("SELECT * FROM manager;") @@ -373,8 +373,8 @@ def test_add_column_with_index(self): self.assertEqual( fake_out.getvalue(), ( - """ - [preview forwards]... \n ALTER TABLE manager ADD COLUMN "email" VARCHAR(100) UNIQUE DEFAULT '';\n""" # noqa: E501 - """\n CREATE INDEX manager_email ON manager USING btree ("email");\n""" # noqa: E501 + """ - [preview forwards]... \n ALTER TABLE "manager" ADD COLUMN "email" VARCHAR(100) UNIQUE DEFAULT '';\n""" # noqa: E501 + """\n CREATE INDEX manager_email ON "manager" USING btree ("email");\n""" # noqa: E501 ), ) self.assertTrue(index_name not in Manager.indexes().run_sync()) @@ -1028,3 +1028,26 @@ def test_drop_table( self.assertTrue(self.table_exists("musician")) self.run_sync("DROP TABLE IF EXISTS musician;") + + @engines_only("postgres", "cockroach") + def test_change_table_schema(self): + manager = MigrationManager(migration_id="1", app_name="music") + + manager.change_table_schema( + class_name="Manager", + tablename="manager", + new_schema="schema_1", + old_schema=None, + ) + + # Preview + manager.preview = True + with patch("sys.stdout", new=StringIO()) as fake_out: + asyncio.run(manager.run()) + + output = fake_out.getvalue() + + self.assertEqual( + output, + ' - 1 [preview forwards]... CREATE SCHEMA IF NOT EXISTS "schema_1"\nALTER TABLE "manager" SET SCHEMA "schema_1"\n', # noqa: E501 + ) diff --git a/tests/apps/migrations/auto/test_schema_differ.py b/tests/apps/migrations/auto/test_schema_differ.py index 565aa332d..63ff06ea5 100644 --- a/tests/apps/migrations/auto/test_schema_differ.py +++ b/tests/apps/migrations/auto/test_schema_differ.py @@ -32,7 +32,7 @@ def test_add_table(self): self.assertTrue(len(create_tables.statements) == 1) self.assertEqual( create_tables.statements[0], - "manager.add_table('Band', tablename='band')", + "manager.add_table(class_name='Band', tablename='band', schema=None, columns=None)", # noqa: E501 ) new_table_columns = schema_differ.new_table_columns @@ -57,7 +57,7 @@ def test_drop_table(self): self.assertTrue(len(schema_differ.drop_tables.statements) == 1) self.assertEqual( schema_differ.drop_tables.statements[0], - "manager.drop_table(class_name='Band', tablename='band')", + "manager.drop_table(class_name='Band', tablename='band', schema=None)", # noqa: E501 ) def test_rename_table(self): @@ -85,12 +85,47 @@ def test_rename_table(self): self.assertTrue(len(schema_differ.rename_tables.statements) == 1) self.assertEqual( schema_differ.rename_tables.statements[0], - "manager.rename_table(old_class_name='Band', old_tablename='band', new_class_name='Act', new_tablename='act')", # noqa + "manager.rename_table(old_class_name='Band', old_tablename='band', new_class_name='Act', new_tablename='act')", # noqa: E501 ) self.assertEqual(schema_differ.create_tables.statements, []) self.assertEqual(schema_differ.drop_tables.statements, []) + def test_change_schema(self): + """ + Testing changing the schema. + """ + schema: t.List[DiffableTable] = [ + DiffableTable( + class_name="Band", + tablename="band", + columns=[], + schema="schema_1", + ) + ] + schema_snapshot: t.List[DiffableTable] = [ + DiffableTable( + class_name="Band", + tablename="band", + columns=[], + schema=None, + ) + ] + + schema_differ = SchemaDiffer( + schema=schema, schema_snapshot=schema_snapshot, auto_input="y" + ) + + self.assertEqual(len(schema_differ.change_table_schemas.statements), 1) + + self.assertEqual( + schema_differ.change_table_schemas.statements[0], + "manager.change_table_schema(class_name='Band', tablename='band', new_schema='schema_1', old_schema=None)", # noqa: E501 + ) + + self.assertListEqual(schema_differ.create_tables.statements, []) + self.assertListEqual(schema_differ.drop_tables.statements, []) + def test_add_column(self): """ Test adding a column to an existing table. @@ -123,7 +158,7 @@ def test_add_column(self): self.assertTrue(len(schema_differ.add_columns.statements) == 1) self.assertEqual( schema_differ.add_columns.statements[0], - "manager.add_column(table_class_name='Band', tablename='band', column_name='genre', db_column_name='genre', column_class_name='Varchar', column_class=Varchar, params={'length': 255, 'default': '', 'null': False, 'primary_key': False, 'unique': False, 'index': False, 'index_method': IndexMethod.btree, 'choices': None, 'db_column_name': None, 'secret': False})", # noqa + "manager.add_column(table_class_name='Band', tablename='band', column_name='genre', db_column_name='genre', column_class_name='Varchar', column_class=Varchar, params={'length': 255, 'default': '', 'null': False, 'primary_key': False, 'unique': False, 'index': False, 'index_method': IndexMethod.btree, 'choices': None, 'db_column_name': None, 'secret': False})", # noqa: E501 ) def test_drop_column(self): @@ -158,7 +193,7 @@ def test_drop_column(self): self.assertTrue(len(schema_differ.drop_columns.statements) == 1) self.assertEqual( schema_differ.drop_columns.statements[0], - "manager.drop_column(table_class_name='Band', tablename='band', column_name='genre', db_column_name='genre')", # noqa + "manager.drop_column(table_class_name='Band', tablename='band', column_name='genre', db_column_name='genre')", # noqa: E501 ) def test_rename_column(self): @@ -196,7 +231,7 @@ def test_rename_column(self): self.assertEqual( schema_differ.rename_columns.statements, [ - "manager.rename_column(table_class_name='Band', tablename='band', old_column_name='title', new_column_name='name', old_db_column_name='title', new_db_column_name='name')" # noqa + "manager.rename_column(table_class_name='Band', tablename='band', old_column_name='title', new_column_name='name', old_db_column_name='title', new_db_column_name='name')" # noqa: E501 ], ) diff --git a/tests/apps/schema/commands/test_generate.py b/tests/apps/schema/commands/test_generate.py index fc350405c..a08521038 100644 --- a/tests/apps/schema/commands/test_generate.py +++ b/tests/apps/schema/commands/test_generate.py @@ -20,8 +20,8 @@ Varchar, ) from piccolo.columns.indexes import IndexMethod -from piccolo.engine import Engine, engine_finder -from piccolo.table import Table +from piccolo.schema import SchemaManager +from piccolo.table import Table, create_db_tables_sync from piccolo.utils.sync import run_sync from tests.base import AsyncMock, engines_only, engines_skip from tests.example_apps.mega.tables import MegaTable, SmallTable @@ -212,11 +212,11 @@ def test_index(self): ############################################################################### -class Publication(Table, tablename="schema2.publication"): +class Publication(Table, tablename="publication", schema="schema_2"): name = Varchar(length=50) -class Writer(Table, tablename="schema1.writer"): +class Writer(Table, tablename="writer", schema="schema_1"): name = Varchar(length=50) publication = ForeignKey(Publication, null=True) @@ -229,26 +229,26 @@ class Book(Table): @engines_only("postgres") class TestGenerateWithSchema(TestCase): - def setUp(self) -> None: - engine: t.Optional[Engine] = engine_finder() - class Schema(Table, db=engine): - """ - Only for raw query execution - """ + tables = [Publication, Writer, Book] - pass + schema_manager = SchemaManager() - Schema.raw("CREATE SCHEMA IF NOT EXISTS schema1").run_sync() - Schema.raw("CREATE SCHEMA IF NOT EXISTS schema2").run_sync() - Publication.create_table().run_sync() - Writer.create_table().run_sync() - Book.create_table().run_sync() + def setUp(self) -> None: + for schema_name in ("schema_1", "schema_2"): + self.schema_manager.create_schema( + schema_name=schema_name, if_not_exists=True + ).run_sync() + + create_db_tables_sync(*self.tables) def tearDown(self) -> None: Book.alter().drop_table().run_sync() - Writer.alter().drop_table().run_sync() - Publication.alter().drop_table().run_sync() + + for schema_name in ("schema_1", "schema_2"): + self.schema_manager.drop_schema( + schema_name=schema_name, if_exists=True, cascade=True + ).run_sync() def test_reference_to_another_schema(self): output_schema: OutputSchema = run_sync(get_output_schema()) @@ -283,8 +283,8 @@ def tearDown(self): ) def test_exception(self, create_table_class_from_db_mock: AsyncMock): """ - Make sure that a GenerateError exception - is raised with all the exceptions gathered. + Make sure that a GenerateError exception is raised with all the + exceptions gathered. """ create_table_class_from_db_mock.side_effect = [ ValueError("Test"), diff --git a/tests/base.py b/tests/base.py index f08312fc8..6978aba98 100644 --- a/tests/base.py +++ b/tests/base.py @@ -56,10 +56,12 @@ def engines_only(*engine_names: str): """ Test decorator. Choose what engines can run a test. - Example + For example:: + @engines_only('cockroach', 'postgres') def test_unknown_column_type(...): self.assertTrue(...) + """ if ENGINE: current_engine_name = ENGINE.engine_type @@ -85,10 +87,12 @@ def engines_skip(*engine_names: str): """ Test decorator. Choose what engines can run a test. - Example + For example:: + @engines_skip('cockroach', 'postgres') def test_unknown_column_type(...): self.assertTrue(...) + """ if ENGINE: current_engine_name = ENGINE.engine_type diff --git a/tests/columns/foreign_key/test_schema.py b/tests/columns/foreign_key/test_schema.py new file mode 100644 index 000000000..121e32ebd --- /dev/null +++ b/tests/columns/foreign_key/test_schema.py @@ -0,0 +1,103 @@ +import datetime +from unittest import TestCase + +from piccolo.columns import Date, ForeignKey, Varchar +from piccolo.schema import SchemaManager +from piccolo.table import Table, create_db_tables_sync +from tests.base import engines_only + + +class Manager(Table, schema="schema_1"): + name = Varchar(length=50) + + +class Band(Table, schema="schema_1"): + name = Varchar(length=50) + manager = ForeignKey(Manager) + + +class Concert(Table, schema="schema_1"): + start_date = Date() + band = ForeignKey(Band) + + +TABLES = [Band, Manager, Concert] + + +@engines_only("postgres", "cockroach") +class TestForeignKeyWithSchema(TestCase): + """ + Make sure that foreign keys work with Postgres schemas. + """ + + schema_manager = SchemaManager() + schema_name = "schema_1" + + def setUp(self) -> None: + self.schema_manager.create_schema( + schema_name=self.schema_name + ).run_sync() + create_db_tables_sync(*TABLES) + + def tearDown(self) -> None: + self.schema_manager.drop_schema( + schema_name=self.schema_name, if_exists=True, cascade=True + ).run_sync() + + def test_with_schema(self): + """ + Make sure that foreign keys work with schemas. + """ + manager = Manager({Manager.name: "Guido"}) + manager.save().run_sync() + + band = Band({Band.manager: manager, Band.name: "Pythonistas"}) + band.save().run_sync() + + concert = Concert( + { + Concert.band: band, + Concert.start_date: datetime.date(year=2023, month=1, day=1), + } + ) + concert.save().run_sync() + + ####################################################################### + # Test single level join. + + query = Band.select( + Band.name, + Band.manager.name.as_alias("manager_name"), + ) + self.assertIn('"schema_1"."band"', query.__str__()) + self.assertIn('"schema_1"."manager"', query.__str__()) + + response = query.run_sync() + self.assertListEqual( + response, + [{"name": "Pythonistas", "manager_name": "Guido"}], + ) + + ####################################################################### + # Test two level join. + + query = Concert.select( + Concert.start_date, + Concert.band.name.as_alias("band_name"), + Concert.band.manager.name.as_alias("manager_name"), + ) + self.assertIn('"schema_1"."concert"', query.__str__()) + self.assertIn('"schema_1"."band"', query.__str__()) + self.assertIn('"schema_1"."manager"', query.__str__()) + + response = query.run_sync() + self.assertListEqual( + response, + [ + { + "start_date": datetime.date(2023, 1, 1), + "band_name": "Pythonistas", + "manager_name": "Guido", + } + ], + ) diff --git a/tests/conf/test_apps.py b/tests/conf/test_apps.py index 42a10820c..6df249e37 100644 --- a/tests/conf/test_apps.py +++ b/tests/conf/test_apps.py @@ -221,38 +221,44 @@ def test_get_table_classes(self): """ finder = Finder() - self.assertEqual( - finder.get_table_classes(), + self.assertListEqual( + sorted(finder.get_table_classes(), key=lambda i: i.__name__), [ - Manager, Band, - Venue, Concert, - Ticket, + Manager, + MegaTable, Poster, - Shirt, RecordingStudio, - MegaTable, + Shirt, SmallTable, + Ticket, + Venue, ], ) - self.assertEqual( - finder.get_table_classes(include_apps=["music"]), + self.assertListEqual( + sorted( + finder.get_table_classes(include_apps=["music"]), + key=lambda i: i.__name__, + ), [ - Manager, Band, - Venue, Concert, - Ticket, + Manager, Poster, - Shirt, RecordingStudio, + Shirt, + Ticket, + Venue, ], ) - self.assertEqual( - finder.get_table_classes(exclude_apps=["music"]), + self.assertListEqual( + sorted( + finder.get_table_classes(exclude_apps=["music"]), + key=lambda i: i.__name__, + ), [ MegaTable, SmallTable, diff --git a/tests/example_apps/music/piccolo_app.py b/tests/example_apps/music/piccolo_app.py index f37f92f18..cb473faf7 100644 --- a/tests/example_apps/music/piccolo_app.py +++ b/tests/example_apps/music/piccolo_app.py @@ -1,33 +1,13 @@ import os -from piccolo.conf.apps import AppConfig - -from .tables import ( - Band, - Concert, - Manager, - Poster, - RecordingStudio, - Shirt, - Ticket, - Venue, -) +from piccolo.conf.apps import AppConfig, table_finder CURRENT_DIRECTORY = os.path.dirname(os.path.abspath(__file__)) APP_CONFIG = AppConfig( app_name="music", - table_classes=[ - Manager, - Band, - Venue, - Concert, - Ticket, - Poster, - Shirt, - RecordingStudio, - ], + table_classes=table_finder(["tests.example_apps.music.tables"]), migrations_folder_path=os.path.join( CURRENT_DIRECTORY, "piccolo_migrations" ), diff --git a/tests/table/test_alter.py b/tests/table/test_alter.py index 6a1e55eb0..e8df1b004 100644 --- a/tests/table/test_alter.py +++ b/tests/table/test_alter.py @@ -8,6 +8,7 @@ from piccolo.columns import BigInt, Integer, Numeric, Varchar from piccolo.columns.base import Column from piccolo.columns.column_types import ForeignKey, Text +from piccolo.schema import SchemaManager from piccolo.table import Table from tests.base import ( DBTestCase, @@ -323,6 +324,65 @@ def test_set_default(self): self.assertEqual(manager.name, "Pending") +@engines_only("postgres", "cockroach") +class TestSetSchema(TestCase): + + schema_manager = SchemaManager() + schema_name = "schema_1" + + def setUp(self): + Manager.create_table().run_sync() + self.schema_manager.create_schema( + schema_name=self.schema_name + ).run_sync() + + def tearDown(self): + Manager.alter().drop_table(if_exists=True).run_sync() + self.schema_manager.drop_schema( + schema_name=self.schema_name, cascade=True + ).run_sync() + + def test_set_schema(self): + Manager.alter().set_schema(schema_name=self.schema_name).run_sync() + + self.assertIn( + Manager._meta.tablename, + self.schema_manager.list_tables( + schema_name=self.schema_name + ).run_sync(), + ) + + +@engines_only("postgres", "cockroach") +class TestDropTable(TestCase): + class Manager(Table, schema="schema_1"): + pass + + schema_manager = SchemaManager() + + def tearDown(self): + self.schema_manager.drop_schema( + schema_name="schema_1", if_exists=True, cascade=True + ).run_sync() + + def test_drop_table_with_schema(self): + Manager = self.Manager + + Manager.create_table().run_sync() + + self.assertIn( + "manager", + self.schema_manager.list_tables(schema_name="schema_1").run_sync(), + ) + + Manager.alter().drop_table().run_sync() + + self.assertNotIn( + "manager", + self.schema_manager.list_tables(schema_name="schema_1").run_sync(), + ) + + ############################################################################### diff --git a/tests/table/test_create.py b/tests/table/test_create.py index 6f95fa381..7dd936e59 100644 --- a/tests/table/test_create.py +++ b/tests/table/test_create.py @@ -1,7 +1,9 @@ from unittest import TestCase from piccolo.columns import Varchar +from piccolo.schema import SchemaManager from piccolo.table import Table +from tests.base import engines_only from tests.example_apps.music.tables import Manager @@ -44,3 +46,52 @@ def test_create_if_not_exists_with_indexes(self): query.ddl[0].__str__().startswith("CREATE TABLE IF NOT EXISTS"), query.ddl[1].__str__().startswith("CREATE INDEX IF NOT EXISTS"), ) + + +@engines_only("postgres", "cockroach") +class TestCreateWithSchema(TestCase): + + manager = SchemaManager() + + class Band(Table, tablename="band", schema="schema_1"): + name = Varchar(length=50, index=True) + + def tearDown(self) -> None: + self.manager.drop_schema( + schema_name="schema_1", cascade=True + ).run_sync() + + def test_table_created(self): + """ + Make sure that tables can be created in specific schemas. + """ + Band = self.Band + Band.create_table().run_sync() + + self.assertIn( + "band", + self.manager.list_tables(schema_name="schema_1").run_sync(), + ) + + +@engines_only("postgres", "cockroach") +class TestCreateWithPublicSchema(TestCase): + class Band(Table, tablename="band", schema="public"): + name = Varchar(length=50, index=True) + + def tearDown(self) -> None: + self.Band.alter().drop_table(if_exists=True).run_sync() + + def test_table_created(self): + """ + Make sure that if the schema is explicitly set a 'public' rather than + ``None``, that we don't try creating the schema which would cause it + to fail. + """ + Band = self.Band + Band.create_table().run_sync() + + self.assertIn( + "band", + SchemaManager().list_tables(schema_name="public").run_sync(), + ) diff --git a/tests/table/test_drop_db_tables.py b/tests/table/test_drop_db_tables.py index 320b374dd..bfbf85890 100644 --- a/tests/table/test_drop_db_tables.py +++ b/tests/table/test_drop_db_tables.py @@ -16,22 +16,22 @@ def test_drop_db_tables(self): """ Make sure the tables are dropped. """ - self.assertEqual(Manager.table_exists().run_sync(), True) - self.assertEqual(Band.table_exists().run_sync(), True) + self.assertTrue(Manager.table_exists().run_sync()) + self.assertTrue(Band.table_exists().run_sync()) drop_db_tables_sync(Manager, Band) - self.assertEqual(Manager.table_exists().run_sync(), False) - self.assertEqual(Band.table_exists().run_sync(), False) + self.assertFalse(Manager.table_exists().run_sync()) + self.assertFalse(Band.table_exists().run_sync()) def test_drop_tables(self): """ This is a deprecated function, which just acts as a proxy. """ - self.assertEqual(Manager.table_exists().run_sync(), True) - self.assertEqual(Band.table_exists().run_sync(), True) + self.assertTrue(Manager.table_exists().run_sync()) + self.assertTrue(Band.table_exists().run_sync()) drop_tables(Manager, Band) - self.assertEqual(Manager.table_exists().run_sync(), False) - self.assertEqual(Band.table_exists().run_sync(), False) + self.assertFalse(Manager.table_exists().run_sync()) + self.assertFalse(Band.table_exists().run_sync()) diff --git a/tests/table/test_metaclass.py b/tests/table/test_metaclass.py index a89bd005d..9b884f575 100644 --- a/tests/table/test_metaclass.py +++ b/tests/table/test_metaclass.py @@ -1,4 +1,5 @@ from unittest import TestCase +from unittest.mock import MagicMock, patch from piccolo.columns import Secret from piccolo.columns.column_types import ( @@ -42,6 +43,38 @@ class Manager(Table, help_text=help_text): self.assertEqual(Manager._meta.help_text, help_text) + def test_schema(self): + """ + Make sure schema can be set for the Table. + """ + schema = "schema_1" + + class Manager(Table, schema=schema): + pass + + self.assertEqual(Manager._meta.schema, schema) + + @patch("piccolo.table.warnings") + def test_schema_from_tablename(self, warnings: MagicMock): + """ + If the tablename contains a '.' we extract the schema name. + """ + table = "manager" + schema = "schema_1" + + tablename = f"{schema}.{table}" + + class Manager(Table, tablename=tablename): + pass + + self.assertEqual(Manager._meta.schema, schema) + self.assertEqual(Manager._meta.tablename, table) + + warnings.warn.assert_called_once_with( + "There's a '.' in the tablename - please use the `schema` " + "argument instead." + ) + def test_foreign_key_columns(self): """ Make sure TableMeta.foreign_keys and TableMeta.foreign_key_references diff --git a/tests/table/test_raw.py b/tests/table/test_raw.py index f49f1c9a6..09abb9de8 100644 --- a/tests/table/test_raw.py +++ b/tests/table/test_raw.py @@ -6,7 +6,7 @@ class TestRaw(DBTestCase): def test_raw_without_args(self): self.insert_row() - response = Band.raw("select * from band").run_sync() + response = Band.raw("SELECT * FROM band").run_sync() if engine_is("cockroach"): self.assertDictEqual( @@ -33,7 +33,7 @@ def test_raw_with_args(self): self.insert_rows() response = Band.raw( - "select * from band where name = {}", "Pythonistas" + "SELECT * FROM band WHERE name = {}", "Pythonistas" ).run_sync() self.assertEqual(len(response), 1) diff --git a/tests/table/test_table_exists.py b/tests/table/test_table_exists.py index 8cdd18b76..6b31afa00 100644 --- a/tests/table/test_table_exists.py +++ b/tests/table/test_table_exists.py @@ -1,5 +1,9 @@ from unittest import TestCase +from piccolo.columns import Varchar +from piccolo.schema import SchemaManager +from piccolo.table import Table +from tests.base import engines_skip from tests.example_apps.music.tables import Manager @@ -7,9 +11,31 @@ class TestTableExists(TestCase): def setUp(self): Manager.create_table().run_sync() + def tearDown(self): + Manager.alter().drop_table().run_sync() + def test_table_exists(self): response = Manager.table_exists().run_sync() self.assertTrue(response) + +class Band(Table, schema="schema_1"): + name = Varchar() + + +@engines_skip("sqlite") +class TestTableExistsSchema(TestCase): + def setUp(self): + Band.create_table(auto_create_schema=True).run_sync() + def tearDown(self): - Manager.alter().drop_table().run_sync() + SchemaManager().drop_schema( + "schema_1", if_exists=True, cascade=True + ).run_sync() + + def test_table_exists(self): + """ + Make sure it works correctly if the table is in a Postgres schema. + """ + response = Band.table_exists().run_sync() + self.assertTrue(response) diff --git a/tests/test_schema.py b/tests/test_schema.py new file mode 100644 index 000000000..29a4652db --- /dev/null +++ b/tests/test_schema.py @@ -0,0 +1,184 @@ +from unittest import TestCase + +from piccolo.schema import SchemaManager +from piccolo.table import Table +from tests.base import engines_skip + + +class Band(Table, schema="schema_1"): + pass + + +@engines_skip("sqlite") +class TestListTables(TestCase): + def setUp(self): + Band.create_table().run_sync() + + def tearDown(self): + Band.alter().drop_table().run_sync() + + def test_list_tables(self): + """ + Make sure we can list all the tables in a schema. + """ + table_list = ( + SchemaManager() + .list_tables(schema_name=Band._meta.schema) + .run_sync() + ) + self.assertListEqual(table_list, [Band._meta.tablename]) + + +@engines_skip("sqlite") +class TestCreateAndDrop(TestCase): + def test_create_and_drop(self): + """ + Make sure a schema can be created, and dropped. + """ + manager = SchemaManager() + + # Make sure schema names with spaces, and clashing with keywords work. + for schema_name in ("test_schema", "test schema", "user"): + manager.create_schema(schema_name=schema_name).run_sync() + + self.assertIn(schema_name, manager.list_schemas().run_sync()) + + manager.drop_schema(schema_name=schema_name).run_sync() + self.assertNotIn(schema_name, manager.list_schemas().run_sync()) + + +@engines_skip("sqlite") +class TestMoveTable(TestCase): + + new_schema = "schema_2" + + def setUp(self): + Band.create_table(if_not_exists=True).run_sync() + SchemaManager().create_schema( + self.new_schema, if_not_exists=True + ).run_sync() + + def tearDown(self): + Band.alter().drop_table(if_exists=True).run_sync() + SchemaManager().drop_schema( + self.new_schema, if_exists=True, cascade=True + ).run_sync() + + def test_move_table(self): + """ + Make sure we can move a table to a different schema. + """ + manager = SchemaManager() + + manager.move_table( + table_name=Band._meta.tablename, + new_schema=self.new_schema, + current_schema=Band._meta.schema, + ).run_sync() + + self.assertIn( + Band._meta.tablename, + manager.list_tables(schema_name=self.new_schema).run_sync(), + ) + + self.assertNotIn( + Band._meta.tablename, + manager.list_tables(schema_name="schema_1").run_sync(), + ) + + +@engines_skip("sqlite") +class TestRenameSchema(TestCase): + + manager = SchemaManager() + schema_name = "test_schema" + new_schema_name = "test_schema_2" + + def tearDown(self): + for schema_name in (self.schema_name, self.new_schema_name): + self.manager.drop_schema( + schema_name=schema_name, if_exists=True + ).run_sync() + + def test_rename_schema(self): + """ + Make sure we can rename a schema. + """ + self.manager.create_schema( + schema_name=self.schema_name, if_not_exists=True + ).run_sync() + + self.manager.rename_schema( + schema_name=self.schema_name, new_schema_name=self.new_schema_name + ).run_sync() + + self.assertIn( + self.new_schema_name, self.manager.list_schemas().run_sync() + ) + + +@engines_skip("sqlite") +class TestDDL(TestCase): + manager = SchemaManager() + + def test_create_schema(self): + self.assertEqual( + self.manager.create_schema( + schema_name="schema_1", if_not_exists=False + ).ddl, + 'CREATE SCHEMA "schema_1"', + ) + + self.assertEqual( + self.manager.create_schema( + schema_name="schema_1", if_not_exists=True + ).ddl, + 'CREATE SCHEMA IF NOT EXISTS "schema_1"', + ) + + def test_drop_schema(self): + self.assertEqual( + self.manager.drop_schema( + schema_name="schema_1", if_exists=False + ).ddl, + 'DROP SCHEMA "schema_1"', + ) + + self.assertEqual( + self.manager.drop_schema( + schema_name="schema_1", if_exists=True + ).ddl, + 'DROP SCHEMA IF EXISTS "schema_1"', + ) + + self.assertEqual( + self.manager.drop_schema( + schema_name="schema_1", if_exists=True, cascade=True + ).ddl, + 'DROP SCHEMA IF EXISTS "schema_1" CASCADE', + ) + + self.assertEqual( + self.manager.drop_schema( + schema_name="schema_1", if_exists=False, cascade=True + ).ddl, + 'DROP SCHEMA "schema_1" CASCADE', + ) + + def test_move_table(self): + self.assertEqual( + self.manager.move_table( + table_name="band", + new_schema="schema_2", + current_schema="schema_1", + ).ddl, + 'ALTER TABLE "schema_1"."band" SET SCHEMA "schema_2"', + ) + + self.assertEqual( + self.manager.move_table( + table_name="band", + new_schema="schema_2", + ).ddl, + 'ALTER TABLE "band" SET SCHEMA "schema_2"', + ) From ae1f09a11a664a1a712cbdbea52c1f32e0d940d4 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 24 May 2023 14:22:03 +0100 Subject: [PATCH 472/727] Schema enhancements (#830) * fix bug with creating a new table in a schema * more tests * remove repetition * remove print statement --- .../apps/migrations/auto/migration_manager.py | 5 +- .../auto/integration/test_migrations.py | 105 ++++++++++++++++-- 2 files changed, 100 insertions(+), 10 deletions(-) diff --git a/piccolo/apps/migrations/auto/migration_manager.py b/piccolo/apps/migrations/auto/migration_manager.py index 2bfde6ff5..acc739650 100644 --- a/piccolo/apps/migrations/auto/migration_manager.py +++ b/piccolo/apps/migrations/auto/migration_manager.py @@ -710,7 +710,10 @@ async def _run_add_tables(self, backwards=False): ] = self.add_columns.for_table_class_name(add_table.class_name) _Table: t.Type[Table] = create_table_class( class_name=add_table.class_name, - class_kwargs={"tablename": add_table.tablename}, + class_kwargs={ + "tablename": add_table.tablename, + "schema": add_table.schema, + }, class_members={ add_column.column._meta.name: add_column.column for add_column in add_columns diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index 2c0925a22..a4c02850d 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -1079,33 +1079,65 @@ class TestSchemas(MigrationTestCase): def setUp(self) -> None: self.schema_manager = SchemaManager() + self.manager_1 = create_table_class(class_name="Manager") + self.manager_2 = create_table_class( + class_name="Manager", class_kwargs={"schema": self.new_schema} + ) def tearDown(self) -> None: self.schema_manager.drop_schema( self.new_schema, if_exists=True, cascade=True ).run_sync() + Migration.alter().drop_table(if_exists=True).run_sync() - def test_schemas(self): + self.manager_1.alter().drop_table(if_exists=True).run_sync() + + def test_create_table_in_schema(self): """ - Make sure migrations still work when a foreign key references a column - other than the primary key. + Make sure we can create a new table in a schema. """ - manager_1 = create_table_class(class_name="Manager") - manager_2 = create_table_class( - class_name="Manager", class_kwargs={"schema": self.new_schema} + self._test_migrations(table_snapshots=[[self.manager_2]]) + + # The schema should automaticaly be created. + self.assertIn( + self.new_schema, + self.schema_manager.list_schemas().run_sync(), ) + # Make sure that the table is in the new schema. + self.assertListEqual( + self.schema_manager.list_tables( + schema_name=self.new_schema + ).run_sync(), + ["manager"], + ) + + # Roll it backwards to make sure the table no longer exists. + self._run_backwards(migration_id="1") + + # Make sure that the table is in the new schema. + self.assertNotIn( + "manager", + self.schema_manager.list_tables( + schema_name=self.new_schema + ).run_sync(), + ) + + def test_move_schemas(self): + """ + Make sure the auto migrations detect that a table's schema has changed. + """ self._test_migrations( table_snapshots=[ - [manager_1], - [manager_2], + [self.manager_1], + [self.manager_2], ], ) # The schema should automaticaly be created. self.assertIn( - manager_2._meta.schema, + self.new_schema, self.schema_manager.list_schemas().run_sync(), ) @@ -1134,3 +1166,58 @@ def test_schemas(self): self.new_schema, self.schema_manager.list_schemas().run_sync(), ) + + +@engines_only("postgres", "cockroach") +class TestSameTableName(MigrationTestCase): + """ + Tables with the same name are allowed in multiple schemas. + """ + + new_schema = "schema_1" + tablename = "manager" + + def setUp(self) -> None: + self.schema_manager = SchemaManager() + + self.manager_1 = create_table_class( + class_name="Manager1", class_kwargs={"tablename": self.tablename} + ) + + self.manager_2 = create_table_class( + class_name="Manager2", + class_kwargs={"tablename": self.tablename, "schema": "schema_1"}, + ) + + def tearDown(self) -> None: + self.schema_manager.drop_schema( + self.new_schema, if_exists=True, cascade=True + ).run_sync() + + self.manager_1.alter().drop_table(if_exists=True).run_sync() + + Migration.alter().drop_table(if_exists=True).run_sync() + + def test_schemas(self): + """ + Make sure we can create a table with the same name in multiple schemas. + """ + + self._test_migrations( + table_snapshots=[ + [self.manager_1], + [self.manager_1, self.manager_2], + ], + ) + + # Make sure that both tables exist (in the correct schemas): + self.assertIn( + "manager", + self.schema_manager.list_tables(schema_name="public").run_sync(), + ) + self.assertIn( + "manager", + self.schema_manager.list_tables( + schema_name=self.new_schema + ).run_sync(), + ) From b81e12b49280acd997d5d512f11f299e19c93024 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 24 May 2023 14:23:49 +0100 Subject: [PATCH 473/727] bumped version --- CHANGES.rst | 32 ++++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 750e1b0ae..2af477653 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,38 @@ Changes ======= +0.112.0 +------- + +Added support for schemas in Postgres and CockroachDB. + +For example: + +.. code-block:: python + + class Band(Table, schema="music"): + ... + +When creating the table, the schema will be created automatically if it doesn't +already exist. + +.. code-block:: python + + await Band.create_table() + +It also works with migrations. If we change the ``schema`` value for the table, +Piccolo will detect this, and create a migration for moving it to the new schema. + +.. code-block:: python + + class Band(Table, schema="music_2"): + ... + + # Piccolo will detect that the table needs to be moved to a new schema. + >>> piccolo migrations new my_app --auto + +------------------------------------------------------------------------------- + 0.111.1 ------- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index c84038cb9..6e33b1e07 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.111.1" +__VERSION__ = "0.112.0" From f966501cc204f487c2dd2d653400a2c83c5b4202 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 24 May 2023 15:49:35 +0100 Subject: [PATCH 474/727] fix serialised tables which are in a specific schema (#831) * fix serialised tables which are in a specific schema * remove print statement --- piccolo/apps/migrations/auto/serialisation.py | 8 +++- .../auto/integration/test_migrations.py | 45 +++++++++++++++++++ .../migrations/auto/test_serialisation.py | 31 ++++--------- 3 files changed, 61 insertions(+), 23 deletions(-) diff --git a/piccolo/apps/migrations/auto/serialisation.py b/piccolo/apps/migrations/auto/serialisation.py index 2b45dfba7..320cf74f7 100644 --- a/piccolo/apps/migrations/auto/serialisation.py +++ b/piccolo/apps/migrations/auto/serialisation.py @@ -392,9 +392,15 @@ def __repr__(self): ####################################################################### + schema_str = ( + "None" + if self.table_type._meta.schema is None + else f'"{self.table_type._meta.schema}"' + ) + definition = ( f"class {self.table_class_name}" - f'({UniqueGlobalNames.TABLE}, tablename="{tablename}"): ' + f'({UniqueGlobalNames.TABLE}, tablename="{tablename}", schema={schema_str}): ' # noqa: E501 f"{pk_column_name} = {serialised_pk_column}" ) diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index a4c02850d..d30e5a801 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -1071,6 +1071,7 @@ def test_target_column(self): ############################################################################### +# Testing migrations which involve schemas. @engines_only("postgres", "cockroach") @@ -1221,3 +1222,47 @@ def test_schemas(self): schema_name=self.new_schema ).run_sync(), ) + + +@engines_only("postgres", "cockroach") +class TestForeignKey(MigrationTestCase): + """ + Make sure that migrations with foreign keys involving schemas work + correctly. + """ + + schema = "schema_1" + schema_manager = SchemaManager() + + def setUp(self) -> None: + self.manager = create_table_class( + class_name="Manager", class_kwargs={"schema": self.schema} + ) + + self.band = create_table_class( + class_name="Band", + class_kwargs={"schema": self.schema}, + class_members={"manager": ForeignKey(self.manager)}, + ) + + def tearDown(self) -> None: + self.schema_manager.drop_schema( + self.schema, if_exists=True, cascade=True + ).run_sync() + + Migration.alter().drop_table(if_exists=True).run_sync() + + def test_foreign_key(self): + self._test_migrations( + table_snapshots=[ + [self.manager, self.band], + ], + ) + + tables_in_schema = self.schema_manager.list_tables( + schema_name=self.schema + ).run_sync() + + # Make sure that both tables exist (in the correct schemas): + for tablename in ("manager", "band"): + self.assertIn(tablename, tables_in_schema) diff --git a/tests/apps/migrations/auto/test_serialisation.py b/tests/apps/migrations/auto/test_serialisation.py index 688bdf09e..2920b72c9 100644 --- a/tests/apps/migrations/auto/test_serialisation.py +++ b/tests/apps/migrations/auto/test_serialisation.py @@ -19,7 +19,6 @@ from piccolo.columns.column_types import Varchar from piccolo.columns.defaults import UUID4, DateNow, TimeNow, TimestampNow from piccolo.columns.reference import LazyTableReference -from tests.base import engine_is class TestUniqueGlobalNamesMeta: @@ -243,27 +242,15 @@ def test_lazy_table_reference(self): self.assertTrue(len(serialised.extra_definitions) == 1) - if engine_is("postgres"): - self.assertEqual( - serialised.extra_definitions[0].__str__(), - ( - 'class Manager(Table, tablename="manager"): ' - "id = Serial(null=False, primary_key=True, unique=False, " # noqa: E501 - "index=False, index_method=IndexMethod.btree, " - "choices=None, db_column_name='id', secret=False)" - ), - ) - - if engine_is("cockroach"): - self.assertEqual( - serialised.extra_definitions[0].__str__(), - ( - 'class Manager(Table, tablename="manager"): ' - "id = Serial(null=False, primary_key=True, unique=False, " # noqa: E501 - "index=False, index_method=IndexMethod.btree, " - "choices=None, db_column_name='id', secret=False)" - ), - ) + self.assertEqual( + serialised.extra_definitions[0].__str__(), + ( + 'class Manager(Table, tablename="manager", schema=None): ' + "id = Serial(null=False, primary_key=True, unique=False, " + "index=False, index_method=IndexMethod.btree, " + "choices=None, db_column_name='id', secret=False)" + ), + ) def test_function(self): serialised = serialise_params(params={"default": example_function}) From cf276a8b18f229cf62023bed9f22c1524f8c63b6 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 24 May 2023 15:51:09 +0100 Subject: [PATCH 475/727] bumped version --- CHANGES.rst | 7 +++++++ piccolo/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2af477653..853390ff6 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,13 @@ Changes ======= +0.112.1 +------- + +Fixed a bug with serialising table classes in migrations. + +------------------------------------------------------------------------------- + 0.112.0 ------- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 6e33b1e07..b4cc7210d 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.112.0" +__VERSION__ = "0.112.1" From 5c2b3dbdc1a3b54ef4fcf31f1521c1cf4b24d56a Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 28 May 2023 09:18:43 +0100 Subject: [PATCH 476/727] fix type annotation for `MigrationManager.add_table` (#833) --- piccolo/apps/migrations/auto/migration_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piccolo/apps/migrations/auto/migration_manager.py b/piccolo/apps/migrations/auto/migration_manager.py index acc739650..2b9d9a2db 100644 --- a/piccolo/apps/migrations/auto/migration_manager.py +++ b/piccolo/apps/migrations/auto/migration_manager.py @@ -160,7 +160,7 @@ def add_table( self, class_name: str, tablename: str, - schema: str = None, + schema: t.Optional[str] = None, columns: t.Optional[t.List[Column]] = None, ): if not columns: From b9f7ba06637f16f33831b31b32bcaaa33207d702 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 28 May 2023 17:55:19 +0100 Subject: [PATCH 477/727] Improved UX for renaming tables (#835) * improved UX for renaming tables * remove unused function Was left over from the schema PR. * add tests for `RenameTableCollection` and `RenameColumnCollection` * add integration test for renaming tables --- piccolo/apps/migrations/auto/schema_differ.py | 26 ++++-- .../auto/integration/test_migrations.py | 85 +++++++++++++++++-- .../migrations/auto/test_schema_differ.py | 60 ++++++++++++- 3 files changed, 157 insertions(+), 14 deletions(-) diff --git a/piccolo/apps/migrations/auto/schema_differ.py b/piccolo/apps/migrations/auto/schema_differ.py index 5ea83ab6e..f2a01ee3b 100644 --- a/piccolo/apps/migrations/auto/schema_differ.py +++ b/piccolo/apps/migrations/auto/schema_differ.py @@ -39,6 +39,15 @@ def old_class_names(self): def new_class_names(self): return [i.new_class_name for i in self.rename_tables] + def was_renamed_from(self, old_class_name: str) -> bool: + """ + Returns ``True`` if the given class name was renamed. + """ + for rename_table in self.rename_tables: + if rename_table.old_class_name == old_class_name: + return True + return False + def renamed_from(self, new_class_name: str) -> t.Optional[str]: """ Returns the old class name, if it exists. @@ -95,10 +104,6 @@ def extend(self, alter_statements: AlterStatements): return self -def optional_str_repr(value: t.Optional[str]) -> str: - return f"'{value}'" if value else "None" - - @dataclass class SchemaDiffer: """ @@ -156,6 +161,16 @@ def check_rename_tables(self) -> RenameTableCollection: i._meta.db_column_name for i in new_table.columns ] for drop_table in drop_tables: + if collection.was_renamed_from( + old_class_name=drop_table.class_name + ): + # We've already detected a table that was renamed from + # this, so we can continue. + # This can happen if we're renaming lots of tables in a + # single migration. + # https://github.com/piccolo-orm/piccolo/discussions/832 + continue + drop_column_names = [ i._meta.db_column_name for i in new_table.columns ] @@ -178,7 +193,7 @@ def check_rename_tables(self) -> RenameTableCollection: new_tablename=new_table.tablename, ) ) - continue + break user_response = ( self.auto_input @@ -199,6 +214,7 @@ def check_rename_tables(self) -> RenameTableCollection: new_tablename=new_table.tablename, ) ) + break return collection diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index d30e5a801..2a3182457 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -11,6 +11,7 @@ import uuid from unittest.mock import MagicMock, patch +from piccolo.apps.migrations.auto.operations import RenameTable from piccolo.apps.migrations.commands.backwards import ( BackwardsMigrationManager, ) @@ -117,6 +118,13 @@ def _get_migrations_folder_path(self) -> str: ) return migrations_folder_path + def _get_app_config(self) -> AppConfig: + return AppConfig( + app_name="test_app", + migrations_folder_path=self._get_migrations_folder_path(), + table_classes=[], + ) + def _test_migrations( self, table_snapshots: t.List[t.List[t.Type[Table]]], @@ -135,19 +143,15 @@ def _test_migrations( test passes, otherwise ``False``. """ - migrations_folder_path = self._get_migrations_folder_path() + app_config = self._get_app_config() + + migrations_folder_path = app_config.migrations_folder_path if os.path.exists(migrations_folder_path): shutil.rmtree(migrations_folder_path) _create_migrations_folder(migrations_folder_path) - app_config = AppConfig( - app_name="test_app", - migrations_folder_path=migrations_folder_path, - table_classes=[], - ) - for table_snapshot in table_snapshots: app_config.table_classes = table_snapshot meta = run_sync( @@ -178,6 +182,15 @@ def _test_migrations( msg=f"Meta is incorrect: {row_meta}", ) + def _get_migration_managers(self): + app_config = self._get_app_config() + + return run_sync( + ForwardsMigrationManager( + app_name=app_config.app_name + ).get_migration_managers(app_config=app_config) + ) + def _run_backwards(self, migration_id: str): """ After running :meth:`_test_migrations`, if you call `_run_backwards` @@ -1225,7 +1238,7 @@ def test_schemas(self): @engines_only("postgres", "cockroach") -class TestForeignKey(MigrationTestCase): +class TestForeignKeyWithSchema(MigrationTestCase): """ Make sure that migrations with foreign keys involving schemas work correctly. @@ -1266,3 +1279,59 @@ def test_foreign_key(self): # Make sure that both tables exist (in the correct schemas): for tablename in ("manager", "band"): self.assertIn(tablename, tables_in_schema) + + +############################################################################### + + +@engines_only("postgres", "cockroach") +class TestRenameTable(MigrationTestCase): + """ + Make sure that tables can be renamed. + """ + + schema_manager = SchemaManager() + manager = create_table_class( + class_name="Manager", class_members={"name": Varchar()} + ) + manager_1 = create_table_class( + class_name="Manager", + class_kwargs={"tablename": "manager_1"}, + class_members={"name": Varchar()}, + ) + + def setUp(self) -> None: + pass + + def tearDown(self) -> None: + drop_db_tables_sync(self.manager, self.manager_1, Migration) + + def test_rename_table(self): + self._test_migrations( + table_snapshots=[ + [self.manager], + [self.manager_1], + ], + ) + + tables = self.schema_manager.list_tables( + schema_name="public" + ).run_sync() + + self.assertIn("manager_1", tables) + self.assertNotIn("manager", tables) + + # Make sure the table was renamed, and not dropped and recreated. + migration_managers = self._get_migration_managers() + + self.assertListEqual( + migration_managers[-1].rename_tables, + [ + RenameTable( + old_class_name="Manager", + old_tablename="manager", + new_class_name="Manager", + new_tablename="manager_1", + ) + ], + ) diff --git a/tests/apps/migrations/auto/test_schema_differ.py b/tests/apps/migrations/auto/test_schema_differ.py index 63ff06ea5..7a5f63fb0 100644 --- a/tests/apps/migrations/auto/test_schema_differ.py +++ b/tests/apps/migrations/auto/test_schema_differ.py @@ -4,7 +4,14 @@ from unittest import TestCase from unittest.mock import MagicMock, call, patch -from piccolo.apps.migrations.auto import DiffableTable, SchemaDiffer +from piccolo.apps.migrations.auto.schema_differ import ( + DiffableTable, + RenameColumn, + RenameColumnCollection, + RenameTable, + RenameTableCollection, + SchemaDiffer, +) from piccolo.columns.column_types import Numeric, Varchar @@ -484,3 +491,54 @@ def test_db_column_name(self): def test_alter_default(self): pass + + +class TestRenameTableCollection(TestCase): + collection = RenameTableCollection( + rename_tables=[ + RenameTable( + old_class_name="Manager", + old_tablename="manager", + new_class_name="Manager1", + new_tablename="manager_1", + ) + ] + ) + + def test_was_renamed_from(self): + self.assertTrue( + self.collection.was_renamed_from(old_class_name="Manager") + ) + self.assertFalse( + self.collection.was_renamed_from(old_class_name="Band") + ) + + def test_renamed_from(self): + self.assertEqual( + self.collection.renamed_from(new_class_name="Manager1"), "Manager" + ) + self.assertIsNone( + self.collection.renamed_from(new_class_name="Band"), + ) + + +class TestRenameColumnCollection(TestCase): + def test_for_table_class_name(self): + rename_column = RenameColumn( + table_class_name="Manager", + tablename="manager", + old_column_name="name", + new_column_name="full_name", + old_db_column_name="name", + new_db_column_name="full_name", + ) + + collection = RenameColumnCollection(rename_columns=[rename_column]) + + self.assertListEqual( + collection.for_table_class_name(table_class_name="Manager"), + [rename_column], + ) + self.assertListEqual( + collection.for_table_class_name(table_class_name="Band"), [] + ) From 87dc9de542a3c5c39b194610a1df94d27f292da2 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 28 May 2023 18:03:38 +0100 Subject: [PATCH 478/727] bumped version --- CHANGES.rst | 12 ++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 853390ff6..398ed546c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,18 @@ Changes ======= +0.113.0 +------- + +If Piccolo detects a renamed table in an auto migration, it asks the user for +confirmation. When lots of tables have been renamed, Piccolo is now more +intelligent about when to ask for confirmation. Thanks to @sumitsharansatsangi +for suggesting this change, and @sinisaos for reviewing. + +Also, fixed the type annotations for ``MigrationManager.add_table``. + +------------------------------------------------------------------------------- + 0.112.1 ------- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index b4cc7210d..597fa6367 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.112.1" +__VERSION__ = "0.113.0" From a9e0f220d6a514741f5957e04cd630f33d225634 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 3 Jun 2023 22:12:34 +0100 Subject: [PATCH 479/727] `count` query improvements - support for `distinct` and better docs (#836) * Add Count.distinct * Update tests * Fix DistinctOnError * tweaked API for count distinct * refactor to make query more efficient * started overhauling the docs for `count` queries * add some more tests * implement @sinisaos suggestions - multiple distinct columns now work with sqlite * add missing param to docstring * improve count docs within select.rst * add test for `ValueError` * improve docs --------- Co-authored-by: Manh Luong --- docs/src/piccolo/api_reference/index.rst | 13 ++ docs/src/piccolo/query_clauses/group_by.rst | 20 +-- docs/src/piccolo/query_clauses/where.rst | 1 + docs/src/piccolo/query_types/count.rst | 137 ++++++++++++++++++++ docs/src/piccolo/query_types/index.rst | 1 + docs/src/piccolo/query_types/select.rst | 17 ++- piccolo/query/methods/count.py | 33 +++-- piccolo/query/methods/select.py | 95 ++++++++++---- piccolo/table.py | 48 ++++++- tests/table/test_count.py | 76 ++++++++++- tests/table/test_objects.py | 5 +- 11 files changed, 381 insertions(+), 65 deletions(-) create mode 100644 docs/src/piccolo/query_types/count.rst diff --git a/docs/src/piccolo/api_reference/index.rst b/docs/src/piccolo/api_reference/index.rst index ce8804a2f..6159548a2 100644 --- a/docs/src/piccolo/api_reference/index.rst +++ b/docs/src/piccolo/api_reference/index.rst @@ -29,6 +29,19 @@ Column .. autoclass:: Column :members: + +------------------------------------------------------------------------------- + +Aggregate functions +------------------- + +Count +~~~~~ + +.. currentmodule:: piccolo.query.methods.select + +.. autoclass:: Count + ------------------------------------------------------------------------------- Refresh diff --git a/docs/src/piccolo/query_clauses/group_by.rst b/docs/src/piccolo/query_clauses/group_by.rst index 984a124f0..d3eb5af87 100644 --- a/docs/src/piccolo/query_clauses/group_by.rst +++ b/docs/src/piccolo/query_clauses/group_by.rst @@ -7,8 +7,8 @@ You can use ``group_by`` clauses with the following queries: * :ref:`Select` -It is used in combination with aggregate functions - ``Count`` is currently -supported. +It is used in combination with the :ref:`aggregate functions ` +- for example, ``Count``. ------------------------------------------------------------------------------- @@ -22,20 +22,20 @@ In the following query, we get a count of the number of bands per manager: >>> from piccolo.query.methods.select import Count >>> await Band.select( - ... Band.manager.name, - ... Count(Band.manager) + ... Band.manager.name.as_alias('manager_name'), + ... Count(alias='band_count') ... ).group_by( ... Band.manager ... ) [ - {"manager.name": "Graydon", "count": 1}, - {"manager.name": "Guido", "count": 1} + {"manager_name": "Graydon", "band_count": 1}, + {"manager_name": "Guido", "band_count": 1} ] -Source -~~~~~~ +------------------------------------------------------------------------------- -.. currentmodule:: piccolo.query.methods.select +Other aggregate functions +------------------------- -.. autoclass:: Count +These work the same as ``Count``. See :ref:`aggregate functions `. diff --git a/docs/src/piccolo/query_clauses/where.rst b/docs/src/piccolo/query_clauses/where.rst index 245f3b88a..7f2bd9ef8 100644 --- a/docs/src/piccolo/query_clauses/where.rst +++ b/docs/src/piccolo/query_clauses/where.rst @@ -5,6 +5,7 @@ where You can use ``where`` clauses with the following queries: +* :ref:`Count` * :ref:`Delete` * :ref:`Exists` * :ref:`Objects` diff --git a/docs/src/piccolo/query_types/count.rst b/docs/src/piccolo/query_types/count.rst new file mode 100644 index 000000000..4a60a2bd8 --- /dev/null +++ b/docs/src/piccolo/query_types/count.rst @@ -0,0 +1,137 @@ +.. _Count: + +Count +===== + +The ``count`` query makes it really easy to retrieve the number of rows in a +table: + +.. code-block:: python + + >>> await Band.count() + 3 + +It's equivalent to this ``select`` query: + +.. code-block:: python + + from piccolo.query.methods.select import Count + + >>> response = await Band.select(Count()) + >>> response[0]['count'] + 3 + +As you can see, the ``count`` query is more convenient. + +Non-null columns +---------------- + +If you want to retrieve the number of rows where a given column isn't null, we +can do so as follows: + +.. code-block:: python + + await Band.count(column=Band.name) + + # Or simply: + await Band.count(Band.name) + +Note, this is equivalent to: + +.. code-block:: python + + await Band.count().where(Band.name.is_not_null()) + +Example +~~~~~~~ + +If we have the following database table: + +.. code-block:: python + + class Band(Table): + name = Varchar() + popularity = Integer(null=True) + +With the following data: + +.. table:: + :widths: auto + + ============ ========== + name popularity + ============ ========== + Pythonistas 1000 + Rustaceans 800 + C-Sharps ``null`` + ============ ========== + +Then we get the following results: + +.. code-block:: python + + >>> await Band.count() + 3 + + >>> await Band.count(Band.popularity) + 2 + +distinct +-------- + +We can count the number of distinct (i.e. unique) rows. + +.. code-block:: python + + await Band.count(distinct=[Band.name]) + +With the following data: + +.. table:: + :widths: auto + + ============ ========== + name popularity + ============ ========== + Pythonistas 1000 + Pythonistas 1000 + Pythonistas 800 + Rustaceans 800 + ============ ========== + +Note how we have duplicate band names. + +.. hint:: + This is bad database design as we should add a unique constraint to + prevent this, but go with it for this example! + +Let's compare queries with and without ``distinct``: + +.. code-block:: python + + >>> await Band.count() + 4 + + >>> await Band.count(distinct=[Band.name]) + 2 + +We can specify multiple columns: + +.. code-block:: python + + >>> await Band.count(distinct=[Band.name, Band.popularity]) + 3 + +In the above example, this means we count rows where the combination of +``name`` and ``popularity`` is unique. + +So ``('Pythonistas', 1000)`` is a distinct value from ``('Pythonistas', 800)``, +because even though the ``name`` is the same, the ``popularity`` is different. + +Clauses +------- + +where +~~~~~ + +See :ref:`where`. diff --git a/docs/src/piccolo/query_types/index.rst b/docs/src/piccolo/query_types/index.rst index ac013acf9..19a012aa2 100644 --- a/docs/src/piccolo/query_types/index.rst +++ b/docs/src/piccolo/query_types/index.rst @@ -12,6 +12,7 @@ typical ORM. ./select ./objects + ./count ./alter ./create_table ./delete diff --git a/docs/src/piccolo/query_types/select.rst b/docs/src/piccolo/query_types/select.rst index e55a11626..1591e3580 100644 --- a/docs/src/piccolo/query_types/select.rst +++ b/docs/src/piccolo/query_types/select.rst @@ -165,18 +165,29 @@ convenient. ------------------------------------------------------------------------------- +.. _AggregateFunctions: + Aggregate functions ------------------- +.. note:: These can all be used in conjunction with the :ref:`group_by` clause. + Count ~~~~~ -Returns the number of rows which match the query: +.. hint:: You can use the :ref:`count` query as a quick way of getting + the number of rows in a table. + +Returns the number of matching rows. .. code-block:: python - >>> await Band.count().where(Band.name == 'Pythonistas') - 1 + from piccolo.query.methods.select import Count + + >> await Band.select(Count()).where(Band.popularity > 100) + [{'count': 3}] + +To find out more about the options available, see :class:`Count `. Avg ~~~ diff --git a/piccolo/query/methods/count.py b/piccolo/query/methods/count.py index 75a6dd467..a0e25243b 100644 --- a/piccolo/query/methods/count.py +++ b/piccolo/query/methods/count.py @@ -4,19 +4,29 @@ from piccolo.custom_types import Combinable from piccolo.query.base import Query -from piccolo.query.methods.select import Select +from piccolo.query.methods.select import Count as SelectCount from piccolo.query.mixins import WhereDelegate from piccolo.querystring import QueryString if t.TYPE_CHECKING: # pragma: no cover + from piccolo.columns import Column from piccolo.table import Table class Count(Query): - __slots__ = ("where_delegate",) - def __init__(self, table: t.Type[Table], **kwargs): + __slots__ = ("where_delegate", "column", "distinct") + + def __init__( + self, + table: t.Type[Table], + column: t.Optional[Column] = None, + distinct: t.Optional[t.Sequence[Column]] = None, + **kwargs, + ): super().__init__(table, **kwargs) + self.column = column + self.distinct = distinct self.where_delegate = WhereDelegate() ########################################################################### @@ -33,14 +43,15 @@ async def response_handler(self, response) -> bool: @property def default_querystrings(self) -> t.Sequence[QueryString]: - select = Select(self.table) - select.where_delegate._where = self.where_delegate._where - return [ - QueryString( - 'SELECT COUNT(*) AS "count" FROM ({}) AS "subquery"', - select.querystrings[0], - ) - ] + table: t.Type[Table] = self.table + + query = table.select( + SelectCount(column=self.column, distinct=self.distinct) + ) + + query.where_delegate._where = self.where_delegate._where + + return query.querystrings Self = t.TypeVar("Self", bound=Count) diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 5720380e5..85b2ac6b4 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -99,43 +99,84 @@ def get_select_string( class Count(Selectable): """ - Used in conjunction with the ``group_by`` clause in ``Select`` queries. + Used in ``Select`` queries, usually in conjunction with the ``group_by`` + clause:: - If a column is specified, the count is for non-null values in that - column. If no column is specified, the count is for all rows, whether - they have null values or not. + >>> await Band.select( + ... Band.manager.name.as_alias('manager_name'), + ... Count(alias='band_count') + ... ).group_by(Band.manager) + [{'manager_name': 'Guido', 'count': 1}, ...] - .. code-block:: python - - await Band.select(Band.name, Count()).group_by(Band.name) - - # We can use an alias. These two are equivalent: - - await Band.select( - Band.name, Count(alias="total") - ).group_by(Band.name) + It can also be used without the ``group_by`` clause (though you may prefer + to the :meth:`Table.count ` method instead, as + it's more convenient):: - await Band.select( - Band.name, - Count().as_alias("total") - ).group_by(Band.name) + >>> await Band.select(Count()) + [{'count': 3}] """ def __init__( - self, column: t.Optional[Column] = None, alias: str = "count" + self, + column: t.Optional[Column] = None, + distinct: t.Optional[t.Sequence[Column]] = None, + alias: str = "count", ): + """ + :param column: + If specified, the count is for non-null values in that column. + :param distinct: + If specified, the count is for distinct values in those columns. + :param alias: + The name of the value in the response:: + + # These two are equivalent: + + await Band.select( + Band.name, Count(alias="total") + ).group_by(Band.name) + + await Band.select( + Band.name, + Count().as_alias("total") + ).group_by(Band.name) + + """ + if distinct and column: + raise ValueError("Only specify `column` or `distinct`") + self.column = column + self.distinct = distinct self._alias = alias def get_select_string( self, engine_type: str, with_alias: bool = True ) -> str: - if self.column is None: - column_name = "*" + expression: str + + if self.distinct: + if engine_type == "sqlite": + # SQLite doesn't allow us to specify multiple columns, so + # instead we concatenate the values. + column_names = " || ".join( + i._meta.get_full_name(with_alias=False) + for i in self.distinct + ) + else: + column_names = ", ".join( + i._meta.get_full_name(with_alias=False) + for i in self.distinct + ) + + expression = f"DISTINCT ({column_names})" else: - column_name = self.column._meta.get_full_name(with_alias=False) - return f'COUNT({column_name}) AS "{self._alias}"' + if self.column: + expression = self.column._meta.get_full_name(with_alias=False) + else: + expression = "*" + + return f'COUNT({expression}) AS "{self._alias}"' class Max(Selectable): @@ -737,12 +778,10 @@ def default_querystrings(self) -> t.Sequence[QueryString]: query = "SELECT" distinct = self.distinct_delegate._distinct - if distinct: - if distinct.on: - distinct.validate_on(self.order_by_delegate._order_by) - - query += "{}" - args.append(distinct.querystring) + if distinct.on: + distinct.validate_on(self.order_by_delegate._order_by) + query += "{}" + args.append(distinct.querystring) query += f" {columns_str} FROM {self.table._meta.get_formatted_tablename()}" # noqa: E501 diff --git a/piccolo/table.py b/piccolo/table.py index 91cd21ce3..b0aeb7687 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -1114,16 +1114,54 @@ def objects( return Objects[TableInstance](table=cls, prefetch=prefetch) @classmethod - def count(cls) -> Count: - """ - Count the number of matching rows. + def count( + cls, + column: t.Optional[Column] = None, + distinct: t.Optional[t.Sequence[Column]] = None, + ) -> Count: - .. code-block:: python + """ + Count the number of matching rows:: await Band.count().where(Band.popularity > 1000) + :param column: + If specified, just count rows where this column isn't null. + + :param distinct: + Counts the number of distinct values for these columns. For + example, if we have a concerts table:: + + class Concert(Table): + band = Varchar() + start_date = Date() + + With this data: + + .. table:: + :widths: auto + + =========== ========== + band start_date + =========== ========== + Pythonistas 2023-01-01 + Pythonistas 2023-02-03 + Rustaceans 2023-01-01 + =========== ========== + + Without the ``distinct`` argument, we get the count of all + rows:: + + >>> await Concert.count() + 3 + + To get the number of unique concert dates:: + + >>> await Concert.count(distinct=[Concert.start_date]) + 2 + """ - return Count(table=cls) + return Count(table=cls, column=column, distinct=distinct) @classmethod def exists(cls) -> Exists: diff --git a/tests/table/test_count.py b/tests/table/test_count.py index b6eafd11d..3b777e6f7 100644 --- a/tests/table/test_count.py +++ b/tests/table/test_count.py @@ -1,11 +1,73 @@ -from tests.base import DBTestCase -from tests.example_apps.music.tables import Band +from unittest import TestCase +from piccolo.columns import Integer, Varchar +from piccolo.table import Table -class TestCount(DBTestCase): - def test_exists(self): - self.insert_rows() - response = Band.count().where(Band.name == "Pythonistas").run_sync() +class Band(Table): + name = Varchar() + popularity = Integer() - self.assertEqual(response, 1) + +class TestCount(TestCase): + def setUp(self) -> None: + Band.create_table().run_sync() + + def tearDown(self) -> None: + Band.alter().drop_table().run_sync() + + def test_count(self): + Band.insert( + Band(name="Pythonistas", popularity=10), + Band(name="Rustaceans", popularity=10), + Band(name="C-Sharps", popularity=5), + ).run_sync() + + response = Band.count().run_sync() + + self.assertEqual(response, 3) + + def test_count_where(self): + Band.insert( + Band(name="Pythonistas", popularity=10), + Band(name="Rustaceans", popularity=10), + Band(name="C-Sharps", popularity=5), + ).run_sync() + + response = Band.count().where(Band.popularity == 10).run_sync() + + self.assertEqual(response, 2) + + def test_count_distinct(self): + Band.insert( + Band(name="Pythonistas", popularity=10), + Band(name="Rustaceans", popularity=10), + Band(name="C-Sharps", popularity=5), + Band(name="Fortranists", popularity=2), + ).run_sync() + + response = Band.count(distinct=[Band.popularity]).run_sync() + + self.assertEqual(response, 3) + + def test_count_distinct_multiple(self): + Band.insert( + Band(name="Pythonistas", popularity=10), + Band(name="Pythonistas", popularity=10), + Band(name="Rustaceans", popularity=10), + Band(name="C-Sharps", popularity=5), + Band(name="Fortranists", popularity=2), + ).run_sync() + + response = Band.count(distinct=[Band.name, Band.popularity]).run_sync() + + self.assertEqual(response, 4) + + def test_value_error(self): + """ + Make sure specifying `column` and `distinct` raises an error. + """ + with self.assertRaises(ValueError): + Band.count( + column=Band.name, distinct=[Band.name, Band.popularity] + ).run_sync() diff --git a/tests/table/test_objects.py b/tests/table/test_objects.py index f8c699ef2..853de6a13 100644 --- a/tests/table/test_objects.py +++ b/tests/table/test_objects.py @@ -33,9 +33,12 @@ def test_offset_postgres(self): Postgres can do an offset without a limit clause. """ self.insert_rows() + response = Band.objects().order_by(Band.name).offset(1).run_sync() + self.assertEqual( - [i.name for i in response], ["Pythonistas", "Rustaceans"] + [i.name for i in response], + ["Pythonistas", "Rustaceans"], ) @sqlite_only From b49d0aabdf9cb1ee587fcf3f7750bf5d305beb2d Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 3 Jun 2023 22:35:57 +0100 Subject: [PATCH 480/727] bumped version --- CHANGES.rst | 37 +++++++++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 398ed546c..e998f3051 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,43 @@ Changes ======= +0.114.0 +------- + +``count`` queries can now return the number of distinct rows. For example, if +we have this table: + +.. code-block:: python + + class Concert(Table): + band = Varchar() + start_date = Date() + +With this data: + +.. table:: + :widths: auto + + =========== ========== + band start_date + =========== ========== + Pythonistas 2023-01-01 + Pythonistas 2023-02-03 + Rustaceans 2023-01-01 + =========== ========== + +We can easily get the number of unique concert dates:: + + >>> await Concert.count(distinct=[Concert.start_date]) + 2 + +Also, the docs for the ``count`` query, aggregate functions, and +``group_by`` clause were significantly improved. + +Many thanks to @lqmanh and @sinisaos for their help with this. + +------------------------------------------------------------------------------- + 0.113.0 ------- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 597fa6367..73b2b393f 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.113.0" +__VERSION__ = "0.114.0" From 59f36a298b52070517ea8ae6268cea43cd076da7 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 3 Jun 2023 22:39:59 +0100 Subject: [PATCH 481/727] add explicit code block --- CHANGES.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index e998f3051..7c8d44d9e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -26,7 +26,9 @@ With this data: Rustaceans 2023-01-01 =========== ========== -We can easily get the number of unique concert dates:: +We can easily get the number of unique concert dates: + +.. code-block:: python >>> await Concert.count(distinct=[Concert.start_date]) 2 From cd5595c08d94dcf048f30ae68812379e9bb0d992 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 3 Jun 2023 22:47:42 +0100 Subject: [PATCH 482/727] tweak changelog --- CHANGES.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 7c8d44d9e..b531947a2 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -33,6 +33,15 @@ We can easily get the number of unique concert dates: >>> await Concert.count(distinct=[Concert.start_date]) 2 +We could have just done this instead: + +.. code-block:: python + + len(await Concert.select(Concert.start_date).distinct()) + +But it's far less efficient when you have lots of rows, because all of the +distinct rows need to be returned from the database. + Also, the docs for the ``count`` query, aggregate functions, and ``group_by`` clause were significantly improved. From f0abe2ce3622c4ac3d78afc5131cf59779d2b342 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 19 Jun 2023 15:31:03 +0100 Subject: [PATCH 483/727] Fix M2M queries when in a schema (#843) * fix m2m queries when in a schema * make sure linking table also uses schema * add tests * fix moving a table from a schema to the public schema thanks to @sinisaos for reporting this * fix black error * exclude sqlite --- .../apps/migrations/auto/migration_manager.py | 2 +- piccolo/columns/m2m.py | 18 +- .../auto/integration/test_migrations.py | 36 +- tests/columns/m2m/__init__.py | 0 tests/columns/{test_m2m.py => m2m/base.py} | 481 ++---------------- tests/columns/m2m/test_m2m.py | 436 ++++++++++++++++ tests/columns/m2m/test_m2m_schema.py | 48 ++ 7 files changed, 585 insertions(+), 436 deletions(-) create mode 100644 tests/columns/m2m/__init__.py rename tests/columns/{test_m2m.py => m2m/base.py} (50%) create mode 100644 tests/columns/m2m/test_m2m.py create mode 100644 tests/columns/m2m/test_m2m_schema.py diff --git a/piccolo/apps/migrations/auto/migration_manager.py b/piccolo/apps/migrations/auto/migration_manager.py index 2b9d9a2db..ad1175d81 100644 --- a/piccolo/apps/migrations/auto/migration_manager.py +++ b/piccolo/apps/migrations/auto/migration_manager.py @@ -826,7 +826,7 @@ async def _run_change_table_schema(self, backwards=False): await self._run_query( schema_manager.move_table( table_name=change_table_schema.tablename, - new_schema=change_table_schema.new_schema, + new_schema=change_table_schema.new_schema or "public", current_schema=change_table_schema.old_schema, ) ) diff --git a/piccolo/columns/m2m.py b/piccolo/columns/m2m.py index bb7f30b39..69a647244 100644 --- a/piccolo/columns/m2m.py +++ b/piccolo/columns/m2m.py @@ -57,30 +57,34 @@ def __init__( ) def get_select_string(self, engine_type: str, with_alias=True) -> str: - m2m_table_name = self.m2m._meta.resolved_joining_table._meta.tablename + m2m_table_name_with_schema = ( + self.m2m._meta.resolved_joining_table._meta.get_formatted_tablename() # noqa: E501 + ) # noqa: E501 m2m_relationship_name = self.m2m._meta.name fk_1 = self.m2m._meta.primary_foreign_key fk_1_name = fk_1._meta.db_column_name table_1 = fk_1._foreign_key_meta.resolved_references table_1_name = table_1._meta.tablename + table_1_name_with_schema = table_1._meta.get_formatted_tablename() table_1_pk_name = table_1._meta.primary_key._meta.db_column_name fk_2 = self.m2m._meta.secondary_foreign_key fk_2_name = fk_2._meta.db_column_name table_2 = fk_2._foreign_key_meta.resolved_references table_2_name = table_2._meta.tablename + table_2_name_with_schema = table_2._meta.get_formatted_tablename() table_2_pk_name = table_2._meta.primary_key._meta.db_column_name inner_select = f""" - "{m2m_table_name}" - JOIN "{table_1_name}" "inner_{table_1_name}" ON ( - "{m2m_table_name}"."{fk_1_name}" = "inner_{table_1_name}"."{table_1_pk_name}" + {m2m_table_name_with_schema} + JOIN {table_1_name_with_schema} "inner_{table_1_name}" ON ( + {m2m_table_name_with_schema}."{fk_1_name}" = "inner_{table_1_name}"."{table_1_pk_name}" ) - JOIN "{table_2_name}" "inner_{table_2_name}" ON ( - "{m2m_table_name}"."{fk_2_name}" = "inner_{table_2_name}"."{table_2_pk_name}" + JOIN {table_2_name_with_schema} "inner_{table_2_name}" ON ( + {m2m_table_name_with_schema}."{fk_2_name}" = "inner_{table_2_name}"."{table_2_pk_name}" ) - WHERE "{m2m_table_name}"."{fk_1_name}" = "{table_1_name}"."{table_1_pk_name}" + WHERE {m2m_table_name_with_schema}."{fk_1_name}" = "{table_1_name}"."{table_1_pk_name}" """ # noqa: E501 if engine_type in ("postgres", "cockroach"): diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index 2a3182457..dbcdbf032 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -1138,9 +1138,10 @@ def test_create_table_in_schema(self): ).run_sync(), ) - def test_move_schemas(self): + def test_move_table_from_public_schema(self): """ - Make sure the auto migrations detect that a table's schema has changed. + Make sure the auto migrations can move a table from the public schema + to a different schema. """ self._test_migrations( table_snapshots=[ @@ -1181,6 +1182,37 @@ def test_move_schemas(self): self.schema_manager.list_schemas().run_sync(), ) + def test_move_table_to_public_schema(self): + """ + Make sure the auto migrations can move a table from a schema to the + public schema. + """ + self._test_migrations( + table_snapshots=[ + [self.manager_2], + [self.manager_1], + ], + ) + + # Make sure that the table is in the public schema. + self.assertIn( + "manager", + self.schema_manager.list_tables(schema_name="public").run_sync(), + ) + + ####################################################################### + + # Reverse the last migration, which should move the table back to the + # non-public schema. + self._run_backwards(migration_id="1") + + self.assertIn( + "manager", + self.schema_manager.list_tables( + schema_name=self.new_schema + ).run_sync(), + ) + @engines_only("postgres", "cockroach") class TestSameTableName(MigrationTestCase): diff --git a/tests/columns/m2m/__init__.py b/tests/columns/m2m/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/columns/test_m2m.py b/tests/columns/m2m/base.py similarity index 50% rename from tests/columns/test_m2m.py rename to tests/columns/m2m/base.py index 653202ed8..f91d8d7d6 100644 --- a/tests/columns/test_m2m.py +++ b/tests/columns/m2m/base.py @@ -1,69 +1,29 @@ -import asyncio -import datetime -import decimal -import uuid -from unittest import TestCase +import typing as t -from tests.base import engine_is, engines_skip - -try: - from asyncpg.pgproto.pgproto import UUID as asyncpgUUID -except ImportError: - # In case someone is running the tests for SQLite and doesn't have asyncpg - # installed. - from uuid import UUID as asyncpgUUID - -from piccolo.columns.column_types import ( - JSON, - JSONB, - UUID, - Array, - BigInt, - Boolean, - Bytea, - Date, - DoublePrecision, - ForeignKey, - Integer, - Interval, - LazyTableReference, - Numeric, - Real, - SmallInt, - Text, - Timestamp, - Timestamptz, - Varchar, -) -from piccolo.columns.m2m import M2M from piccolo.engine.finder import engine_finder from piccolo.table import Table, create_db_tables_sync, drop_db_tables_sync +from tests.base import engine_is, engines_skip engine = engine_finder() -class Band(Table): - name = Varchar() - genres = M2M(LazyTableReference("GenreToBand", module_path=__name__)) - - -class Genre(Table): - name = Varchar() - bands = M2M(LazyTableReference("GenreToBand", module_path=__name__)) - - -class GenreToBand(Table): - band = ForeignKey(Band) - genre = ForeignKey(Genre) - reason = Text(help_text="For testing additional columns on join tables.") - - -SIMPLE_SCHEMA = [Band, Genre, GenreToBand] +class M2MBase: + """ + This allows us to test M2M when the tables are in different schemas + (public vs non-public). + """ + band: t.Type[Table] + genre: t.Type[Table] + genre_to_band: t.Type[Table] + all_tables: t.List[t.Type[Table]] -class TestM2M(TestCase): def setUp(self): - create_db_tables_sync(*SIMPLE_SCHEMA, if_not_exists=True) + Band = self.band + Genre = self.genre + GenreToBand = self.genre_to_band + + create_db_tables_sync(*self.all_tables, if_not_exists=True) if engine_is("cockroach"): bands = ( @@ -115,13 +75,16 @@ def setUp(self): ).run_sync() def tearDown(self): - drop_db_tables_sync(*SIMPLE_SCHEMA) + drop_db_tables_sync(*self.all_tables) @engines_skip("cockroach") def test_select_name(self): """ 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg """ # noqa: E501 + Band = self.band + Genre = self.genre + response = Band.select( Band.name, Band.genres(Genre.name, as_list=True) ).run_sync() @@ -155,6 +118,10 @@ def test_no_related(self): """ 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg """ # noqa: E501 + Band = self.band + Genre = self.genre + GenreToBand = self.genre_to_band + GenreToBand.delete(force=True).run_sync() # Try it with a list response @@ -189,6 +156,9 @@ def test_select_multiple(self): """ 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg """ # noqa: E501 + Band = self.band + Genre = self.genre + response = Band.select( Band.name, Band.genres(Genre.id, Genre.name) ).run_sync() @@ -248,6 +218,9 @@ def test_select_id(self): """ 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg """ # noqa: E501 + Band = self.band + Genre = self.genre + response = Band.select( Band.name, Band.genres(Genre.id, as_list=True) ).run_sync() @@ -284,6 +257,9 @@ def test_select_all_columns(self): 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg """ # noqa: E501 + Band = self.band + Genre = self.genre + response = Band.select( Band.name, Band.genres(Genre.all_columns(exclude=(Genre.id,))) ).run_sync() @@ -312,6 +288,10 @@ def test_add_m2m(self): """ Make sure we can add items to the joining table. """ + Band = self.band + Genre = self.genre + GenreToBand = self.genre_to_band + band: Band = Band.objects().get(Band.name == "Pythonistas").run_sync() band.add_m2m(Genre(name="Punk Rock"), m2m=Band.genres).run_sync() @@ -334,6 +314,10 @@ def test_extra_columns_str(self): Make sure the ``extra_column_values`` parameter for ``add_m2m`` works correctly when the dictionary keys are strings. """ + Band = self.band + Genre = self.genre + GenreToBand = self.genre_to_band + reason = "Their second album was very punk rock." band: Band = Band.objects().get(Band.name == "Pythonistas").run_sync() @@ -361,6 +345,10 @@ def test_extra_columns_class(self): Make sure the ``extra_column_values`` parameter for ``add_m2m`` works correctly when the dictionary keys are ``Column`` classes. """ + Band = self.band + Genre = self.genre + GenreToBand = self.genre_to_band + reason = "Their second album was very punk rock." band: Band = Band.objects().get(Band.name == "Pythonistas").run_sync() @@ -387,6 +375,10 @@ def test_add_m2m_existing(self): """ Make sure we can add an existing element to the joining table. """ + Band = self.band + Genre = self.genre + GenreToBand = self.genre_to_band + band: Band = Band.objects().get(Band.name == "Pythonistas").run_sync() genre: Genre = ( @@ -414,6 +406,8 @@ def test_get_m2m(self): """ Make sure we can get related items via the joining table. """ + Band = self.band + band: Band = Band.objects().get(Band.name == "Pythonistas").run_sync() genres = band.get_m2m(Band.genres).run_sync() @@ -426,6 +420,10 @@ def test_remove_m2m(self): """ Make sure we can remove related items via the joining table. """ + Band = self.band + Genre = self.genre + GenreToBand = self.genre_to_band + band: Band = Band.objects().get(Band.name == "Pythonistas").run_sync() genre = Genre.objects().get(Genre.name == "Rock").run_sync() @@ -462,372 +460,3 @@ def test_remove_m2m(self): .run_sync(), 1, ) - - -############################################################################### - -# A schema using custom primary keys - - -class Customer(Table): - uuid = UUID(primary_key=True) - name = Varchar() - concerts = M2M( - LazyTableReference("CustomerToConcert", module_path=__name__) - ) - - -class Concert(Table): - uuid = UUID(primary_key=True) - name = Varchar() - customers = M2M( - LazyTableReference("CustomerToConcert", module_path=__name__) - ) - - -class CustomerToConcert(Table): - customer = ForeignKey(Customer) - concert = ForeignKey(Concert) - - -CUSTOM_PK_SCHEMA = [Customer, Concert, CustomerToConcert] - - -class TestM2MCustomPrimaryKey(TestCase): - """ - Make sure the M2M functionality works correctly when the tables have custom - primary key columns. - """ - - def setUp(self): - create_db_tables_sync(*CUSTOM_PK_SCHEMA, if_not_exists=True) - - bob = Customer.objects().create(name="Bob").run_sync() - sally = Customer.objects().create(name="Sally").run_sync() - fred = Customer.objects().create(name="Fred").run_sync() - - rockfest = Concert.objects().create(name="Rockfest").run_sync() - folkfest = Concert.objects().create(name="Folkfest").run_sync() - classicfest = Concert.objects().create(name="Classicfest").run_sync() - - CustomerToConcert.insert( - CustomerToConcert(customer=bob, concert=rockfest), - CustomerToConcert(customer=bob, concert=classicfest), - CustomerToConcert(customer=sally, concert=rockfest), - CustomerToConcert(customer=sally, concert=folkfest), - CustomerToConcert(customer=fred, concert=classicfest), - ).run_sync() - - def tearDown(self): - drop_db_tables_sync(*CUSTOM_PK_SCHEMA) - - @engines_skip("cockroach") - def test_select(self): - """ - 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg - """ # noqa: E501 - response = Customer.select( - Customer.name, Customer.concerts(Concert.name, as_list=True) - ).run_sync() - - self.assertListEqual( - response, - [ - {"name": "Bob", "concerts": ["Rockfest", "Classicfest"]}, - {"name": "Sally", "concerts": ["Rockfest", "Folkfest"]}, - {"name": "Fred", "concerts": ["Classicfest"]}, - ], - ) - - # Now try it in reverse. - response = Concert.select( - Concert.name, Concert.customers(Customer.name, as_list=True) - ).run_sync() - - self.assertListEqual( - response, - [ - {"name": "Rockfest", "customers": ["Bob", "Sally"]}, - {"name": "Folkfest", "customers": ["Sally"]}, - {"name": "Classicfest", "customers": ["Bob", "Fred"]}, - ], - ) - - def test_add_m2m(self): - """ - Make sure we can add items to the joining table. - """ - customer: Customer = ( - Customer.objects().get(Customer.name == "Bob").run_sync() - ) - customer.add_m2m( - Concert(name="Jazzfest"), m2m=Customer.concerts - ).run_sync() - - self.assertTrue( - Concert.exists().where(Concert.name == "Jazzfest").run_sync() - ) - - self.assertEqual( - CustomerToConcert.count() - .where( - CustomerToConcert.customer.name == "Bob", - CustomerToConcert.concert.name == "Jazzfest", - ) - .run_sync(), - 1, - ) - - def test_add_m2m_within_transaction(self): - """ - Make sure we can add items to the joining table, when within an - existing transaction. - - https://github.com/piccolo-orm/piccolo/issues/674 - - """ - engine = Customer._meta.db - - async def add_m2m_in_transaction(): - async with engine.transaction(): - customer: Customer = await Customer.objects().get( - Customer.name == "Bob" - ) - await customer.add_m2m( - Concert(name="Jazzfest"), m2m=Customer.concerts - ) - - asyncio.run(add_m2m_in_transaction()) - - self.assertTrue( - Concert.exists().where(Concert.name == "Jazzfest").run_sync() - ) - - self.assertEqual( - CustomerToConcert.count() - .where( - CustomerToConcert.customer.name == "Bob", - CustomerToConcert.concert.name == "Jazzfest", - ) - .run_sync(), - 1, - ) - - def test_get_m2m(self): - """ - Make sure we can get related items via the joining table. - """ - customer: Customer = ( - Customer.objects().get(Customer.name == "Bob").run_sync() - ) - - concerts = customer.get_m2m(Customer.concerts).run_sync() - - self.assertTrue(all(isinstance(i, Table) for i in concerts)) - - self.assertCountEqual( - [i.name for i in concerts], ["Rockfest", "Classicfest"] - ) - - -############################################################################### - -# Test a very complex schema - - -class SmallTable(Table): - varchar_col = Varchar() - mega_rows = M2M(LazyTableReference("SmallToMega", module_path=__name__)) - - -if engine.engine_type != "cockroach": # type: ignore - - class MegaTable(Table): # type: ignore - """ - A table containing all of the column types and different column kwargs - """ - - array_col = Array(Varchar()) - bigint_col = BigInt() - boolean_col = Boolean() - bytea_col = Bytea() - date_col = Date() - double_precision_col = DoublePrecision() - integer_col = Integer() - interval_col = Interval() - json_col = JSON() - jsonb_col = JSONB() - numeric_col = Numeric(digits=(5, 2)) - real_col = Real() - smallint_col = SmallInt() - text_col = Text() - timestamp_col = Timestamp() - timestamptz_col = Timestamptz() - uuid_col = UUID() - varchar_col = Varchar() - -else: - - class MegaTable(Table): # type: ignore - """ - Special version for Cockroach. - A table containing all of the column types and different column kwargs - """ - - array_col = Array(Varchar()) - bigint_col = BigInt() - boolean_col = Boolean() - bytea_col = Bytea() - date_col = Date() - double_precision_col = DoublePrecision() - integer_col = BigInt() - interval_col = Interval() - json_col = JSONB() - jsonb_col = JSONB() - numeric_col = Numeric(digits=(5, 2)) - real_col = Real() - smallint_col = SmallInt() - text_col = Text() - timestamp_col = Timestamp() - timestamptz_col = Timestamptz() - uuid_col = UUID() - varchar_col = Varchar() - - -class SmallToMega(Table): - small = ForeignKey(MegaTable) - mega = ForeignKey(SmallTable) - - -COMPLEX_SCHEMA = [MegaTable, SmallTable, SmallToMega] - - -class TestM2MComplexSchema(TestCase): - """ - By using a very complex schema containing every column type, we can catch - more edge cases. - """ - - def setUp(self): - create_db_tables_sync(*COMPLEX_SCHEMA, if_not_exists=True) - - small_table = SmallTable(varchar_col="Test") - small_table.save().run_sync() - - mega_table = MegaTable( - array_col=["bob", "sally"], - bigint_col=1, - boolean_col=True, - bytea_col="hello".encode("utf8"), - date_col=datetime.date(year=2021, month=1, day=1), - double_precision_col=1.344, - integer_col=1, - interval_col=datetime.timedelta(seconds=10), - json_col={"a": 1}, - jsonb_col={"a": 1}, - numeric_col=decimal.Decimal("1.1"), - real_col=1.1, - smallint_col=1, - text_col="hello", - timestamp_col=datetime.datetime(year=2021, month=1, day=1), - timestamptz_col=datetime.datetime( - year=2021, month=1, day=1, tzinfo=datetime.timezone.utc - ), - uuid_col=uuid.UUID("12783854-c012-4c15-8183-8eecb46f2c4e"), - varchar_col="hello", - ) - mega_table.save().run_sync() - - SmallToMega(small=small_table, mega=mega_table).save().run_sync() - - self.mega_table = mega_table - - def tearDown(self): - drop_db_tables_sync(*COMPLEX_SCHEMA) - - @engines_skip("cockroach") - def test_select_all(self): - """ - Fetch all of the columns from the related table to make sure they're - returned correctly. - """ - """ - 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg - """ # noqa: E501 - response = SmallTable.select( - SmallTable.varchar_col, SmallTable.mega_rows(load_json=True) - ).run_sync() - - self.assertEqual(len(response), 1) - mega_rows = response[0]["mega_rows"] - - self.assertEqual(len(mega_rows), 1) - mega_row = mega_rows[0] - - for key, value in mega_row.items(): - # Make sure that every value in the response matches what we saved. - self.assertAlmostEqual( - getattr(self.mega_table, key), - value, - msg=f"{key} doesn't match", - ) - - @engines_skip("cockroach") - def test_select_single(self): - """ - Make sure each column can be selected one at a time. - """ - """ - 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg - """ # noqa: E501 - for column in MegaTable._meta.columns: - response = SmallTable.select( - SmallTable.varchar_col, - SmallTable.mega_rows(column, load_json=True), - ).run_sync() - - data = response[0]["mega_rows"][0] - column_name = column._meta.name - - original_value = getattr(self.mega_table, column_name) - returned_value = data[column_name] - - if type(column) == UUID: - self.assertIn(type(returned_value), (uuid.UUID, asyncpgUUID)) - else: - self.assertEqual( - type(original_value), - type(returned_value), - msg=f"{column_name} type isn't correct", - ) - - self.assertAlmostEqual( - original_value, - returned_value, - msg=f"{column_name} doesn't match", - ) - - # Test it as a list too - response = SmallTable.select( - SmallTable.varchar_col, - SmallTable.mega_rows(column, as_list=True, load_json=True), - ).run_sync() - - original_value = getattr(self.mega_table, column_name) - returned_value = response[0]["mega_rows"][0] - - if type(column) == UUID: - self.assertIn(type(returned_value), (uuid.UUID, asyncpgUUID)) - self.assertEqual(str(original_value), str(returned_value)) - else: - self.assertEqual( - type(original_value), - type(returned_value), - msg=f"{column_name} type isn't correct", - ) - - self.assertAlmostEqual( - original_value, - returned_value, - msg=f"{column_name} doesn't match", - ) diff --git a/tests/columns/m2m/test_m2m.py b/tests/columns/m2m/test_m2m.py new file mode 100644 index 000000000..e928a251f --- /dev/null +++ b/tests/columns/m2m/test_m2m.py @@ -0,0 +1,436 @@ +import asyncio +import datetime +import decimal +import uuid +from unittest import TestCase + +from tests.base import engines_skip + +try: + from asyncpg.pgproto.pgproto import UUID as asyncpgUUID +except ImportError: + # In case someone is running the tests for SQLite and doesn't have asyncpg + # installed. + from uuid import UUID as asyncpgUUID + +from piccolo.columns.column_types import ( + JSON, + JSONB, + UUID, + Array, + BigInt, + Boolean, + Bytea, + Date, + DoublePrecision, + ForeignKey, + Integer, + Interval, + LazyTableReference, + Numeric, + Real, + SmallInt, + Text, + Timestamp, + Timestamptz, + Varchar, +) +from piccolo.columns.m2m import M2M +from piccolo.engine.finder import engine_finder +from piccolo.table import Table, create_db_tables_sync, drop_db_tables_sync + +from .base import M2MBase + +engine = engine_finder() + + +class Band(Table): + name = Varchar() + genres = M2M(LazyTableReference("GenreToBand", module_path=__name__)) + + +class Genre(Table): + name = Varchar() + bands = M2M(LazyTableReference("GenreToBand", module_path=__name__)) + + +class GenreToBand(Table): + band = ForeignKey(Band) + genre = ForeignKey(Genre) + reason = Text(help_text="For testing additional columns on join tables.") + + +class TestM2M(M2MBase, TestCase): + band = Band + genre = Genre + genre_to_band = GenreToBand + all_tables = [Band, Genre, GenreToBand] + + +############################################################################### + +# A schema using custom primary keys + + +class Customer(Table): + uuid = UUID(primary_key=True) + name = Varchar() + concerts = M2M( + LazyTableReference("CustomerToConcert", module_path=__name__) + ) + + +class Concert(Table): + uuid = UUID(primary_key=True) + name = Varchar() + customers = M2M( + LazyTableReference("CustomerToConcert", module_path=__name__) + ) + + +class CustomerToConcert(Table): + customer = ForeignKey(Customer) + concert = ForeignKey(Concert) + + +CUSTOM_PK_SCHEMA = [Customer, Concert, CustomerToConcert] + + +class TestM2MCustomPrimaryKey(TestCase): + """ + Make sure the M2M functionality works correctly when the tables have custom + primary key columns. + """ + + def setUp(self): + create_db_tables_sync(*CUSTOM_PK_SCHEMA, if_not_exists=True) + + bob = Customer.objects().create(name="Bob").run_sync() + sally = Customer.objects().create(name="Sally").run_sync() + fred = Customer.objects().create(name="Fred").run_sync() + + rockfest = Concert.objects().create(name="Rockfest").run_sync() + folkfest = Concert.objects().create(name="Folkfest").run_sync() + classicfest = Concert.objects().create(name="Classicfest").run_sync() + + CustomerToConcert.insert( + CustomerToConcert(customer=bob, concert=rockfest), + CustomerToConcert(customer=bob, concert=classicfest), + CustomerToConcert(customer=sally, concert=rockfest), + CustomerToConcert(customer=sally, concert=folkfest), + CustomerToConcert(customer=fred, concert=classicfest), + ).run_sync() + + def tearDown(self): + drop_db_tables_sync(*CUSTOM_PK_SCHEMA) + + @engines_skip("cockroach") + def test_select(self): + """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + """ # noqa: E501 + response = Customer.select( + Customer.name, Customer.concerts(Concert.name, as_list=True) + ).run_sync() + + self.assertListEqual( + response, + [ + {"name": "Bob", "concerts": ["Rockfest", "Classicfest"]}, + {"name": "Sally", "concerts": ["Rockfest", "Folkfest"]}, + {"name": "Fred", "concerts": ["Classicfest"]}, + ], + ) + + # Now try it in reverse. + response = Concert.select( + Concert.name, Concert.customers(Customer.name, as_list=True) + ).run_sync() + + self.assertListEqual( + response, + [ + {"name": "Rockfest", "customers": ["Bob", "Sally"]}, + {"name": "Folkfest", "customers": ["Sally"]}, + {"name": "Classicfest", "customers": ["Bob", "Fred"]}, + ], + ) + + def test_add_m2m(self): + """ + Make sure we can add items to the joining table. + """ + customer: Customer = ( + Customer.objects().get(Customer.name == "Bob").run_sync() + ) + customer.add_m2m( + Concert(name="Jazzfest"), m2m=Customer.concerts + ).run_sync() + + self.assertTrue( + Concert.exists().where(Concert.name == "Jazzfest").run_sync() + ) + + self.assertEqual( + CustomerToConcert.count() + .where( + CustomerToConcert.customer.name == "Bob", + CustomerToConcert.concert.name == "Jazzfest", + ) + .run_sync(), + 1, + ) + + def test_add_m2m_within_transaction(self): + """ + Make sure we can add items to the joining table, when within an + existing transaction. + + https://github.com/piccolo-orm/piccolo/issues/674 + + """ + engine = Customer._meta.db + + async def add_m2m_in_transaction(): + async with engine.transaction(): + customer: Customer = await Customer.objects().get( + Customer.name == "Bob" + ) + await customer.add_m2m( + Concert(name="Jazzfest"), m2m=Customer.concerts + ) + + asyncio.run(add_m2m_in_transaction()) + + self.assertTrue( + Concert.exists().where(Concert.name == "Jazzfest").run_sync() + ) + + self.assertEqual( + CustomerToConcert.count() + .where( + CustomerToConcert.customer.name == "Bob", + CustomerToConcert.concert.name == "Jazzfest", + ) + .run_sync(), + 1, + ) + + def test_get_m2m(self): + """ + Make sure we can get related items via the joining table. + """ + customer: Customer = ( + Customer.objects().get(Customer.name == "Bob").run_sync() + ) + + concerts = customer.get_m2m(Customer.concerts).run_sync() + + self.assertTrue(all(isinstance(i, Table) for i in concerts)) + + self.assertCountEqual( + [i.name for i in concerts], ["Rockfest", "Classicfest"] + ) + + +############################################################################### + +# Test a very complex schema + + +class SmallTable(Table): + varchar_col = Varchar() + mega_rows = M2M(LazyTableReference("SmallToMega", module_path=__name__)) + + +if engine.engine_type != "cockroach": # type: ignore + + class MegaTable(Table): # type: ignore + """ + A table containing all of the column types and different column kwargs + """ + + array_col = Array(Varchar()) + bigint_col = BigInt() + boolean_col = Boolean() + bytea_col = Bytea() + date_col = Date() + double_precision_col = DoublePrecision() + integer_col = Integer() + interval_col = Interval() + json_col = JSON() + jsonb_col = JSONB() + numeric_col = Numeric(digits=(5, 2)) + real_col = Real() + smallint_col = SmallInt() + text_col = Text() + timestamp_col = Timestamp() + timestamptz_col = Timestamptz() + uuid_col = UUID() + varchar_col = Varchar() + +else: + + class MegaTable(Table): # type: ignore + """ + Special version for Cockroach. + A table containing all of the column types and different column kwargs + """ + + array_col = Array(Varchar()) + bigint_col = BigInt() + boolean_col = Boolean() + bytea_col = Bytea() + date_col = Date() + double_precision_col = DoublePrecision() + integer_col = BigInt() + interval_col = Interval() + json_col = JSONB() + jsonb_col = JSONB() + numeric_col = Numeric(digits=(5, 2)) + real_col = Real() + smallint_col = SmallInt() + text_col = Text() + timestamp_col = Timestamp() + timestamptz_col = Timestamptz() + uuid_col = UUID() + varchar_col = Varchar() + + +class SmallToMega(Table): + small = ForeignKey(MegaTable) + mega = ForeignKey(SmallTable) + + +COMPLEX_SCHEMA = [MegaTable, SmallTable, SmallToMega] + + +class TestM2MComplexSchema(TestCase): + """ + By using a very complex schema containing every column type, we can catch + more edge cases. + """ + + def setUp(self): + create_db_tables_sync(*COMPLEX_SCHEMA, if_not_exists=True) + + small_table = SmallTable(varchar_col="Test") + small_table.save().run_sync() + + mega_table = MegaTable( + array_col=["bob", "sally"], + bigint_col=1, + boolean_col=True, + bytea_col="hello".encode("utf8"), + date_col=datetime.date(year=2021, month=1, day=1), + double_precision_col=1.344, + integer_col=1, + interval_col=datetime.timedelta(seconds=10), + json_col={"a": 1}, + jsonb_col={"a": 1}, + numeric_col=decimal.Decimal("1.1"), + real_col=1.1, + smallint_col=1, + text_col="hello", + timestamp_col=datetime.datetime(year=2021, month=1, day=1), + timestamptz_col=datetime.datetime( + year=2021, month=1, day=1, tzinfo=datetime.timezone.utc + ), + uuid_col=uuid.UUID("12783854-c012-4c15-8183-8eecb46f2c4e"), + varchar_col="hello", + ) + mega_table.save().run_sync() + + SmallToMega(small=small_table, mega=mega_table).save().run_sync() + + self.mega_table = mega_table + + def tearDown(self): + drop_db_tables_sync(*COMPLEX_SCHEMA) + + @engines_skip("cockroach") + def test_select_all(self): + """ + Fetch all of the columns from the related table to make sure they're + returned correctly. + """ + """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + """ # noqa: E501 + response = SmallTable.select( + SmallTable.varchar_col, SmallTable.mega_rows(load_json=True) + ).run_sync() + + self.assertEqual(len(response), 1) + mega_rows = response[0]["mega_rows"] + + self.assertEqual(len(mega_rows), 1) + mega_row = mega_rows[0] + + for key, value in mega_row.items(): + # Make sure that every value in the response matches what we saved. + self.assertAlmostEqual( + getattr(self.mega_table, key), + value, + msg=f"{key} doesn't match", + ) + + @engines_skip("cockroach") + def test_select_single(self): + """ + Make sure each column can be selected one at a time. + """ + """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + """ # noqa: E501 + for column in MegaTable._meta.columns: + response = SmallTable.select( + SmallTable.varchar_col, + SmallTable.mega_rows(column, load_json=True), + ).run_sync() + + data = response[0]["mega_rows"][0] + column_name = column._meta.name + + original_value = getattr(self.mega_table, column_name) + returned_value = data[column_name] + + if type(column) == UUID: + self.assertIn(type(returned_value), (uuid.UUID, asyncpgUUID)) + else: + self.assertEqual( + type(original_value), + type(returned_value), + msg=f"{column_name} type isn't correct", + ) + + self.assertAlmostEqual( + original_value, + returned_value, + msg=f"{column_name} doesn't match", + ) + + # Test it as a list too + response = SmallTable.select( + SmallTable.varchar_col, + SmallTable.mega_rows(column, as_list=True, load_json=True), + ).run_sync() + + original_value = getattr(self.mega_table, column_name) + returned_value = response[0]["mega_rows"][0] + + if type(column) == UUID: + self.assertIn(type(returned_value), (uuid.UUID, asyncpgUUID)) + self.assertEqual(str(original_value), str(returned_value)) + else: + self.assertEqual( + type(original_value), + type(returned_value), + msg=f"{column_name} type isn't correct", + ) + + self.assertAlmostEqual( + original_value, + returned_value, + msg=f"{column_name} doesn't match", + ) diff --git a/tests/columns/m2m/test_m2m_schema.py b/tests/columns/m2m/test_m2m_schema.py new file mode 100644 index 000000000..f9b958838 --- /dev/null +++ b/tests/columns/m2m/test_m2m_schema.py @@ -0,0 +1,48 @@ +from unittest import TestCase + +from piccolo.columns.column_types import ( + ForeignKey, + LazyTableReference, + Text, + Varchar, +) +from piccolo.columns.m2m import M2M +from piccolo.schema import SchemaManager +from piccolo.table import Table +from tests.base import engines_skip + +from .base import M2MBase + + +class Band(Table, schema="schema_1"): + name = Varchar() + genres = M2M(LazyTableReference("GenreToBand", module_path=__name__)) + + +class Genre(Table, schema="schema_1"): + name = Varchar() + bands = M2M(LazyTableReference("GenreToBand", module_path=__name__)) + + +class GenreToBand(Table, schema="schema_1"): + band = ForeignKey(Band) + genre = ForeignKey(Genre) + reason = Text(help_text="For testing additional columns on join tables.") + + +@engines_skip("sqlite") +class TestM2MWithSchema(M2MBase, TestCase): + """ + Make sure that when the tables exist in a non-public schema, that M2M still + works. + """ + + band = Band + genre = Genre + genre_to_band = GenreToBand + all_tables = [Band, Genre, GenreToBand] + + def tearDown(self): + SchemaManager().drop_schema( + schema_name="schema_1", cascade=True + ).run_sync() From 4c56cde574ff5f81fe52d29849cfe73f8d3bfebe Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 19 Jun 2023 20:06:53 +0100 Subject: [PATCH 484/727] Allow upserts when loading fixtures (#840) * allow upserts when loading fixtures * fix typo * add docs * add docs for chunk size * break up fixture docs into sections --- .../projects_and_apps/included_apps.rst | 49 +++++++++++++++-- piccolo/apps/fixtures/commands/load.py | 53 ++++++++++++++++--- .../apps/fixtures/commands/test_dump_load.py | 40 +++++++++++++- 3 files changed, 132 insertions(+), 10 deletions(-) diff --git a/docs/src/piccolo/projects_and_apps/included_apps.rst b/docs/src/piccolo/projects_and_apps/included_apps.rst index 5762803ae..620e20194 100644 --- a/docs/src/piccolo/projects_and_apps/included_apps.rst +++ b/docs/src/piccolo/projects_and_apps/included_apps.rst @@ -48,13 +48,17 @@ Once you have created a fixture, it can be used by your colleagues when setting up an application on their local machines, or when deploying to a new environment. -Databases such as Postgres have inbuilt ways of dumping and restoring data +Databases such as Postgres have built-in ways of dumping and restoring data (via ``pg_dump`` and ``pg_restore``). Some reasons to use the fixtures app instead: -* When you want the data to be loadable in a range of database versions. +* When you want the data to be loadable in a range of database types and + versions. * Fixtures are stored in JSON, which are a bit friendlier for source control. +dump +^^^^ + To dump the data into a new fixture file: .. code-block:: bash @@ -68,15 +72,54 @@ a subset of apps and tables instead, for example: piccolo fixtures dump --apps=blog --tables=Post > fixtures.json - # Or for multiple apps / tables +Or for multiple apps / tables: + +.. code-block:: bash + piccolo fixtures dump --apps=blog,shop --tables=Post,Product > fixtures.json + +load +^^^^ + To load the fixture: .. code-block:: bash piccolo fixtures load fixtures.json +If you load the fixture again, you will get primary key errors because the rows +already exist in the database. But what if we need to run it again, because we +had a typo in our fixture, or were missing some data? We can upsert the data +using ``--on_conflict``. + +There are two options: + +1. ``DO NOTHING`` - if any of the rows already exist in the database, just + leave them as they are, and don't raise an exception. +2. ``DO UPDATE`` - if any of the rows already exist in the database, override + them with the latest data in the fixture file. + +.. code-block:: bash + + # DO NOTHING + piccolo fixtures load fixtures.json --on_conflict='DO NOTHING' + + # DO UPDATE + piccolo fixtures load fixtures.json --on_conflict='DO UPDATE' + +And finally, if you're loading a really large fixture, you can specify the +``chunk_size``. By default, Piccolo inserts up to 1,000 rows at a time, as +the database adapter will complain if a single insert query is too large. So +if your fixture containts 10,000 rows, this will mean 10 insert queries. + +You can tune this number higher or lower if you want (lower if the +table has a lot of columns, or higher if the table has few columns). + +.. code-block:: bash + + piccolo fixtures load fixtures.json --chunk_size=500 + ------------------------------------------------------------------------------- meta diff --git a/piccolo/apps/fixtures/commands/load.py b/piccolo/apps/fixtures/commands/load.py index 26363451e..cb330b207 100644 --- a/piccolo/apps/fixtures/commands/load.py +++ b/piccolo/apps/fixtures/commands/load.py @@ -1,19 +1,27 @@ from __future__ import annotations +import sys import typing as t +import typing_extensions + from piccolo.apps.fixtures.commands.shared import ( FixtureConfig, create_pydantic_fixture_model, ) from piccolo.conf.apps import Finder from piccolo.engine import engine_finder +from piccolo.query.mixins import OnConflictAction from piccolo.table import Table, sort_table_classes from piccolo.utils.encoding import load_json from piccolo.utils.list import batch -async def load_json_string(json_string: str, chunk_size: int = 1000): +async def load_json_string( + json_string: str, + chunk_size: int = 1000, + on_conflict_action: t.Optional[OnConflictAction] = None, +): """ Parses the JSON string, and inserts the parsed data into the database. """ @@ -71,10 +79,23 @@ async def load_json_string(json_string: str, chunk_size: int = 1000): rows = data[table_class] for chunk in batch(data=rows, chunk_size=chunk_size): - await table_class.insert(*chunk).run() - - -async def load(path: str = "fixture.json", chunk_size: int = 1000): + query = table_class.insert(*chunk) + if on_conflict_action is not None: + query = query.on_conflict( + target=table_class._meta.primary_key, + action=on_conflict_action, + values=table_class._meta.columns, + ) + await query.run() + + +async def load( + path: str = "fixture.json", + chunk_size: int = 1000, + on_conflict: t.Optional[ + typing_extensions.Literal["DO NOTHING", "DO UPDATE"] + ] = None, +): """ Reads the fixture file, and loads the contents into the database. @@ -86,8 +107,28 @@ async def load(path: str = "fixture.json", chunk_size: int = 1000): determined by the database adapter, which has a max number of parameters per query. + :param on_conflict: + If specified, the fixture will be upserted, meaning that if a row + already exists with a matching primary key, then it will be overridden + if "DO UPDATE", or it will be ignored if "DO NOTHING". + """ with open(path, "r") as f: contents = f.read() - await load_json_string(contents, chunk_size=chunk_size) + on_conflict_action: t.Optional[OnConflictAction] = None + + if on_conflict: + try: + on_conflict_action = OnConflictAction(on_conflict.upper()) + except ValueError: + sys.exit( + f"{on_conflict} isn't a valid option - use 'DO NOTHING' or " + "'DO UPDATE'." + ) + + await load_json_string( + contents, + chunk_size=chunk_size, + on_conflict_action=on_conflict_action, + ) diff --git a/tests/apps/fixtures/commands/test_dump_load.py b/tests/apps/fixtures/commands/test_dump_load.py index eee930ac1..407bd4504 100644 --- a/tests/apps/fixtures/commands/test_dump_load.py +++ b/tests/apps/fixtures/commands/test_dump_load.py @@ -1,5 +1,7 @@ import datetime import decimal +import os +import tempfile import typing as t import uuid from unittest import TestCase @@ -8,7 +10,7 @@ FixtureConfig, dump_to_json_string, ) -from piccolo.apps.fixtures.commands.load import load_json_string +from piccolo.apps.fixtures.commands.load import load, load_json_string from piccolo.utils.sync import run_sync from tests.base import engines_only from tests.example_apps.mega.tables import MegaTable, SmallTable @@ -240,3 +242,39 @@ def test_dump_load_cockroach(self): "not_null_col": "hello", }, ) + + +class TestOnConflict(TestCase): + def setUp(self) -> None: + SmallTable.create_table().run_sync() + SmallTable({SmallTable.varchar_col: "Test"}).save().run_sync() + + def tearDown(self) -> None: + SmallTable.alter().drop_table().run_sync() + + def test_on_conflict(self): + temp_dir = tempfile.gettempdir() + + json_file_path = os.path.join(temp_dir, "fixture.json") + + json_string = run_sync( + dump_to_json_string( + fixture_configs=[ + FixtureConfig( + app_name="mega", + table_class_names=["SmallTable"], + ) + ] + ) + ) + + if os.path.exists(json_file_path): + os.unlink(json_file_path) + + with open(json_file_path, "w") as f: + f.write(json_string) + + run_sync(load(path=json_file_path, on_conflict="DO NOTHING")) + run_sync(load(path=json_file_path, on_conflict="DO UPDATE")) + run_sync(load(path=json_file_path, on_conflict="do nothing")) + run_sync(load(path=json_file_path, on_conflict="do update")) From f1a47ec944b32d671978848c60d76af2ede27ec2 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 19 Jun 2023 20:31:25 +0100 Subject: [PATCH 485/727] add `distinct` method to `count` queries (#846) --- docs/src/piccolo/query_types/count.rst | 3 +++ piccolo/query/methods/count.py | 10 +++++++--- tests/table/test_count.py | 4 ++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/src/piccolo/query_types/count.rst b/docs/src/piccolo/query_types/count.rst index 4a60a2bd8..033667c18 100644 --- a/docs/src/piccolo/query_types/count.rst +++ b/docs/src/piccolo/query_types/count.rst @@ -85,6 +85,9 @@ We can count the number of distinct (i.e. unique) rows. await Band.count(distinct=[Band.name]) + # This also works - use whichever you prefer: + await Band.count().distinct([Band.name]) + With the following data: .. table:: diff --git a/piccolo/query/methods/count.py b/piccolo/query/methods/count.py index a0e25243b..fdd0972cf 100644 --- a/piccolo/query/methods/count.py +++ b/piccolo/query/methods/count.py @@ -15,7 +15,7 @@ class Count(Query): - __slots__ = ("where_delegate", "column", "distinct") + __slots__ = ("where_delegate", "column", "_distinct") def __init__( self, @@ -26,7 +26,7 @@ def __init__( ): super().__init__(table, **kwargs) self.column = column - self.distinct = distinct + self._distinct = distinct self.where_delegate = WhereDelegate() ########################################################################### @@ -36,6 +36,10 @@ def where(self: Self, *where: Combinable) -> Self: self.where_delegate.where(*where) return self + def distinct(self: Self, columns: t.Optional[t.Sequence[Column]]) -> Self: + self._distinct = columns + return self + ########################################################################### async def response_handler(self, response) -> bool: @@ -46,7 +50,7 @@ def default_querystrings(self) -> t.Sequence[QueryString]: table: t.Type[Table] = self.table query = table.select( - SelectCount(column=self.column, distinct=self.distinct) + SelectCount(column=self.column, distinct=self._distinct) ) query.where_delegate._where = self.where_delegate._where diff --git a/tests/table/test_count.py b/tests/table/test_count.py index 3b777e6f7..7da90bea0 100644 --- a/tests/table/test_count.py +++ b/tests/table/test_count.py @@ -50,6 +50,10 @@ def test_count_distinct(self): self.assertEqual(response, 3) + # Test the method also works + response = Band.count().distinct([Band.popularity]).run_sync() + self.assertEqual(response, 3) + def test_count_distinct_multiple(self): Band.insert( Band(name="Pythonistas", popularity=10), From 1457ded03a491da1d34b913be1eb9e4f724a5650 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 19 Jun 2023 21:15:17 +0100 Subject: [PATCH 486/727] bumped version --- CHANGES.rst | 56 +++++++++++++++++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index b531947a2..c598885cc 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,62 @@ Changes ======= +0.115.0 +------- + +Fixture upserting +~~~~~~~~~~~~~~~~~ + +Fixtures can now be upserted. For example: + +.. code-block:: bash + + piccolo fixtures load my_fixture.json --on_conflict='DO UPDATE' + +The options are: + +* ``DO NOTHING``, meaning any rows with a matching primary key will be left + alone. +* ``DO UPDATE``, meaning any rows with a matching primary key will be updated. + +This is really useful, as you can now edit fixtures and load them multiple +times without getting foreign key constraint errors. + +Schema fixes +~~~~~~~~~~~~ + +We recently added support for schemas, for example: + +.. code-block:: python + + class Band(Table, schema='music'): + ... + +This release contains: + +* A fix for migrations when changing a table's schema back to 'public' (thanks to + @sinisaos for discovering this). +* A fix for ``M2M`` queries, when the tables are in a schema other than + 'public' (thanks to @quinnalfaro for reporing this). + +Added ``distinct`` method to ``count`` queries +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We recently added support for ``COUNT DISTINCT`` queries. The syntax is: + +.. code-block:: python + + await Concert.count(distinct=[Concert.start_date]) + +The following alternative syntax now also works (just to be consistent with +other queries like ``select``): + +.. code-block:: python + + await Concert.count().distinct([Concert.start_date]) + +------------------------------------------------------------------------------- + 0.114.0 ------- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 73b2b393f..07b651f95 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.114.0" +__VERSION__ = "0.115.0" From 74ea10d427464b05d4a0caba05ce317582d74203 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 19 Jun 2023 21:18:15 +0100 Subject: [PATCH 487/727] fix typo in changelog --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index c598885cc..ed54d54fa 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -37,7 +37,7 @@ This release contains: * A fix for migrations when changing a table's schema back to 'public' (thanks to @sinisaos for discovering this). * A fix for ``M2M`` queries, when the tables are in a schema other than - 'public' (thanks to @quinnalfaro for reporing this). + 'public' (thanks to @quinnalfaro for reporting this). Added ``distinct`` method to ``count`` queries ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 29f4c2ffd5a9f297779898ca903224d4052a8c87 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Tue, 20 Jun 2023 18:24:01 +0200 Subject: [PATCH 488/727] format the json nicely when dumping fixtures (#848) --- piccolo/apps/fixtures/commands/dump.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piccolo/apps/fixtures/commands/dump.py b/piccolo/apps/fixtures/commands/dump.py index 59c563162..5d79d5562 100644 --- a/piccolo/apps/fixtures/commands/dump.py +++ b/piccolo/apps/fixtures/commands/dump.py @@ -62,7 +62,7 @@ async def dump_to_json_string( pydantic_model = create_pydantic_fixture_model( fixture_configs=fixture_configs ) - return pydantic_model(**dump).json() + return pydantic_model(**dump).json(indent=4) def parse_args(apps: str, tables: str) -> t.List[FixtureConfig]: From ea594203daf8263fb43474eb99d1be4b9818b4bf Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 28 Jun 2023 23:08:47 +0100 Subject: [PATCH 489/727] `PROTECTED_TABLENAMES` just triggers a warning (#852) * `PROTECTED_TABLENAMES` just triggers a warning * added tests --- piccolo/table.py | 9 +++++---- tests/table/test_create_table_class.py | 21 ++++++++++++++------- tests/table/test_metaclass.py | 12 +++++++++--- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/piccolo/table.py b/piccolo/table.py index b0aeb7687..ddbe04fa6 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -58,6 +58,10 @@ from piccolo.columns import Selectable PROTECTED_TABLENAMES = ("user",) +TABLENAME_WARNING = ( + "We recommend giving your table a different name as `{tablename}` is a " + "reserved keyword. It should still work, but avoid if possible." +) TABLE_REGISTRY: t.List[t.Type[Table]] = [] @@ -257,10 +261,7 @@ def __init_subclass__( schema, tablename = tablename.split(".", maxsplit=1) if tablename in PROTECTED_TABLENAMES: - raise ValueError( - f"{tablename} is a protected name, please give your table a " - "different name." - ) + warnings.warn(TABLENAME_WARNING.format(tablename=tablename)) columns: t.List[Column] = [] default_columns: t.List[Column] = [] diff --git a/tests/table/test_create_table_class.py b/tests/table/test_create_table_class.py index 9c6f013d3..ef18f6b17 100644 --- a/tests/table/test_create_table_class.py +++ b/tests/table/test_create_table_class.py @@ -1,7 +1,8 @@ from unittest import TestCase +from unittest.mock import patch from piccolo.columns import Varchar -from piccolo.table import create_table_class +from piccolo.table import TABLENAME_WARNING, create_table_class class TestCreateTableClass(TestCase): @@ -28,15 +29,21 @@ def test_protected_tablenames(self): Make sure that the logic around protected tablenames still works as expected. """ - with self.assertRaises(ValueError): + expected_warning = TABLENAME_WARNING.format(tablename="user") + + with patch("piccolo.table.warnings") as warnings: create_table_class(class_name="User") + warnings.warn.assert_called_once_with(expected_warning) - with self.assertRaises(ValueError): + with patch("piccolo.table.warnings") as warnings: create_table_class( class_name="MyUser", class_kwargs={"tablename": "user"} ) + warnings.warn.assert_called_once_with(expected_warning) - # This shouldn't raise an error: - create_table_class( - class_name="User", class_kwargs={"tablename": "my_user"} - ) + # This shouldn't output a warning: + with patch("piccolo.table.warnings") as warnings: + create_table_class( + class_name="User", class_kwargs={"tablename": "my_user"} + ) + warnings.warn.assert_not_called() diff --git a/tests/table/test_metaclass.py b/tests/table/test_metaclass.py index 9b884f575..acc0176c8 100644 --- a/tests/table/test_metaclass.py +++ b/tests/table/test_metaclass.py @@ -9,7 +9,7 @@ ForeignKey, Varchar, ) -from piccolo.table import Table +from piccolo.table import TABLENAME_WARNING, Table from tests.example_apps.music.tables import Band @@ -22,16 +22,22 @@ def test_protected_table_names(self): Some tablenames are forbidden because they're reserved words in the database, and can potentially cause issues. """ - with self.assertRaises(ValueError): + expected_warning = TABLENAME_WARNING.format(tablename="user") + + with patch("piccolo.table.warnings") as warnings: class User(Table): pass - with self.assertRaises(ValueError): + warnings.warn.assert_called_with(expected_warning) + + with patch("piccolo.table.warnings") as warnings: class MyUser(Table, tablename="user"): pass + warnings.warn.assert_called_with(expected_warning) + def test_help_text(self): """ Make sure help_text can be set for the Table. From c768f1483f796df2d9cf49baef7654c96c0f8ac0 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 28 Jun 2023 23:10:33 +0100 Subject: [PATCH 490/727] re-export `WhereRaw` (#851) --- piccolo/query/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/piccolo/query/__init__.py b/piccolo/query/__init__.py index ed4ef65d4..000a47e76 100644 --- a/piccolo/query/__init__.py +++ b/piccolo/query/__init__.py @@ -1,3 +1,5 @@ +from piccolo.columns.combination import WhereRaw + from .base import Query from .methods import ( Alter, @@ -42,4 +44,5 @@ "Sum", "TableExists", "Update", + "WhereRaw", ] From 36e80f58b87822244e405736be141e68b5ae7fa4 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 28 Jun 2023 23:35:11 +0100 Subject: [PATCH 491/727] bumped version --- CHANGES.rst | 59 +++++++++++++++++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index ed54d54fa..9a12e6b01 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,65 @@ Changes ======= +0.116.0 +------- + +Fixture formatting +~~~~~~~~~~~~~~~~~~ + +When creating a fixture: + +.. code-block:: bash + + piccolo fixtures dump + +The JSON output is now nicely formatted, which is useful because we can pipe +it straight to a file, and commit it to Git without having to manually run a +formatter on it. + +.. code-block:: bash + + piccolo fixtures dump > my_fixture.json + +Thanks to @sinisaos for this. + +Protected table names +~~~~~~~~~~~~~~~~~~~~~ + +We used to raise a ``ValueError`` if a table was called ``user``. + +.. code-block:: python + + class User(Table): # ValueError! + ... + +It's because ``user`` is already used by Postgres (e.g. try ``SELECT user`` or +``SELECT * FROM user``). + +We now emit a warning instead for these reasons: + +* Piccolo wraps table names in quotes to avoid clashes with reserved keywords. +* Sometimes you're stuck with a table name from a pre-existing schema, and + can't easily rename it. + +Re-export ``WhereRaw`` +~~~~~~~~~~~~~~~~~~~~~~ + +If you want to write raw SQL in your where queries you use ``WhereRaw``: + +.. code-block:: python + + >>> Band.select().where(WhereRaw('TRIM(name) = {}', 'Pythonistas')) + +You can now import it from ``piccolo.query`` to be consistent with +``SelectRaw`` and ``OrderByRaw``. + +.. code-block:: python + + from piccolo.query import WhereRaw + +------------------------------------------------------------------------------- + 0.115.0 ------- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 07b651f95..fceb53780 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.115.0" +__VERSION__ = "0.116.0" From fb3c8eb6b9339b4246965344194385acaee4277e Mon Sep 17 00:00:00 2001 From: sinisaos Date: Fri, 7 Jul 2023 23:53:00 +0200 Subject: [PATCH 492/727] update Litestar template to use create_pydantic_model (#859) * update Litestar template to use create_pydantic_model * Update requirements.txt * pin fastapi version in new.py --------- Co-authored-by: Daniel Townsend --- piccolo/apps/asgi/commands/new.py | 2 +- .../templates/app/_litestar_app.py.jinja | 44 +++++++++++-------- .../app/home/_litestar_endpoints.py.jinja | 2 +- requirements/requirements.txt | 2 +- 4 files changed, 28 insertions(+), 22 deletions(-) diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index f4b0e16e3..df344f10a 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -12,7 +12,7 @@ SERVERS = ["uvicorn", "Hypercorn"] ROUTERS = ["starlette", "fastapi", "blacksheep", "litestar"] ROUTER_DEPENDENCIES = { - "litestar": ["litestar==2.0.0a3"], + "fastapi": ["fastapi<0.100.0"], } diff --git a/piccolo/apps/asgi/commands/templates/app/_litestar_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_litestar_app.py.jinja index 62e12c19e..6302972bb 100644 --- a/piccolo/apps/asgi/commands/templates/app/_litestar_app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/_litestar_app.py.jinja @@ -1,18 +1,27 @@ import typing as t -from piccolo.engine import engine_finder -from piccolo_admin.endpoints import create_admin +from home.endpoints import home +from home.piccolo_app import APP_CONFIG +from home.tables import Task from litestar import Litestar, asgi, delete, get, patch, post -from litestar.static_files import StaticFilesConfig -from litestar.template import TemplateConfig from litestar.contrib.jinja import JinjaTemplateEngine -from litestar.contrib.piccolo_orm import PiccoloORMPlugin from litestar.exceptions import NotFoundException +from litestar.static_files import StaticFilesConfig +from litestar.template import TemplateConfig from litestar.types import Receive, Scope, Send +from piccolo.engine import engine_finder +from piccolo.utils.pydantic import create_pydantic_model +from piccolo_admin.endpoints import create_admin -from home.endpoints import home -from home.piccolo_app import APP_CONFIG -from home.tables import Task +TaskModelIn: t.Any = create_pydantic_model( + table=Task, + model_name="TaskModelIn", +) +TaskModelOut: t.Any = create_pydantic_model( + table=Task, + include_default_columns=True, + model_name="TaskModelOut", +) # mounting Piccolo Admin @@ -22,29 +31,27 @@ async def admin(scope: "Scope", receive: "Receive", send: "Send") -> None: @get("/tasks", tags=["Task"]) -async def tasks() -> t.List[Task]: - tasks = await Task.select().order_by(Task.id, ascending=False) - return tasks +async def tasks() -> t.List[TaskModelOut]: + return await Task.select().order_by(Task.id, ascending=False) @post("/tasks", tags=["Task"]) -async def create_task(data: Task) -> Task: - task = Task(**data.to_dict()) +async def create_task(data: TaskModelIn) -> TaskModelOut: + task = Task(**data.dict()) await task.save() - return task + return task.to_dict() @patch("/tasks/{task_id:int}", tags=["Task"]) -async def update_task(task_id: int, data: Task) -> Task: +async def update_task(task_id: int, data: TaskModelIn) -> TaskModelOut: task = await Task.objects().get(Task.id == task_id) if not task: raise NotFoundException("Task does not exist") - for key, value in data.to_dict().items(): - task.id = task_id + for key, value in data.dict().items(): setattr(task, key, value) await task.save() - return task + return task.to_dict() @delete("/tasks/{task_id:int}", tags=["Task"]) @@ -80,7 +87,6 @@ app = Litestar( update_task, delete_task, ], - plugins=[PiccoloORMPlugin()], template_config=TemplateConfig( directory="home/templates", engine=JinjaTemplateEngine ), diff --git a/piccolo/apps/asgi/commands/templates/app/home/_litestar_endpoints.py.jinja b/piccolo/apps/asgi/commands/templates/app/home/_litestar_endpoints.py.jinja index b33586da5..e5dfc0661 100644 --- a/piccolo/apps/asgi/commands/templates/app/home/_litestar_endpoints.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/home/_litestar_endpoints.py.jinja @@ -10,7 +10,7 @@ ENVIRONMENT = jinja2.Environment( ) -@get(path="/", include_in_schema=False) +@get(path="/", include_in_schema=False, sync_to_thread=False) def home(request: Request) -> Response: template = ENVIRONMENT.get_template("home.html.jinja") content = template.render(title="Piccolo + ASGI") diff --git a/requirements/requirements.txt b/requirements/requirements.txt index f6dbc3d2f..dc6479810 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -4,4 +4,4 @@ Jinja2>=2.11.0 targ>=0.3.7 inflection>=0.5.1 typing-extensions>=4.3.0 -pydantic[email]>=1.6 +pydantic[email]>=1.6,<2.0 From 0cd93f009d3ab9d2c2ac8360dab3d0a3ffad2fd9 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 7 Jul 2023 22:54:29 +0100 Subject: [PATCH 493/727] bumped version --- CHANGES.rst | 11 +++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 9a12e6b01..72f9cb88a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,17 @@ Changes ======= +0.117.0 +------- + +Version pinning Pydantic to v1, as v2 has breaking changes. + +We will add support for Pydantic v2 in a future release. + +Thanks to @sinisaos for helping with this. + +------------------------------------------------------------------------------- + 0.116.0 ------- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index fceb53780..08915f68d 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.116.0" +__VERSION__ = "0.117.0" From 97292ccc27c379e17e9e964100a9da804055430b Mon Sep 17 00:00:00 2001 From: KekmaTime <136650032+KekmaTime@users.noreply.github.com> Date: Sat, 8 Jul 2023 12:29:50 +0530 Subject: [PATCH 494/727] Drop Python 3.7 support (#855) * updated-release.yaml * updated-tests.yaml * updated-steup.py * typing-changed * run isort --------- Co-authored-by: Daniel Townsend --- .github/workflows/release.yaml | 2 +- .github/workflows/tests.yaml | 14 +++++++------- piccolo/apps/schema/commands/generate.py | 9 ++++----- piccolo/columns/column_types.py | 6 ++---- piccolo/query/methods/insert.py | 4 +--- piccolo/query/mixins.py | 4 +--- requirements/test-requirements.txt | 1 + setup.py | 3 +-- 8 files changed, 18 insertions(+), 25 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c4c643bd9..806b1de20 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -14,7 +14,7 @@ jobs: - uses: "actions/checkout@v3" - uses: "actions/setup-python@v1" with: - python-version: 3.7 + python-version: 3.11 - name: "Install dependencies" run: "pip install -r requirements/dev-requirements.txt" - name: "Publish to PyPI" diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 4b7c7519a..fca6144c9 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -16,7 +16,7 @@ jobs: timeout-minutes: 30 strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 @@ -85,7 +85,7 @@ jobs: timeout-minutes: 30 strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11"] postgres-version: [10, 11, 12, 13, 14, 15] # Service containers to run with `container-job` @@ -134,14 +134,14 @@ jobs: PG_PASSWORD: postgres - name: Upload coverage uses: codecov/codecov-action@v1 - if: matrix.python-version == '3.7' + if: matrix.python-version == '3.11' cockroach: runs-on: ubuntu-latest timeout-minutes: 30 strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11"] cockroachdb-version: ["v22.2.0"] steps: - uses: actions/checkout@v3 @@ -168,14 +168,14 @@ jobs: PG_DATABASE: piccolo - name: Upload coverage uses: codecov/codecov-action@v1 - if: matrix.python-version == '3.7' + if: matrix.python-version == '3.11' sqlite: runs-on: ubuntu-latest timeout-minutes: 30 strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 @@ -193,4 +193,4 @@ jobs: run: ./scripts/test-sqlite.sh - name: Upload coverage uses: codecov/codecov-action@v1 - if: matrix.python-version == '3.7' + if: matrix.python-version == '3.11' diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index 3cbe729e4..2f2fc9596 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -10,7 +10,6 @@ from datetime import date, datetime import black -from typing_extensions import Literal from piccolo.apps.migrations.auto.serialisation import serialise_params from piccolo.apps.schema.commands.exceptions import GenerateError @@ -62,13 +61,13 @@ class ConstraintTable: class RowMeta: column_default: str column_name: str - is_nullable: Literal["YES", "NO"] + is_nullable: t.Literal["YES", "NO"] table_name: str character_maximum_length: t.Optional[int] data_type: str numeric_precision: t.Optional[t.Union[int, str]] numeric_scale: t.Optional[t.Union[int, str]] - numeric_precision_radix: t.Optional[Literal[2, 10]] + numeric_precision_radix: t.Optional[t.Literal[2, 10]] @classmethod def get_column_name_str(cls) -> str: @@ -77,7 +76,7 @@ def get_column_name_str(cls) -> str: @dataclasses.dataclass class Constraint: - constraint_type: Literal["PRIMARY KEY", "UNIQUE", "FOREIGN KEY", "CHECK"] + constraint_type: t.Literal["PRIMARY KEY", "UNIQUE", "FOREIGN KEY", "CHECK"] constraint_name: str constraint_schema: t.Optional[str] = None column_name: t.Optional[str] = None @@ -141,7 +140,7 @@ class Trigger: table_name: str column_name: str on_update: str - on_delete: Literal[ + on_delete: t.Literal[ "NO ACTION", "RESTRICT", "CASCADE", "SET NULL", "SET_DEFAULT" ] references_table: str diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 855e8cae0..9934eebed 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -35,8 +35,6 @@ class Band(Table): from datetime import date, datetime, time, timedelta from enum import Enum -from typing_extensions import Literal - from piccolo.columns.base import Column, ForeignKeyMeta, OnDelete, OnUpdate from piccolo.columns.combination import Where from piccolo.columns.defaults.date import DateArg, DateCustom, DateNow @@ -131,7 +129,7 @@ class MathDelegate: def get_querystring( self, column_name: str, - operator: Literal["+", "-", "/", "*"], + operator: t.Literal["+", "-", "/", "*"], value: t.Union[int, float, Integer], reverse: bool = False, ) -> QueryString: @@ -227,7 +225,7 @@ def get_sqlite_interval_string(self, interval: timedelta) -> str: def get_querystring( self, column: Column, - operator: Literal["+", "-"], + operator: t.Literal["+", "-"], value: timedelta, engine_type: str, ) -> QueryString: diff --git a/piccolo/query/methods/insert.py b/piccolo/query/methods/insert.py index 2963d397a..5c06169d8 100644 --- a/piccolo/query/methods/insert.py +++ b/piccolo/query/methods/insert.py @@ -2,8 +2,6 @@ import typing as t -from typing_extensions import Literal - from piccolo.custom_types import Combinable, TableInstance from piccolo.query.base import Query from piccolo.query.mixins import ( @@ -48,7 +46,7 @@ def on_conflict( self: Self, target: t.Optional[t.Union[str, Column, t.Tuple[Column, ...]]] = None, action: t.Union[ - OnConflictAction, Literal["DO NOTHING", "DO UPDATE"] + OnConflictAction, t.Literal["DO NOTHING", "DO UPDATE"] ] = OnConflictAction.do_nothing, values: t.Optional[ t.Sequence[t.Union[Column, t.Tuple[Column, t.Any]]] diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index 43d126436..ac474b906 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -7,8 +7,6 @@ from dataclasses import dataclass, field from enum import Enum, auto -from typing_extensions import Literal - from piccolo.columns import And, Column, Or, Where from piccolo.columns.column_types import ForeignKey from piccolo.custom_types import Combinable @@ -761,7 +759,7 @@ def on_conflict( self, target: t.Optional[t.Union[str, Column, t.Tuple[Column, ...]]] = None, action: t.Union[ - OnConflictAction, Literal["DO NOTHING", "DO UPDATE"] + OnConflictAction, t.Literal["DO NOTHING", "DO UPDATE"] ] = OnConflictAction.do_nothing, values: t.Optional[ t.Sequence[t.Union[Column, t.Tuple[Column, t.Any]]] diff --git a/requirements/test-requirements.txt b/requirements/test-requirements.txt index 2ab821f3f..2f350ed31 100644 --- a/requirements/test-requirements.txt +++ b/requirements/test-requirements.txt @@ -2,3 +2,4 @@ coveralls==3.3.1 pytest-cov==3.0.0 pytest==6.2.5 python-dateutil==2.8.2 +typing-extensions>=4.3.0 diff --git a/setup.py b/setup.py index fa8d25ac3..155768811 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ def extras_require() -> t.Dict[str, t.List[str]]: long_description_content_type="text/markdown", author="Daniel Townsend", author_email="dan@dantownsend.co.uk", - python_requires=">=3.7.0", + python_requires=">=3.8.0", url="https://github.com/piccolo-orm/piccolo", packages=find_packages(exclude=("tests",)), package_data={ @@ -84,7 +84,6 @@ def extras_require() -> t.Dict[str, t.List[str]]: "License :: OSI Approved :: MIT License", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", From a3a6b875bff2c813e16cb36dcbaec8d76aec9fd8 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 13 Jul 2023 15:13:25 +0100 Subject: [PATCH 495/727] update docs for running migrations (#863) --- docs/src/piccolo/migrations/running.rst | 70 +++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 4 deletions(-) diff --git a/docs/src/piccolo/migrations/running.rst b/docs/src/piccolo/migrations/running.rst index 278d48ca5..74bc9cd50 100644 --- a/docs/src/piccolo/migrations/running.rst +++ b/docs/src/piccolo/migrations/running.rst @@ -4,9 +4,6 @@ Running migrations .. hint:: To see all available options for these commands, use the ``--help`` flag, for example ``piccolo migrations forwards --help``. -.. hint:: To see the SQL queries of a migration without actually running them , use the ``--preview`` - flag, for example: ``piccolo migrations forwards my_app --preview`` or ``piccolo migrations backwards 2018-09-04T19:44:09 --preview``. - Forwards -------- @@ -16,6 +13,30 @@ When the migration is run, the forwards function is executed. To do this: piccolo migrations forwards my_app + +Multiple apps +~~~~~~~~~~~~~ + +If you have multiple apps you can run them all using: + +.. code-block:: bash + + piccolo migrations forwards all + +Fake +~~~~ + +We can 'fake' running a migration - we record that it ran in the database +without actually running it. + +.. code-block:: bash + + piccolo migrations forwards my_app 2022-09-04T19:44:09 --fake + +This is useful if we started from an existing database using +``piccolo schema generate``, and the initial migration we generated is for +tables which already exist, hence we fake run it. + ------------------------------------------------------------------------------- Reversing migrations @@ -26,7 +47,7 @@ migration: .. code-block:: bash - piccolo migrations backwards my_app 2018-09-04T19:44:09 + piccolo migrations backwards my_app 2022-09-04T19:44:09 Piccolo will then reverse the migrations for the given app, starting with the most recent migration, up to and including the migration with the specified ID. @@ -36,6 +57,26 @@ expected. ------------------------------------------------------------------------------- +Preview +------- + +To see the SQL queries of a migration without actually running them, use the +``--preview`` flag. + +This works when running migrations forwards: + +.. code-block:: bash + + piccolo migrations forwards my_app --preview + +Or backwards: + +.. code-block:: bash + + piccolo migrations backwards 2022-09-04T19:44:09 --preview + +------------------------------------------------------------------------------- + Checking migrations ------------------- @@ -44,3 +85,24 @@ You can easily check which migrations have and haven't ran using the following: .. code-block:: bash piccolo migrations check + +------------------------------------------------------------------------------- + +Source +------ + +These are the underlying Python functions which are called, so you can see +all available options. These functions are convered into a CI using +`targ `_. + +.. currentmodule:: piccolo.apps.migrations.commands.forwards + +.. autofunction:: forwards + +.. currentmodule:: piccolo.apps.migrations.commands.backwards + +.. autofunction:: backwards + +.. currentmodule:: piccolo.apps.migrations.commands.check + +.. autofunction:: check From 2e2327187f8a29e74da5ae1f7961ccaa985f1f18 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 14 Jul 2023 12:30:12 +0100 Subject: [PATCH 496/727] 862: allow `piccolo migrations new all --auto` (#864) * allow `piccolo migrations new all --auto` * remove unused import * fix old docstring * make sure the correct app config is being used * rearrange print statements * improve UI when making migrations * add test for app sorting --- piccolo/apps/migrations/commands/backwards.py | 4 +- piccolo/apps/migrations/commands/forwards.py | 4 +- piccolo/apps/migrations/commands/new.py | 49 +++++++--- piccolo/conf/apps.py | 97 +++++++++++++------ piccolo/utils/printing.py | 12 ++- tests/apps/migrations/commands/test_new.py | 47 +++++++-- tests/conf/test_apps.py | 34 +++++++ 7 files changed, 191 insertions(+), 56 deletions(-) diff --git a/piccolo/apps/migrations/commands/backwards.py b/piccolo/apps/migrations/commands/backwards.py index 464612011..a0a454d90 100644 --- a/piccolo/apps/migrations/commands/backwards.py +++ b/piccolo/apps/migrations/commands/backwards.py @@ -11,6 +11,7 @@ ) from piccolo.apps.migrations.tables import Migration from piccolo.conf.apps import AppConfig, MigrationModule +from piccolo.utils.printing import print_heading class BackwardsMigrationManager(BaseMigrationManager): @@ -136,8 +137,7 @@ async def run_backwards( if _continue not in "yY": return MigrationResult(success=False, message="user cancelled") for _app_name in sorted_app_names: - print(f"\n{_app_name.upper():^64}") - print("-" * 64) + print_heading(_app_name) manager = BackwardsMigrationManager( app_name=_app_name, migration_id="all", diff --git a/piccolo/apps/migrations/commands/forwards.py b/piccolo/apps/migrations/commands/forwards.py index b284bd43a..f060b493a 100644 --- a/piccolo/apps/migrations/commands/forwards.py +++ b/piccolo/apps/migrations/commands/forwards.py @@ -10,6 +10,7 @@ ) from piccolo.apps.migrations.tables import Migration from piccolo.conf.apps import AppConfig, MigrationModule +from piccolo.utils.printing import print_heading class ForwardsMigrationManager(BaseMigrationManager): @@ -109,8 +110,7 @@ async def run_forwards( if app_name == "all": sorted_app_names = BaseMigrationManager().get_sorted_app_names() for _app_name in sorted_app_names: - print(f"\n{_app_name.upper():^64}") - print("-" * 64) + print_heading(_app_name) manager = ForwardsMigrationManager( app_name=_app_name, migration_id="all", diff --git a/piccolo/apps/migrations/commands/new.py b/piccolo/apps/migrations/commands/new.py index a9a23cc0f..09f6ac9e8 100644 --- a/piccolo/apps/migrations/commands/new.py +++ b/piccolo/apps/migrations/commands/new.py @@ -21,6 +21,7 @@ ) from piccolo.conf.apps import AppConfig, Finder from piccolo.engine import SQLiteEngine +from piccolo.utils.printing import print_heading from .base import BaseMigrationManager @@ -217,7 +218,8 @@ async def new( Creates a new migration file in the migrations folder. :param app_name: - The app to create a migration for. + The app to create a migration for. Specify a value of 'all' to create + migrations for all apps (use in conjunction with --auto). :param auto: Auto create the migration contents. :param desc: @@ -228,22 +230,41 @@ async def new( entered. For example, --auto_input='y'. """ - print("Creating new migration ...") - engine = Finder().get_engine() if auto and isinstance(engine, SQLiteEngine): sys.exit("Auto migrations aren't currently supported by SQLite.") - app_config = Finder().get_app_config(app_name=app_name) + if app_name == "all" and not auto: + raise ValueError( + "Only use `--app_name=all` in conjunction with `--auto`." + ) - _create_migrations_folder(app_config.migrations_folder_path) - try: - await _create_new_migration( - app_config=app_config, - auto=auto, - description=desc, - auto_input=auto_input, + app_names = ( + sorted( + BaseMigrationManager().get_app_names( + sort_by_migration_dependencies=False + ) ) - except NoChanges: - print("No changes detected - exiting.") - sys.exit(0) + if app_name == "all" + else [app_name] + ) + + for app_name in app_names: + print_heading(app_name) + print("🚀 Creating new migration ...") + + app_config = Finder().get_app_config(app_name=app_name) + + _create_migrations_folder(app_config.migrations_folder_path) + + try: + await _create_new_migration( + app_config=app_config, + auto=auto, + description=desc, + auto_input=auto_input, + ) + except NoChanges: + print("🏁 No changes detected.") + + print("\n✅ Finished\n") diff --git a/piccolo/conf/apps.py b/piccolo/conf/apps.py index 6176224af..b88d50ba2 100644 --- a/piccolo/conf/apps.py +++ b/piccolo/conf/apps.py @@ -1,6 +1,5 @@ from __future__ import annotations -import functools import inspect import itertools import os @@ -13,6 +12,7 @@ from piccolo.engine.base import Engine from piccolo.table import Table +from piccolo.utils.graphlib import TopologicalSorter from piccolo.utils.warnings import Level, colored_warning @@ -159,6 +159,10 @@ def __post_init__(self): if isinstance(self.migrations_folder_path, pathlib.Path): self.migrations_folder_path = str(self.migrations_folder_path) + self._migration_dependency_app_configs: t.Optional[ + t.List[AppConfig] + ] = None + def register_table(self, table_class: t.Type[Table]): self.table_classes.append(table_class) return table_class @@ -166,19 +170,27 @@ def register_table(self, table_class: t.Type[Table]): @property def migration_dependency_app_configs(self) -> t.List[AppConfig]: """ - Get all of the AppConfig instances from this app's migration + Get all of the ``AppConfig`` instances from this app's migration dependencies. """ - modules: t.List[PiccoloAppModule] = [ - t.cast(PiccoloAppModule, import_module(module_path)) - for module_path in self.migration_dependencies - ] - return [i.APP_CONFIG for i in modules] + # We cache the value so it's more efficient, and also so we can set the + # underlying value in unit tests for easier mocking. + if self._migration_dependency_app_configs is None: + + modules: t.List[PiccoloAppModule] = [ + t.cast(PiccoloAppModule, import_module(module_path)) + for module_path in self.migration_dependencies + ] + self._migration_dependency_app_configs = [ + i.APP_CONFIG for i in modules + ] + + return self._migration_dependency_app_configs def get_table_with_name(self, table_class_name: str) -> t.Type[Table]: """ - Returns a Table subclass with the given name from this app, if it - exists. Otherwise raises a ValueError. + Returns a ``Table`` subclass with the given name from this app, if it + exists. Otherwise raises a ``ValueError``. """ filtered = [ table_class @@ -426,44 +438,69 @@ def get_app_modules(self) -> t.List[PiccoloAppModule]: return app_modules - def get_app_names(self, sort: bool = True) -> t.List[str]: + def get_app_names( + self, sort_by_migration_dependencies: bool = True + ) -> t.List[str]: """ Return all of the app names. - :param sort: + :param sort_by_migration_dependencies: If True, sorts the app names using the migration dependencies, so dependencies are before dependents in the list. """ - modules = self.get_app_modules() - configs: t.List[AppConfig] = [module.APP_CONFIG for module in modules] + return [ + i.app_name + for i in self.get_app_configs( + sort_by_migration_dependencies=sort_by_migration_dependencies + ) + ] + + def get_sorted_app_names(self) -> t.List[str]: + """ + Just here for backwards compatibility - use ``get_app_names`` directly. + """ + return self.get_app_names(sort_by_migration_dependencies=True) - if not sort: - return [i.app_name for i in configs] + def sort_app_configs( + self, app_configs: t.List[AppConfig] + ) -> t.List[AppConfig]: + app_config_map = { + app_config.app_name: app_config for app_config in app_configs + } - def sort_app_configs(app_config_1: AppConfig, app_config_2: AppConfig): - return ( - app_config_1 in app_config_2.migration_dependency_app_configs - ) + sorted_app_names = TopologicalSorter( + { + app_config.app_name: [ + i.app_name + for i in app_config.migration_dependency_app_configs + ] + for app_config in app_config_map.values() + } + ).static_order() - sorted_configs = sorted( - configs, key=functools.cmp_to_key(sort_app_configs) - ) - return [i.app_name for i in sorted_configs] + return [app_config_map[i] for i in sorted_app_names] - def get_sorted_app_names(self) -> t.List[str]: + def get_app_configs( + self, sort_by_migration_dependencies: bool = True + ) -> t.List[AppConfig]: """ - Just here for backwards compatibility. + Returns a list of ``AppConfig``, optionally sorted by migration + dependencies. """ - return self.get_app_names(sort=True) + app_configs = [i.APP_CONFIG for i in self.get_app_modules()] + + return ( + self.sort_app_configs(app_configs=app_configs) + if sort_by_migration_dependencies + else app_configs + ) def get_app_config(self, app_name: str) -> AppConfig: """ Returns an ``AppConfig`` for the given app name. """ - modules = self.get_app_modules() - for module in modules: - app_config = module.APP_CONFIG + for app_config in self.get_app_configs(): if app_config.app_name == app_name: return app_config raise ValueError(f"No app found with name {app_name}") @@ -487,7 +524,7 @@ def get_table_classes( ) -> t.List[t.Type[Table]]: """ Returns all ``Table`` classes registered with the given apps. If - ``app_names`` is ``None``, then ``Table`` classes will be returned + ``include_apps`` is ``None``, then ``Table`` classes will be returned for all apps. """ if include_apps and exclude_apps: diff --git a/piccolo/utils/printing.py b/piccolo/utils/printing.py index 258594d71..c6adc76dd 100644 --- a/piccolo/utils/printing.py +++ b/piccolo/utils/printing.py @@ -1,8 +1,18 @@ def get_fixed_length_string(string: str, length=20) -> str: """ - Add spacing to the end of the string so it's a fixed length. + Add spacing to the end of the string so it's a fixed length, or truncate + if it's too long. """ if len(string) > length: return f"{string[: length - 3]}..." spacing = "".join(" " for _ in range(length - len(string))) return f"{string}{spacing}" + + +def print_heading(string: str, width: int = 64) -> None: + """ + Prints out a nicely formatted heading to the console. Useful for breaking + up the output in large CLI commands. + """ + print(f"\n{string.upper():^{width}}") + print("-" * width) diff --git a/tests/apps/migrations/commands/test_new.py b/tests/apps/migrations/commands/test_new.py index 08c549ebd..da47877c1 100644 --- a/tests/apps/migrations/commands/test_new.py +++ b/tests/apps/migrations/commands/test_new.py @@ -18,7 +18,7 @@ class TestNewMigrationCommand(TestCase): - def test_create_new_migration(self): + def test_manual(self): """ Create a manual migration (i.e. non-auto). """ @@ -46,17 +46,50 @@ def test_create_new_migration(self): @engines_only("postgres") @patch("piccolo.apps.migrations.commands.new.print") - def test_new_command(self, print_: MagicMock): + def test_auto(self, print_: MagicMock): """ Call the command, when no migration changes are needed. """ - with self.assertRaises(SystemExit) as manager: - run_sync(new(app_name="music", auto=True)) + run_sync(new(app_name="music", auto=True)) - self.assertEqual(manager.exception.code, 0) + self.assertListEqual( + print_.call_args_list, + [ + call("🚀 Creating new migration ..."), + call("🏁 No changes detected."), + call("\n✅ Finished\n"), + ], + ) + + @engines_only("postgres") + @patch("piccolo.apps.migrations.commands.new.print") + def test_auto_all(self, print_: MagicMock): + """ + Try auto migrating all apps. + """ + run_sync(new(app_name="all", auto=True)) + self.assertListEqual( + print_.call_args_list, + [ + call("🚀 Creating new migration ..."), + call("🏁 No changes detected."), + call("🚀 Creating new migration ..."), + call("🏁 No changes detected."), + call("\n✅ Finished\n"), + ], + ) - self.assertTrue( - print_.mock_calls[-1] == call("No changes detected - exiting.") + @engines_only("postgres") + def test_auto_all_error(self): + """ + Call the command, when no migration changes are needed. + """ + with self.assertRaises(ValueError) as manager: + run_sync(new(app_name="all", auto=False)) + + self.assertEqual( + manager.exception.__str__(), + "Only use `--app_name=all` in conjunction with `--auto`.", ) diff --git a/tests/conf/test_apps.py b/tests/conf/test_apps.py index 6df249e37..907064e1f 100644 --- a/tests/conf/test_apps.py +++ b/tests/conf/test_apps.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import pathlib from unittest import TestCase @@ -270,3 +272,35 @@ def test_get_table_classes(self): finder.get_table_classes( exclude_apps=["music"], include_apps=["mega"] ) + + def test_sort_app_configs(self): + """ + Make sure we can sort ``AppConfig`` based on their migration + dependencies. + """ + app_config_1 = AppConfig( + app_name="app_1", + migrations_folder_path="", + ) + + app_config_1._migration_dependency_app_configs = [ + AppConfig( + app_name="app_2", + migrations_folder_path="", + ) + ] + + app_config_2 = AppConfig( + app_name="app_2", + migrations_folder_path="", + ) + + app_config_2._migration_dependency_app_configs = [] + + sorted_app_configs = Finder().sort_app_configs( + [app_config_2, app_config_1] + ) + + self.assertListEqual( + [i.app_name for i in sorted_app_configs], ["app_2", "app_1"] + ) From 056e02ad72ea2a7e5d62d0566704e34b3a66d504 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 14 Jul 2023 12:54:17 +0100 Subject: [PATCH 497/727] bumped version --- CHANGES.rst | 15 +++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 72f9cb88a..a905b60d9 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,21 @@ Changes ======= +0.118.0 +------- + +If you have lots of Piccolo apps, you can now create auto migrations for them +all in one go: + +.. code-block:: bash + + piccolo migrations new all --auto + +The documentation for running migrations has also been improved, as well as +improvements to the sorting of migrations based on their dependencies. + +------------------------------------------------------------------------------- + 0.117.0 ------- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 08915f68d..5907877d0 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.117.0" +__VERSION__ = "0.118.0" From aa49dd7bc7c8f7b42eac73bdba487f8dad9ca58f Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 14 Jul 2023 12:58:28 +0100 Subject: [PATCH 498/727] Update CHANGES.rst --- CHANGES.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index a905b60d9..7c0c61f62 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -11,6 +11,8 @@ all in one go: piccolo migrations new all --auto +Thanks to @hoosnick for suggesting this new feature. + The documentation for running migrations has also been improved, as well as improvements to the sorting of migrations based on their dependencies. From c8c7df571d37f7c880ccd14e2965b591db7570f2 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 16 Jul 2023 22:51:27 +0100 Subject: [PATCH 499/727] update changelog to mention python 3.7 support being removed --- CHANGES.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 7c0c61f62..c8b4cd045 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -16,6 +16,8 @@ Thanks to @hoosnick for suggesting this new feature. The documentation for running migrations has also been improved, as well as improvements to the sorting of migrations based on their dependencies. +Support for Python 3.7 was dropped in this release as it's now end of life. + ------------------------------------------------------------------------------- 0.117.0 From 7f001ccb8e45aaa0e9f5f5e0db2ebbd44c9b8d51 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 21 Jul 2023 14:43:18 +0100 Subject: [PATCH 500/727] fix `LazyTableReference` with `ModelBuilder` (#870) --- piccolo/testing/model_builder.py | 6 +- tests/testing/test_model_builder.py | 139 ++++++++++++++-------------- 2 files changed, 75 insertions(+), 70 deletions(-) diff --git a/piccolo/testing/model_builder.py b/piccolo/testing/model_builder.py index c2bb38007..180c0397c 100644 --- a/piccolo/testing/model_builder.py +++ b/piccolo/testing/model_builder.py @@ -6,7 +6,7 @@ from decimal import Decimal from uuid import UUID -from piccolo.columns import Array, Column +from piccolo.columns import Array, Column, ForeignKey from piccolo.custom_types import TableInstance from piccolo.testing.random_builder import RandomBuilder from piccolo.utils.sync import run_sync @@ -122,9 +122,9 @@ async def _build( if column._meta.name in defaults: continue # Column value exists - if "references" in column._meta.params and persist: + if isinstance(column, ForeignKey) and persist: reference_model = await cls._build( - column._meta.params["references"], + column._foreign_key_meta.resolved_references, persist=True, ) random_value = getattr( diff --git a/tests/testing/test_model_builder.py b/tests/testing/test_model_builder.py index fe6ee77a6..bcc6b9424 100644 --- a/tests/testing/test_model_builder.py +++ b/tests/testing/test_model_builder.py @@ -1,8 +1,18 @@ import asyncio +import typing as t import unittest -from piccolo.columns import Array, Decimal, Integer, Numeric, Real, Varchar -from piccolo.table import Table +from piccolo.columns import ( + Array, + Decimal, + ForeignKey, + Integer, + LazyTableReference, + Numeric, + Real, + Varchar, +) +from piccolo.table import Table, create_db_tables_sync, drop_db_tables_sync from piccolo.testing.model_builder import ModelBuilder from tests.base import engines_skip from tests.example_apps.music.tables import ( @@ -30,62 +40,52 @@ class TableWithDecimal(Table): decimal_with_digits = Decimal(digits=(4, 2)) +class BandWithLazyReference(Table): + manager = ForeignKey( + references=LazyTableReference( + "Manager", module_path="tests.example_apps.music.tables" + ) + ) + + +TABLES = ( + Manager, + Band, + Poster, + RecordingStudio, + Shirt, + Venue, + Concert, + Ticket, + TableWithArrayField, + TableWithDecimal, + BandWithLazyReference, +) + + # Cockroach Bug: Can turn ON when resolved: https://github.com/cockroachdb/cockroach/issues/71908 # noqa: E501 @engines_skip("cockroach") class TestModelBuilder(unittest.TestCase): @classmethod def setUpClass(cls): - - for table_class in ( - Manager, - Band, - Poster, - RecordingStudio, - Shirt, - Venue, - Concert, - Ticket, - TableWithArrayField, - TableWithDecimal, - ): - table_class.create_table().run_sync() + create_db_tables_sync(*TABLES) @classmethod def tearDownClass(cls) -> None: - for table_class in ( - TableWithDecimal, - TableWithArrayField, - Ticket, - Concert, - Venue, - Shirt, - RecordingStudio, - Poster, - Band, - Manager, - ): - table_class.alter().drop_table().run_sync() - - def test_model_builder_async(self): - async def build_model(model): - return await ModelBuilder.build(model) - - asyncio.run(build_model(Manager)) - asyncio.run(build_model(Ticket)) - asyncio.run(build_model(Poster)) - asyncio.run(build_model(RecordingStudio)) - asyncio.run(build_model(TableWithArrayField)) - asyncio.run(build_model(TableWithDecimal)) - - def test_model_builder_sync(self): - ModelBuilder.build_sync(Manager) - ModelBuilder.build_sync(Ticket) - ModelBuilder.build_sync(Poster) - ModelBuilder.build_sync(RecordingStudio) - ModelBuilder.build_sync(TableWithArrayField) - ModelBuilder.build_sync(TableWithDecimal) - - def test_model_builder_with_choices(self): + drop_db_tables_sync(*TABLES) + + def test_async(self): + async def build_model(table_class: t.Type[Table]): + return await ModelBuilder.build(table_class) + + for table_class in TABLES: + asyncio.run(build_model(table_class)) + + def test_sync(self): + for table_class in TABLES: + ModelBuilder.build_sync(table_class) + + def test_choices(self): shirt = ModelBuilder.build_sync(Shirt) queried_shirt = ( Shirt.objects().where(Shirt.id == shirt.id).first().run_sync() @@ -96,30 +96,35 @@ def test_model_builder_with_choices(self): ["s", "l", "m"], ) - def test_model_builder_with_foreign_key(self): - ModelBuilder.build_sync(Band) + def test_foreign_key(self): + model = ModelBuilder.build_sync(Band, persist=True) - def test_model_builder_with_invalid_column(self): + self.assertTrue( + Manager.exists().where(Manager.id == model.manager).run_sync() + ) + + def test_lazy_foreign_key(self): + model = ModelBuilder.build_sync(BandWithLazyReference, persist=True) + + self.assertTrue( + Manager.exists().where(Manager.id == model.manager).run_sync() + ) + + def test_invalid_column(self): with self.assertRaises(ValueError): ModelBuilder.build_sync(Band, defaults={"X": 1}) - def test_model_builder_with_minimal(self): + def test_minimal(self): band = ModelBuilder.build_sync(Band, minimal=True) - self.assertEqual( - Band.exists().where(Band.id == band.id).run_sync(), - True, - ) + self.assertTrue(Band.exists().where(Band.id == band.id).run_sync()) - def test_model_builder_with_no_persist(self): + def test_persist_false(self): band = ModelBuilder.build_sync(Band, persist=False) - self.assertEqual( - Band.exists().where(Band.id == band.id).run_sync(), - False, - ) + self.assertFalse(Band.exists().where(Band.id == band.id).run_sync()) - def test_model_builder_with_valid_column(self): + def test_valid_column(self): manager = ModelBuilder.build_sync( Manager, defaults={Manager.name: "Guido"} ) @@ -133,7 +138,7 @@ def test_model_builder_with_valid_column(self): self.assertEqual(queried_manager.name, "Guido") - def test_model_builder_with_valid_column_string(self): + def test_valid_column_string(self): manager = ModelBuilder.build_sync(Manager, defaults={"name": "Guido"}) queried_manager = ( @@ -145,14 +150,14 @@ def test_model_builder_with_valid_column_string(self): self.assertEqual(queried_manager.name, "Guido") - def test_model_builder_with_valid_foreign_key(self): + def test_valid_foreign_key(self): manager = ModelBuilder.build_sync(Manager) band = ModelBuilder.build_sync(Band, defaults={Band.manager: manager}) self.assertEqual(manager._meta.primary_key, band.manager) - def test_model_builder_with_valid_foreign_key_string(self): + def test_valid_foreign_key_string(self): manager = ModelBuilder.build_sync(Manager) band = ModelBuilder.build_sync(Band, defaults={"manager": manager}) From 1d2a224aa471daec21803badc789f85628f50e3d Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 21 Jul 2023 15:05:10 +0100 Subject: [PATCH 501/727] bumped version --- CHANGES.rst | 26 ++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index c8b4cd045..85dd46a27 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,32 @@ Changes ======= +0.119.0 +------- + +``ModelBuilder`` now works with ``LazyTableReference`` (which is used when we +have circular references caused by a ``ForeignKey``). + +With this table: + +.. code-block:: python + + class Band(Table): + manager = ForeignKey( + LazyTableReference( + 'Manager', + module_path='some.other.folder.tables' + ) + ) + +We can now create a dynamic test fixture: + +.. code-block:: python + + my_model = await ModelBuilder.build(Band) + +------------------------------------------------------------------------------- + 0.118.0 ------- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 5907877d0..ca94cfb55 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.118.0" +__VERSION__ = "0.119.0" From b13c383247532352752828bb41aa8ec830a0dd67 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 26 Jul 2023 19:45:17 +0100 Subject: [PATCH 502/727] fix assertions (#872) --- tests/columns/foreign_key/test_attribute_access.py | 9 +++++---- tests/columns/foreign_key/test_foreign_key_string.py | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/columns/foreign_key/test_attribute_access.py b/tests/columns/foreign_key/test_attribute_access.py index c658d7d98..87caeb78e 100644 --- a/tests/columns/foreign_key/test_attribute_access.py +++ b/tests/columns/foreign_key/test_attribute_access.py @@ -21,7 +21,8 @@ class BandB(Table): class BandC(Table): manager = ForeignKey( references=LazyTableReference( - table_class_name="Manager", module_path=__name__ + table_class_name="Manager", + module_path=__name__, ) ) @@ -37,7 +38,7 @@ def test_attribute_access(self): references. """ for band_table in (BandA, BandB, BandC, BandD): - self.assertTrue(isinstance(band_table.manager.name, Varchar)) + self.assertIsInstance(band_table.manager.name, Varchar) def test_recursion_limit(self): """ @@ -47,7 +48,7 @@ def test_recursion_limit(self): # Should be fine: column: Column = Manager.manager.name self.assertTrue(len(column._meta.call_chain), 1) - self.assertTrue(isinstance(column, Varchar)) + self.assertIsInstance(column, Varchar) with self.assertRaises(Exception): Manager.manager.manager.manager.manager.manager.manager.manager.manager.manager.manager.manager.name # noqa @@ -59,4 +60,4 @@ def test_recursion_time(self): start = time.time() Manager.manager.manager.manager.manager.manager.manager.name end = time.time() - self.assertTrue(end - start < 1.0) + self.assertLess(end - start, 1.0) diff --git a/tests/columns/foreign_key/test_foreign_key_string.py b/tests/columns/foreign_key/test_foreign_key_string.py index 38d9e6bee..1dd8e3aee 100644 --- a/tests/columns/foreign_key/test_foreign_key_string.py +++ b/tests/columns/foreign_key/test_foreign_key_string.py @@ -33,7 +33,7 @@ class TestForeignKeyString(TestCase): def test_foreign_key_string(self): for band_table in (BandA, BandB, BandC): - self.assertEqual( + self.assertIs( band_table.manager._foreign_key_meta.resolved_references, Manager, ) @@ -66,4 +66,4 @@ def test_lazy_reference_to_app(self): table_class_name="Manager", app_name="music" ) - self.assertTrue(reference.resolve() is Manager) + self.assertIs(reference.resolve(), Manager) From 83ea66323ef5a8e4010ea3ee19f34163bc881ace Mon Sep 17 00:00:00 2001 From: sinisaos Date: Fri, 28 Jul 2023 08:19:35 +0200 Subject: [PATCH 503/727] omit graphlib from test coverage (#874) --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 521e112c4..ec6a0655e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.black] line-length = 79 -target-version = ['py37', 'py38', 'py39', 'py310'] +target-version = ['py38', 'py39', 'py310'] [tool.isort] profile = "black" @@ -31,7 +31,8 @@ markers = [ omit = [ "*.jinja", "**/piccolo_migrations/*", - "**/piccolo_app.py" + "**/piccolo_app.py", + "**/utils/graphlib/*", ] [tool.coverage.report] From dc1320072aec32d45a8c92327badcb90676718bc Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 22 Aug 2023 14:32:35 +0100 Subject: [PATCH 504/727] change `ModelBuilder` JSON values (#880) --- piccolo/testing/model_builder.py | 6 +++--- tests/testing/test_model_builder.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/piccolo/testing/model_builder.py b/piccolo/testing/model_builder.py index 180c0397c..8010f2139 100644 --- a/piccolo/testing/model_builder.py +++ b/piccolo/testing/model_builder.py @@ -6,7 +6,7 @@ from decimal import Decimal from uuid import UUID -from piccolo.columns import Array, Column, ForeignKey +from piccolo.columns import JSON, JSONB, Array, Column, ForeignKey from piccolo.custom_types import TableInstance from piccolo.testing.random_builder import RandomBuilder from piccolo.utils.sync import run_sync @@ -169,7 +169,7 @@ def _randomize_attribute(cls, column: Column) -> t.Any: if "length" in column._meta.params and isinstance(random_value, str): return random_value[: column._meta.params["length"]] - elif column.column_type in ["JSON", "JSONB"]: - return json.dumps(random_value) + elif isinstance(column, (JSON, JSONB)): + return json.dumps({"value": random_value}) return random_value diff --git a/tests/testing/test_model_builder.py b/tests/testing/test_model_builder.py index bcc6b9424..8e6e05434 100644 --- a/tests/testing/test_model_builder.py +++ b/tests/testing/test_model_builder.py @@ -1,4 +1,5 @@ import asyncio +import json import typing as t import unittest @@ -163,3 +164,23 @@ def test_valid_foreign_key_string(self): band = ModelBuilder.build_sync(Band, defaults={"manager": manager}) self.assertEqual(manager._meta.primary_key, band.manager) + + def test_json(self): + """ + Make sure the generated JSON can be parsed. + + This is important, because we might have queries like this:: + + >>> await RecordingStudio.select().output(load_json=True) + + """ + studio = ModelBuilder.build_sync(RecordingStudio) + self.assertIsInstance(json.loads(studio.facilities), dict) + self.assertIsInstance(json.loads(studio.facilities_b), dict) + + for facilities in ( + RecordingStudio.select(RecordingStudio.facilities) + .output(load_json=True, as_list=True) + .run_sync() + ): + self.assertIsInstance(facilities, dict) From 022e22a06ebca4de10bc7e3c1e2a09bc41950d87 Mon Sep 17 00:00:00 2001 From: Ethan <47520067+Skelmis@users.noreply.github.com> Date: Sat, 26 Aug 2023 03:48:14 +1200 Subject: [PATCH 505/727] feat: bring password hashing inline with industry best practices (#881) * feat: update iteration count and add support for automatic hash migration * Resolve lint failures * resolve mypy error Mypy complains because iterations was a string, but we change it to an int * add some comments * add unit test --------- Co-authored-by: Daniel Townsend --- piccolo/apps/user/tables.py | 23 ++++++++++---- tests/apps/user/test_tables.py | 55 ++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 6 deletions(-) diff --git a/piccolo/apps/user/tables.py b/piccolo/apps/user/tables.py index 718531942..40463d2c2 100644 --- a/piccolo/apps/user/tables.py +++ b/piccolo/apps/user/tables.py @@ -49,6 +49,9 @@ class BaseUser(Table, tablename="piccolo_user"): _min_password_length = 6 _max_password_length = 128 + # The number of hash iterations recommended by OWASP: + # https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html#pbkdf2 + _pbkdf2_iteration_count = 600_000 def __init__(self, **kwargs): # Generating passwords upfront is expensive, so might need reworking. @@ -131,7 +134,7 @@ async def update_password(cls, user: t.Union[str, int], password: str): @classmethod def hash_password( - cls, password: str, salt: str = "", iterations: int = 10000 + cls, password: str, salt: str = "", iterations: t.Optional[int] = None ) -> str: """ Hashes the password, ready for storage, and for comparing during @@ -147,6 +150,10 @@ def hash_password( if not salt: salt = cls.get_salt() + + if iterations is None: + iterations = cls._pbkdf2_iteration_count + hashed = hashlib.pbkdf2_hmac( "sha256", bytes(password, encoding="utf-8"), @@ -210,14 +217,18 @@ async def login(cls, username: str, password: str) -> t.Optional[int]: stored_password = response["password"] - algorithm, iterations, salt, hashed = cls.split_stored_password( + algorithm, iterations_, salt, hashed = cls.split_stored_password( stored_password ) + iterations = int(iterations_) + + if cls.hash_password(password, salt, iterations) == stored_password: + # If the password was hashed in an earlier Piccolo version, update + # it so it's hashed with the currently recommended number of + # iterations: + if iterations != cls._pbkdf2_iteration_count: + await cls.update_password(username, password) - if ( - cls.hash_password(password, salt, int(iterations)) - == stored_password - ): await cls.update({cls.last_login: datetime.datetime.now()}).where( cls.username == username ) diff --git a/tests/apps/user/test_tables.py b/tests/apps/user/test_tables.py index 497a2df0a..d48e2cc75 100644 --- a/tests/apps/user/test_tables.py +++ b/tests/apps/user/test_tables.py @@ -234,3 +234,58 @@ def test_no_password_error(self): self.assertEqual( manager.exception.__str__(), "A password must be provided." ) + + +class TestAutoHashingUpdate(TestCase): + """ + Make sure that users with passwords which were hashed in earlier Piccolo + versions are automatically re-hashed, meeting current best practices with + the number of hashing iterations. + """ + + def setUp(self): + BaseUser.create_table().run_sync() + + def tearDown(self): + BaseUser.alter().drop_table().run_sync() + + def test_hash_update(self): + # Create a user + username = "bob" + password = "abc123" + user = BaseUser.create_user_sync(username=username, password=password) + + # Update their password, so it uses less than the recommended number + # of hashing iterations. + BaseUser.update( + { + BaseUser.password: BaseUser.hash_password( + password=password, + iterations=int(BaseUser._pbkdf2_iteration_count / 2), + ) + } + ).where(BaseUser.id == user.id).run_sync() + + # Login the user - Piccolo should detect their password needs rehashing + # and update it. + self.assertIsNotNone( + BaseUser.login_sync(username=username, password=password) + ) + + hashed_password = ( + BaseUser.select(BaseUser.password) + .where(BaseUser.id == user.id) + .first() + .run_sync()["password"] + ) + + algorithm, iterations_, salt, hashed = BaseUser.split_stored_password( + hashed_password + ) + + self.assertEqual(int(iterations_), BaseUser._pbkdf2_iteration_count) + + # Make sure subsequent logins work as expected + self.assertIsNotNone( + BaseUser.login_sync(username=username, password=password) + ) From d2b2a5a5d3b59741b883aebac5f1f5719c2aafde Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 8 Sep 2023 17:54:44 +0100 Subject: [PATCH 506/727] fix migrations for tables in schemas (#884) --- .../apps/migrations/auto/diffable_table.py | 3 + .../apps/migrations/auto/migration_manager.py | 42 ++++++++-- piccolo/apps/migrations/auto/operations.py | 5 ++ piccolo/apps/migrations/auto/schema_differ.py | 31 +++++++- .../auto/integration/test_migrations.py | 78 +++++++++++++++++-- .../migrations/auto/test_schema_differ.py | 28 +++---- tests/base.py | 4 +- 7 files changed, 160 insertions(+), 31 deletions(-) diff --git a/piccolo/apps/migrations/auto/diffable_table.py b/piccolo/apps/migrations/auto/diffable_table.py index b4457cc4d..89b312018 100644 --- a/piccolo/apps/migrations/auto/diffable_table.py +++ b/piccolo/apps/migrations/auto/diffable_table.py @@ -129,6 +129,7 @@ def __sub__(self, value: DiffableTable) -> TableDelta: column_class_name=i.column.__class__.__name__, column_class=i.column.__class__, params=i.column._meta.params, + schema=self.schema, ) for i in sorted( {ColumnComparison(column=column) for column in self.columns} @@ -145,6 +146,7 @@ def __sub__(self, value: DiffableTable) -> TableDelta: column_name=i.column._meta.name, db_column_name=i.column._meta.db_column_name, tablename=value.tablename, + schema=self.schema, ) for i in sorted( {ColumnComparison(column=column) for column in value.columns} @@ -184,6 +186,7 @@ def __sub__(self, value: DiffableTable) -> TableDelta: old_params=old_params, column_class=column.__class__, old_column_class=existing_column.__class__, + schema=self.schema, ) ) diff --git a/piccolo/apps/migrations/auto/migration_manager.py b/piccolo/apps/migrations/auto/migration_manager.py index ad1175d81..8c2e9548c 100644 --- a/piccolo/apps/migrations/auto/migration_manager.py +++ b/piccolo/apps/migrations/auto/migration_manager.py @@ -27,6 +27,7 @@ class AddColumnClass: column: Column table_class_name: str tablename: str + schema: t.Optional[str] @dataclass @@ -206,6 +207,7 @@ def rename_table( old_tablename: str, new_class_name: str, new_tablename: str, + schema: t.Optional[str] = None, ): self.rename_tables.append( RenameTable( @@ -213,6 +215,7 @@ def rename_table( old_tablename=old_tablename, new_class_name=new_class_name, new_tablename=new_tablename, + schema=schema, ) ) @@ -225,6 +228,7 @@ def add_column( column_class_name: str = "", column_class: t.Optional[t.Type[Column]] = None, params: t.Dict[str, t.Any] = None, + schema: t.Optional[str] = None, ): """ Add a new column to the table. @@ -255,6 +259,7 @@ def add_column( column=column, tablename=tablename, table_class_name=table_class_name, + schema=schema, ) ) @@ -264,6 +269,7 @@ def drop_column( tablename: str, column_name: str, db_column_name: t.Optional[str] = None, + schema: t.Optional[str] = None, ): self.drop_columns.append( DropColumn( @@ -271,6 +277,7 @@ def drop_column( column_name=column_name, db_column_name=db_column_name or column_name, tablename=tablename, + schema=schema, ) ) @@ -282,6 +289,7 @@ def rename_column( new_column_name: str, old_db_column_name: t.Optional[str] = None, new_db_column_name: t.Optional[str] = None, + schema: t.Optional[str] = None, ): self.rename_columns.append( RenameColumn( @@ -291,6 +299,7 @@ def rename_column( new_column_name=new_column_name, old_db_column_name=old_db_column_name or old_column_name, new_db_column_name=new_db_column_name or new_column_name, + schema=schema, ) ) @@ -304,6 +313,7 @@ def alter_column( old_params: t.Dict[str, t.Any] = None, column_class: t.Optional[t.Type[Column]] = None, old_column_class: t.Optional[t.Type[Column]] = None, + schema: t.Optional[str] = None, ): """ All possible alterations aren't currently supported. @@ -322,6 +332,7 @@ def alter_column( old_params=old_params, column_class=column_class, old_column_class=old_column_class, + schema=schema, ) ) @@ -403,7 +414,10 @@ async def _run_alter_columns(self, backwards=False): _Table: t.Type[Table] = create_table_class( class_name=table_class_name, - class_kwargs={"tablename": alter_columns[0].tablename}, + class_kwargs={ + "tablename": alter_columns[0].tablename, + "schema": alter_columns[0].schema, + }, ) for alter_column in alter_columns: @@ -635,7 +649,10 @@ async def _run_drop_columns(self, backwards=False): _Table: t.Type[Table] = create_table_class( class_name=table_class_name, - class_kwargs={"tablename": columns[0].tablename}, + class_kwargs={ + "tablename": columns[0].tablename, + "schema": columns[0].schema, + }, ) for column in columns: @@ -662,7 +679,11 @@ async def _run_rename_tables(self, backwards=False): ) _Table: t.Type[Table] = create_table_class( - class_name=class_name, class_kwargs={"tablename": tablename} + class_name=class_name, + class_kwargs={ + "tablename": tablename, + "schema": rename_table.schema, + }, ) await self._run_query( @@ -680,7 +701,10 @@ async def _run_rename_columns(self, backwards=False): _Table: t.Type[Table] = create_table_class( class_name=table_class_name, - class_kwargs={"tablename": columns[0].tablename}, + class_kwargs={ + "tablename": columns[0].tablename, + "schema": columns[0].schema, + }, ) for rename_column in columns: @@ -746,7 +770,10 @@ async def _run_add_columns(self, backwards=False): _Table: t.Type[Table] = create_table_class( class_name=add_column.table_class_name, - class_kwargs={"tablename": add_column.tablename}, + class_kwargs={ + "tablename": add_column.tablename, + "schema": add_column.schema, + }, ) await self._run_query( @@ -765,7 +792,10 @@ async def _run_add_columns(self, backwards=False): # sets up the columns correctly. _Table: t.Type[Table] = create_table_class( class_name=add_columns[0].table_class_name, - class_kwargs={"tablename": add_columns[0].tablename}, + class_kwargs={ + "tablename": add_columns[0].tablename, + "schema": add_columns[0].schema, + }, class_members={ add_column.column._meta.name: add_column.column for add_column in add_columns diff --git a/piccolo/apps/migrations/auto/operations.py b/piccolo/apps/migrations/auto/operations.py index 2a5192b3b..0676bdbd4 100644 --- a/piccolo/apps/migrations/auto/operations.py +++ b/piccolo/apps/migrations/auto/operations.py @@ -10,6 +10,7 @@ class RenameTable: old_tablename: str new_class_name: str new_tablename: str + schema: t.Optional[str] = None @dataclass @@ -28,6 +29,7 @@ class RenameColumn: new_column_name: str old_db_column_name: str new_db_column_name: str + schema: t.Optional[str] = None @dataclass @@ -40,6 +42,7 @@ class AlterColumn: old_params: t.Dict[str, t.Any] column_class: t.Optional[t.Type[Column]] = None old_column_class: t.Optional[t.Type[Column]] = None + schema: t.Optional[str] = None @dataclass @@ -48,6 +51,7 @@ class DropColumn: column_name: str db_column_name: str tablename: str + schema: t.Optional[str] = None @dataclass @@ -58,3 +62,4 @@ class AddColumn: column_class_name: str column_class: t.Type[Column] params: t.Dict[str, t.Any] + schema: t.Optional[str] = None diff --git a/piccolo/apps/migrations/auto/schema_differ.py b/piccolo/apps/migrations/auto/schema_differ.py index f2a01ee3b..a8deb2e57 100644 --- a/piccolo/apps/migrations/auto/schema_differ.py +++ b/piccolo/apps/migrations/auto/schema_differ.py @@ -191,6 +191,7 @@ def check_rename_tables(self) -> RenameTableCollection: old_tablename=drop_table.tablename, new_class_name=new_table.class_name, new_tablename=new_table.tablename, + schema=new_table.schema, ) ) break @@ -212,6 +213,7 @@ def check_rename_tables(self) -> RenameTableCollection: old_tablename=drop_table.tablename, new_class_name=new_table.class_name, new_tablename=new_table.tablename, + schema=new_table.schema, ) ) break @@ -288,6 +290,7 @@ def check_renamed_columns(self) -> RenameColumnCollection: new_column_name=add_column.column_name, old_db_column_name=drop_column.db_column_name, new_db_column_name=add_column.db_column_name, + schema=add_column.schema, ) ) break @@ -516,8 +519,14 @@ def alter_columns(self) -> AlterStatements: ) ) + schema_str = ( + "None" + if alter_column.schema is None + else f'"{alter_column.schema}"' + ) + response.append( - f"manager.alter_column(table_class_name='{table.class_name}', tablename='{table.tablename}', column_name='{alter_column.column_name}', db_column_name='{alter_column.db_column_name}', params={new_params.params}, old_params={old_params.params}, column_class={column_class}, old_column_class={old_column_class})" # noqa: E501 + f"manager.alter_column(table_class_name='{table.class_name}', tablename='{table.tablename}', column_name='{alter_column.column_name}', db_column_name='{alter_column.db_column_name}', params={new_params.params}, old_params={old_params.params}, column_class={column_class}, old_column_class={old_column_class}, schema={schema_str})" # noqa: E501 ) return AlterStatements( @@ -543,8 +552,12 @@ def drop_columns(self) -> AlterStatements: ): continue + schema_str = ( + "None" if column.schema is None else f'"{column.schema}"' + ) + response.append( - f"manager.drop_column(table_class_name='{table.class_name}', tablename='{table.tablename}', column_name='{column.column_name}', db_column_name='{column.db_column_name}')" # noqa: E501 + f"manager.drop_column(table_class_name='{table.class_name}', tablename='{table.tablename}', column_name='{column.column_name}', db_column_name='{column.db_column_name}', schema={schema_str})" # noqa: E501 ) return AlterStatements(statements=response) @@ -585,8 +598,14 @@ def add_columns(self) -> AlterStatements: ) ) + schema_str = ( + "None" + if add_column.schema is None + else f'"{add_column.schema}"' + ) + response.append( - f"manager.add_column(table_class_name='{table.class_name}', tablename='{table.tablename}', column_name='{add_column.column_name}', db_column_name='{add_column.db_column_name}', column_class_name='{add_column.column_class_name}', column_class={column_class.__name__}, params={str(cleaned_params)})" # noqa: E501 + f"manager.add_column(table_class_name='{table.class_name}', tablename='{table.tablename}', column_name='{add_column.column_name}', db_column_name='{add_column.db_column_name}', column_class_name='{add_column.column_class_name}', column_class={column_class.__name__}, params={str(cleaned_params)}, schema={schema_str})" # noqa: E501 ) return AlterStatements( statements=response, @@ -647,8 +666,12 @@ def new_table_columns(self) -> AlterStatements: ) ) + schema_str = ( + "None" if table.schema is None else f'"{table.schema}"' + ) + response.append( - f"manager.add_column(table_class_name='{table.class_name}', tablename='{table.tablename}', column_name='{column._meta.name}', db_column_name='{column._meta.db_column_name}', column_class_name='{column.__class__.__name__}', column_class={column.__class__.__name__}, params={str(cleaned_params)})" # noqa: E501 + f"manager.add_column(table_class_name='{table.class_name}', tablename='{table.tablename}', column_name='{column._meta.name}', db_column_name='{column._meta.db_column_name}', column_class_name='{column.__class__.__name__}', column_class={column.__class__.__name__}, params={str(cleaned_params)}, schema={schema_str})" # noqa: E501 ) return AlterStatements( statements=response, diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index dbcdbf032..fcd4ba5ae 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -168,14 +168,14 @@ def _test_migrations( time.sleep(1e-6) if test_function: - column_name = ( - table_snapshots[-1][-1] - ._meta.non_default_columns[0] - ._meta.db_column_name - ) + column = table_snapshots[-1][-1]._meta.non_default_columns[0] + column_name = column._meta.db_column_name + schema = column._meta.table._meta.schema + tablename = column._meta.table._meta.tablename row_meta = self.get_postgres_column_definition( - tablename="my_table", + tablename=tablename, column_name=column_name, + schema=schema or "public", ) self.assertTrue( test_function(row_meta), @@ -1213,6 +1213,72 @@ def test_move_table_to_public_schema(self): ).run_sync(), ) + def test_altering_table_in_schema(self): + """ + Make sure tables in schemas can be altered. + + https://github.com/piccolo-orm/piccolo/issues/883 + + """ + self._test_migrations( + table_snapshots=[ + # Create a table with a single column + [ + create_table_class( + class_name="Manager", + class_kwargs={"schema": self.new_schema}, + class_members={"first_name": Varchar()}, + ) + ], + # Rename the column + [ + create_table_class( + class_name="Manager", + class_kwargs={"schema": self.new_schema}, + class_members={"name": Varchar()}, + ) + ], + # Add a column + [ + create_table_class( + class_name="Manager", + class_kwargs={"schema": self.new_schema}, + class_members={ + "name": Varchar(), + "age": Integer(), + }, + ) + ], + # Remove a column + [ + create_table_class( + class_name="Manager", + class_kwargs={"schema": self.new_schema}, + class_members={ + "name": Varchar(), + }, + ) + ], + # Alter a column + [ + create_table_class( + class_name="Manager", + class_kwargs={"schema": self.new_schema}, + class_members={ + "name": Varchar(length=512), + }, + ) + ], + ], + test_function=lambda x: all( + [ + x.column_name == "name", + x.data_type == "character varying", + x.character_maximum_length == 512, + ] + ), + ) + @engines_only("postgres", "cockroach") class TestSameTableName(MigrationTestCase): diff --git a/tests/apps/migrations/auto/test_schema_differ.py b/tests/apps/migrations/auto/test_schema_differ.py index 7a5f63fb0..cf621a916 100644 --- a/tests/apps/migrations/auto/test_schema_differ.py +++ b/tests/apps/migrations/auto/test_schema_differ.py @@ -46,7 +46,7 @@ def test_add_table(self): self.assertTrue(len(new_table_columns.statements) == 1) self.assertEqual( new_table_columns.statements[0], - "manager.add_column(table_class_name='Band', tablename='band', column_name='name', db_column_name='name', column_class_name='Varchar', column_class=Varchar, params={'length': 255, 'default': '', 'null': False, 'primary_key': False, 'unique': False, 'index': False, 'index_method': IndexMethod.btree, 'choices': None, 'db_column_name': None, 'secret': False})", # noqa + "manager.add_column(table_class_name='Band', tablename='band', column_name='name', db_column_name='name', column_class_name='Varchar', column_class=Varchar, params={'length': 255, 'default': '', 'null': False, 'primary_key': False, 'unique': False, 'index': False, 'index_method': IndexMethod.btree, 'choices': None, 'db_column_name': None, 'secret': False}, schema=None)", # noqa ) def test_drop_table(self): @@ -92,7 +92,7 @@ def test_rename_table(self): self.assertTrue(len(schema_differ.rename_tables.statements) == 1) self.assertEqual( schema_differ.rename_tables.statements[0], - "manager.rename_table(old_class_name='Band', old_tablename='band', new_class_name='Act', new_tablename='act')", # noqa: E501 + "manager.rename_table(old_class_name='Band', old_tablename='band', new_class_name='Act', new_tablename='act', schema=None)", # noqa: E501 ) self.assertEqual(schema_differ.create_tables.statements, []) @@ -165,7 +165,7 @@ def test_add_column(self): self.assertTrue(len(schema_differ.add_columns.statements) == 1) self.assertEqual( schema_differ.add_columns.statements[0], - "manager.add_column(table_class_name='Band', tablename='band', column_name='genre', db_column_name='genre', column_class_name='Varchar', column_class=Varchar, params={'length': 255, 'default': '', 'null': False, 'primary_key': False, 'unique': False, 'index': False, 'index_method': IndexMethod.btree, 'choices': None, 'db_column_name': None, 'secret': False})", # noqa: E501 + "manager.add_column(table_class_name='Band', tablename='band', column_name='genre', db_column_name='genre', column_class_name='Varchar', column_class=Varchar, params={'length': 255, 'default': '', 'null': False, 'primary_key': False, 'unique': False, 'index': False, 'index_method': IndexMethod.btree, 'choices': None, 'db_column_name': None, 'secret': False}, schema=None)", # noqa: E501 ) def test_drop_column(self): @@ -200,7 +200,7 @@ def test_drop_column(self): self.assertTrue(len(schema_differ.drop_columns.statements) == 1) self.assertEqual( schema_differ.drop_columns.statements[0], - "manager.drop_column(table_class_name='Band', tablename='band', column_name='genre', db_column_name='genre')", # noqa: E501 + "manager.drop_column(table_class_name='Band', tablename='band', column_name='genre', db_column_name='genre', schema=None)", # noqa: E501 ) def test_rename_column(self): @@ -238,7 +238,7 @@ def test_rename_column(self): self.assertEqual( schema_differ.rename_columns.statements, [ - "manager.rename_column(table_class_name='Band', tablename='band', old_column_name='title', new_column_name='name', old_db_column_name='title', new_db_column_name='name')" # noqa: E501 + "manager.rename_column(table_class_name='Band', tablename='band', old_column_name='title', new_column_name='name', old_db_column_name='title', new_db_column_name='name', schema=None)" # noqa: E501 ], ) @@ -249,13 +249,13 @@ def test_rename_column(self): self.assertEqual( schema_differ.add_columns.statements, [ - "manager.add_column(table_class_name='Band', tablename='band', column_name='name', db_column_name='name', column_class_name='Varchar', column_class=Varchar, params={'length': 255, 'default': '', 'null': False, 'primary_key': False, 'unique': False, 'index': False, 'index_method': IndexMethod.btree, 'choices': None, 'db_column_name': None, 'secret': False})" # noqa: E501 + "manager.add_column(table_class_name='Band', tablename='band', column_name='name', db_column_name='name', column_class_name='Varchar', column_class=Varchar, params={'length': 255, 'default': '', 'null': False, 'primary_key': False, 'unique': False, 'index': False, 'index_method': IndexMethod.btree, 'choices': None, 'db_column_name': None, 'secret': False}, schema=None)" # noqa: E501 ], ) self.assertEqual( schema_differ.drop_columns.statements, [ - "manager.drop_column(table_class_name='Band', tablename='band', column_name='title', db_column_name='title')" # noqa: E501 + "manager.drop_column(table_class_name='Band', tablename='band', column_name='title', db_column_name='title', schema=None)" # noqa: E501 ], ) self.assertTrue(schema_differ.rename_columns.statements == []) @@ -319,8 +319,8 @@ def mock_input(value: str): self.assertEqual( schema_differ.rename_columns.statements, [ - "manager.rename_column(table_class_name='Band', tablename='band', old_column_name='a1', new_column_name='a2', old_db_column_name='a1', new_db_column_name='a2')", # noqa: E501 - "manager.rename_column(table_class_name='Band', tablename='band', old_column_name='b1', new_column_name='b2', old_db_column_name='b1', new_db_column_name='b2')", # noqa: E501 + "manager.rename_column(table_class_name='Band', tablename='band', old_column_name='a1', new_column_name='a2', old_db_column_name='a1', new_db_column_name='a2', schema=None)", # noqa: E501 + "manager.rename_column(table_class_name='Band', tablename='band', old_column_name='b1', new_column_name='b2', old_db_column_name='b1', new_db_column_name='b2', schema=None)", # noqa: E501 ], ) @@ -391,19 +391,19 @@ def mock_input(value: str): self.assertEqual( schema_differ.add_columns.statements, [ - "manager.add_column(table_class_name='Band', tablename='band', column_name='b2', db_column_name='b2', column_class_name='Varchar', column_class=Varchar, params={'length': 255, 'default': '', 'null': False, 'primary_key': False, 'unique': False, 'index': False, 'index_method': IndexMethod.btree, 'choices': None, 'db_column_name': None, 'secret': False})" # noqa: E501 + "manager.add_column(table_class_name='Band', tablename='band', column_name='b2', db_column_name='b2', column_class_name='Varchar', column_class=Varchar, params={'length': 255, 'default': '', 'null': False, 'primary_key': False, 'unique': False, 'index': False, 'index_method': IndexMethod.btree, 'choices': None, 'db_column_name': None, 'secret': False}, schema=None)" # noqa: E501 ], ) self.assertEqual( schema_differ.drop_columns.statements, [ - "manager.drop_column(table_class_name='Band', tablename='band', column_name='b1', db_column_name='b1')" # noqa: E501 + "manager.drop_column(table_class_name='Band', tablename='band', column_name='b1', db_column_name='b1', schema=None)" # noqa: E501 ], ) self.assertEqual( schema_differ.rename_columns.statements, [ - "manager.rename_column(table_class_name='Band', tablename='band', old_column_name='a1', new_column_name='a2', old_db_column_name='a1', new_db_column_name='a2')", # noqa: E501 + "manager.rename_column(table_class_name='Band', tablename='band', old_column_name='a1', new_column_name='a2', old_db_column_name='a1', new_db_column_name='a2', schema=None)", # noqa: E501 ], ) @@ -448,7 +448,7 @@ def test_alter_column_precision(self): self.assertTrue(len(schema_differ.alter_columns.statements) == 1) self.assertEqual( schema_differ.alter_columns.statements[0], - "manager.alter_column(table_class_name='Ticket', tablename='ticket', column_name='price', db_column_name='price', params={'digits': (4, 2)}, old_params={'digits': (5, 2)}, column_class=Numeric, old_column_class=Numeric)", # noqa + "manager.alter_column(table_class_name='Ticket', tablename='ticket', column_name='price', db_column_name='price', params={'digits': (4, 2)}, old_params={'digits': (5, 2)}, column_class=Numeric, old_column_class=Numeric, schema=None)", # noqa ) def test_db_column_name(self): @@ -486,7 +486,7 @@ def test_db_column_name(self): self.assertTrue(len(schema_differ.alter_columns.statements) == 1) self.assertEqual( schema_differ.alter_columns.statements[0], - "manager.alter_column(table_class_name='Ticket', tablename='ticket', column_name='price', db_column_name='custom', params={'digits': (4, 2)}, old_params={'digits': (5, 2)}, column_class=Numeric, old_column_class=Numeric)", # noqa + "manager.alter_column(table_class_name='Ticket', tablename='ticket', column_name='price', db_column_name='custom', params={'digits': (4, 2)}, old_params={'digits': (5, 2)}, column_class=Numeric, old_column_class=Numeric, schema=None)", # noqa ) def test_alter_default(self): diff --git a/tests/base.py b/tests/base.py index 6978aba98..6f9bfbb55 100644 --- a/tests/base.py +++ b/tests/base.py @@ -174,16 +174,18 @@ def table_exists(self, tablename: str) -> bool: # Postgres specific utils def get_postgres_column_definition( - self, tablename: str, column_name: str + self, tablename: str, column_name: str, schema: str = "public" ) -> RowMeta: query = """ SELECT {columns} FROM information_schema.columns WHERE table_name = '{tablename}' AND table_catalog = 'piccolo' + AND table_schema = '{schema}' AND column_name = '{column_name}' """.format( columns=RowMeta.get_column_name_str(), tablename=tablename, + schema=schema, column_name=column_name, ) response = self.run_sync(query) From e12c4694ed2068fb08e40209b4dcd2609796752b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 8 Sep 2023 18:18:54 +0100 Subject: [PATCH 507/727] bumped version --- CHANGES.rst | 12 ++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 85dd46a27..b04d5b14f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,18 @@ Changes ======= +0.120.0 +------- + +Improved how ``ModelBuilder`` generates JSON data. + +The number of password hash iterations used in ``BaseUser`` has been increased +to keep pace with the latest guidance from OWASP - thanks to @Skelmis for this. + +Fixed a bug with auto migrations when the table is in a schema. + +------------------------------------------------------------------------------- + 0.119.0 ------- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index ca94cfb55..60e3a49c4 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.119.0" +__VERSION__ = "0.120.0" From edcfe3568382922ba3e3b65896e6e7272f972261 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 11 Sep 2023 20:41:25 +0100 Subject: [PATCH 508/727] Merge pull request from GHSA-h7cm-mrvq-wcfr Co-authored-by: skelmis --- piccolo/apps/user/tables.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/piccolo/apps/user/tables.py b/piccolo/apps/user/tables.py index 40463d2c2..878f06708 100644 --- a/piccolo/apps/user/tables.py +++ b/piccolo/apps/user/tables.py @@ -212,7 +212,10 @@ async def login(cls, username: str, password: str) -> t.Optional[int]: .run() ) if not response: - # No match found + # No match found. We still call hash_password + # here to mitigate the ability to enumerate + # users via response timings + cls.hash_password(password) return None stored_password = response["password"] From 59610df2967c4be82b3ae5cc1f20ea5f0ae150b5 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 11 Sep 2023 20:43:43 +0100 Subject: [PATCH 509/727] bumped version --- CHANGES.rst | 8 ++++++++ piccolo/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index b04d5b14f..f5fac3237 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,14 @@ Changes ======= +0.121.0 +------- + +Modified the ``BaseLogin.login`` logic so all code paths take the same time. +Thanks to @Skelmis for this. + +------------------------------------------------------------------------------- + 0.120.0 ------- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 60e3a49c4..34410433e 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.120.0" +__VERSION__ = "0.121.0" From 84e943264161cae314502b9c2e4a3673b492b54a Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 11 Sep 2023 20:45:00 +0100 Subject: [PATCH 510/727] fix typo --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index f5fac3237..8bf950f9c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,7 +4,7 @@ Changes 0.121.0 ------- -Modified the ``BaseLogin.login`` logic so all code paths take the same time. +Modified the ``BaseUser.login`` logic so all code paths take the same time. Thanks to @Skelmis for this. ------------------------------------------------------------------------------- From d1f6c35931b84e9f69a7fe5e366e3ecb27d7817b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 16 Sep 2023 09:25:42 +0100 Subject: [PATCH 511/727] Support Postgres 16 (#886) * support Postgres 16 * remove Postgres 10 --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index fca6144c9..2fd40034c 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -86,7 +86,7 @@ jobs: strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11"] - postgres-version: [10, 11, 12, 13, 14, 15] + postgres-version: [11, 12, 13, 14, 15, 16] # Service containers to run with `container-job` services: From 9e5fae92cab0a2c2d29141d05eb234b313d42ba9 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 20 Oct 2023 04:36:12 +0100 Subject: [PATCH 512/727] v1 (#893) * pydantic v2 support (#866) * pydantic v2 support based off https://github.com/piccolo-orm/piccolo/compare/master...sinisaos:piccolo:pydanticV2_test by @sinisaos * add more schema attributes for Piccolo Admin compatibility * use newer fastapi in asgi template * fix indentation in docstring * update docs * add docs for v1 * fixed typo in version number, `0.108` -> `0.118` * fix link * use `model_validate_json` instead of `parse_raw` * fix deprecation warnings for `dict` method * improve pydantic version specifier * run tests on v1 branch * bumped version * fix banner link * fix typo in link * Simplify `create_pydantic_model` (#892) * Merge pull request from GHSA-h7cm-mrvq-wcfr Co-authored-by: skelmis * bumped version * fix typo * Support Postgres 16 (#886) * support Postgres 16 * remove Postgres 10 * simplify `create_pydantic_model` I didn't like how it was overriding `format` on the JSON schema. All of the custom values we add to the JSON schema are now namespaced under the `extra` key. It makes things much less confusing and more maintainable. * fix tests --------- Co-authored-by: skelmis * bumped version * update docs --------- Co-authored-by: skelmis --- .github/workflows/tests.yaml | 4 +- CHANGES.rst | 22 ++ docs/src/conf.py | 5 +- docs/src/index.rst | 6 + docs/src/piccolo/serialization/index.rst | 21 +- docs/src/piccolo/v1/index.rst | 48 +++++ piccolo/__init__.py | 2 +- piccolo/apps/asgi/commands/new.py | 2 +- piccolo/apps/fixtures/commands/dump.py | 2 +- piccolo/apps/fixtures/commands/load.py | 4 +- piccolo/utils/pydantic.py | 116 +++++------ requirements/requirements.txt | 2 +- tests/utils/test_pydantic.py | 249 +++++++++++++++-------- 13 files changed, 329 insertions(+), 154 deletions(-) create mode 100644 docs/src/piccolo/v1/index.rst diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 2fd40034c..42a82aa10 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -2,11 +2,11 @@ name: Test Suite on: push: - branches: ["master"] + branches: ["master", "v1"] paths-ignore: - "docs/**" pull_request: - branches: ["master"] + branches: ["master", "v1"] paths-ignore: - "docs/**" diff --git a/CHANGES.rst b/CHANGES.rst index 8bf950f9c..4c49f3ec9 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,21 @@ Changes ======= +1.0a3 +----- + +Namespaced all custom values we added to Pydantic's JSON schema for easier +maintenance. + +------------------------------------------------------------------------------- + +1.0a2 +----- + +All of the changes from 0.120.0 merged into the v1 branch. + +------------------------------------------------------------------------------- + 0.121.0 ------- @@ -21,6 +36,13 @@ Fixed a bug with auto migrations when the table is in a schema. ------------------------------------------------------------------------------- +1.0a1 +----- + +Initial alpha release of Piccolo v1, with Pydantic v2 support. + +------------------------------------------------------------------------------- + 0.119.0 ------- diff --git a/docs/src/conf.py b/docs/src/conf.py index 8cb4ebee8..05bafcd39 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -67,7 +67,10 @@ html_short_title = "Piccolo" html_show_sphinx = False globaltoc_maxdepth = 3 -html_theme_options = {"source_url": "https://github.com/piccolo-orm/piccolo/"} +html_theme_options = { + "source_url": "https://github.com/piccolo-orm/piccolo/", + "banner_text": 'Piccolo v1 is now available! Learn more here.', # noqa: E501 +} # -- Options for HTMLHelp output --------------------------------------------- diff --git a/docs/src/index.rst b/docs/src/index.rst index 73d0cf5e6..0609dfd0d 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -1,3 +1,8 @@ +.. note:: These are the docs for **Piccolo v1**. :ref:`Read more here `. + + For v0.x docs `go here `_. + + Piccolo ======= @@ -27,6 +32,7 @@ batteries included. piccolo/changes/index piccolo/help/index piccolo/api_reference/index + piccolo/v1/index ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/serialization/index.rst b/docs/src/piccolo/serialization/index.rst index 64c6aa936..c68f10772 100644 --- a/docs/src/piccolo/serialization/index.rst +++ b/docs/src/piccolo/serialization/index.rst @@ -154,11 +154,15 @@ By default the primary key column isn't included - you can add it using: BandModel = create_pydantic_model(Band, include_default_columns=True) -``pydantic_config_class`` -~~~~~~~~~~~~~~~~~~~~~~~~~ +``pydantic_config`` +~~~~~~~~~~~~~~~~~~~ -You can specify a custom class to use as the base for the Pydantic model's config. -This class should be a subclass of ``pydantic.BaseConfig``. +.. hint:: We used to have a ``pydantic_config_class`` argument in Piccolo prior + to v1, but it has been replaced with ``pydantic_config`` due to changes in + Pydantic v2. + +You can specify a Pydantic ``ConfigDict`` to use as the base for the Pydantic +model's config (see `docs `_). For example, let's set the ``extra`` parameter to tell pydantic how to treat extra fields (that is, fields that would not otherwise be in the generated model). @@ -172,12 +176,15 @@ So if we want to disallow extra fields, we can do: .. code-block:: python - class MyPydanticConfig(pydantic.BaseConfig): - extra = 'forbid' + from pydatic.config import ConfigDict + + config: ConfigDict = { + "extra": "forbid" + } model = create_pydantic_model( table=MyTable, - pydantic_config_class=MyPydanticConfig + pydantic_config=config ) diff --git a/docs/src/piccolo/v1/index.rst b/docs/src/piccolo/v1/index.rst new file mode 100644 index 000000000..0c1a11fa6 --- /dev/null +++ b/docs/src/piccolo/v1/index.rst @@ -0,0 +1,48 @@ +.. _PiccoloV1: + + +About Piccolo v1 +================ + +**20th October** + +Piccolo v1 is now available! + +We migrated to Pydantic v2, and also migrated Piccolo Admin to Vue 3, which +puts the project in a good place moving forward. + +We don't anticipate any major issues for people who are upgrading. If you +encounter any bugs let us know. + +Make sure you have v1 of Piccolo, Piccolo API, and Piccolo Admin. + +**2nd August 2023** + +Piccolo started in August 2018, and as of this writing is close to 5 years old. + +During that time we've had very few, if any, breaking changes. Stability has +always been very important, as we rely on it for our production apps. + +So why release v1 now? We probably should have released v1 several years ago, +but such are things. We now have some unavoidable breaking changes due to one +of our main dependencies (Pydantic) releasing v2. + +In v2, the core of Pydantic has been rewritten in Rust, and has impressive +improvements in performance. Likewise, other libraries in the ecosystem (such +as FastAPI) have moved to Pydantic v2. It only makes sense that Piccolo does it +too. + +In terms of your own code, you shouldn't see much difference. We removed the +``pydantic_config_class`` from ``create_pydantic_model``, and replaced it with +``pydantic_config``, but that's about it. + +However, quite a bit of internal code in Piccolo and its sister libraries +`Piccolo API `_ and +`Piccolo Admin `_ had to be changed to +support Pydantic v2. Supporting both Pydantic v1 and Pydantic v2 would be quite +burdensome. + +So Piccolo v1 will just use Pydantic v2 and above. + +If you can't upgrade to Pydantic v2, then pin your Piccolo version to ``0.118.0``. +You can find the `docs here for 0.118.0 `_. diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 34410433e..2a87c5b23 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "0.121.0" +__VERSION__ = "1.0a3" diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index df344f10a..0618a6814 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -12,7 +12,7 @@ SERVERS = ["uvicorn", "Hypercorn"] ROUTERS = ["starlette", "fastapi", "blacksheep", "litestar"] ROUTER_DEPENDENCIES = { - "fastapi": ["fastapi<0.100.0"], + "fastapi": ["fastapi>=0.100.0"], } diff --git a/piccolo/apps/fixtures/commands/dump.py b/piccolo/apps/fixtures/commands/dump.py index 5d79d5562..4d78eadba 100644 --- a/piccolo/apps/fixtures/commands/dump.py +++ b/piccolo/apps/fixtures/commands/dump.py @@ -62,7 +62,7 @@ async def dump_to_json_string( pydantic_model = create_pydantic_fixture_model( fixture_configs=fixture_configs ) - return pydantic_model(**dump).json(indent=4) + return pydantic_model(**dump).model_dump_json(indent=4) def parse_args(apps: str, tables: str) -> t.List[FixtureConfig]: diff --git a/piccolo/apps/fixtures/commands/load.py b/piccolo/apps/fixtures/commands/load.py index cb330b207..1de1d5a44 100644 --- a/piccolo/apps/fixtures/commands/load.py +++ b/piccolo/apps/fixtures/commands/load.py @@ -44,7 +44,9 @@ async def load_json_string( fixture_configs=fixture_configs ) - fixture_pydantic_model = pydantic_model_class.parse_raw(json_string) + fixture_pydantic_model = pydantic_model_class.model_validate_json( + json_string + ) finder = Finder() engine = engine_finder() diff --git a/piccolo/utils/pydantic.py b/piccolo/utils/pydantic.py index c637b401f..7a456db6a 100644 --- a/piccolo/utils/pydantic.py +++ b/piccolo/utils/pydantic.py @@ -3,8 +3,8 @@ import itertools import json import typing as t -import uuid -from functools import lru_cache +from collections import defaultdict +from functools import partial import pydantic @@ -17,29 +17,20 @@ Email, ForeignKey, Numeric, - Secret, Text, Varchar, ) from piccolo.table import Table from piccolo.utils.encoding import load_json -try: - from asyncpg.pgproto.pgproto import UUID # type: ignore -except ImportError: - JSON_ENCODERS = {uuid.UUID: lambda i: str(i)} -else: - JSON_ENCODERS = {uuid.UUID: lambda i: str(i), UUID: lambda i: str(i)} - - -class Config(pydantic.BaseConfig): - json_encoders: t.Dict[t.Any, t.Callable] = JSON_ENCODERS - arbitrary_types_allowed = True +def pydantic_json_validator(value: t.Optional[str], required: bool = True): + if value is None: + if required: + raise ValueError("The JSON value wasn't provided.") + else: + return value -def pydantic_json_validator(cls, value, field): - if not field.required and value is None: - return value try: load_json(value) except json.JSONDecodeError as e: @@ -77,7 +68,6 @@ def validate_columns( ) -@lru_cache() def create_pydantic_model( table: t.Type[Table], nested: t.Union[bool, t.Tuple[ForeignKey, ...]] = False, @@ -90,8 +80,8 @@ def create_pydantic_model( deserialize_json: bool = False, recursion_depth: int = 0, max_recursion_depth: int = 5, - pydantic_config_class: t.Optional[t.Type[pydantic.BaseConfig]] = None, - **schema_extra_kwargs, + pydantic_config: t.Optional[pydantic.config.ConfigDict] = None, + json_schema_extra: t.Optional[t.Dict[str, t.Any]] = None, ) -> t.Type[pydantic.BaseModel]: """ Create a Pydantic model representing a table. @@ -133,23 +123,24 @@ def create_pydantic_model( Not to be set by the user - used internally to track recursion. :param max_recursion_depth: If using nested models, this specifies the max amount of recursion. - :param pydantic_config_class: - Config class to use as base for the generated pydantic model. You can - create your own subclass of ``pydantic.BaseConfig`` and pass it here. - :param schema_extra_kwargs: + :param pydantic_config: + Allows you to configure some of Pydantic's behaviour. See the + `Pydantic docs `_ + for more info. + :param json_schema_extra: This can be used to add additional fields to the schema. This is very useful when using Pydantic's JSON Schema features. For example: .. code-block:: python >>> my_model = create_pydantic_model(Band, my_extra_field="Hello") - >>> my_model.schema() + >>> my_model.model_json_schema() {..., "my_extra_field": "Hello"} :returns: A Pydantic model. - """ + """ # noqa: E501 if exclude_columns and include_columns: raise ValueError( "`include_columns` and `exclude_columns` can't be used at the " @@ -229,9 +220,14 @@ def create_pydantic_model( value_type = pydantic.Json else: value_type = column.value_type - validators[f"{column_name}_is_json"] = pydantic.validator( - column_name, allow_reuse=True - )(pydantic_json_validator) + validator = partial( + pydantic_json_validator, required=not is_optional + ) + validators[ + f"{column_name}_is_json" + ] = pydantic.field_validator(column_name)( + validator # type: ignore + ) else: value_type = column.value_type @@ -239,10 +235,9 @@ def create_pydantic_model( ####################################################################### - params: t.Dict[str, t.Any] = { - "default": None if is_optional else ..., - "nullable": column._meta.null, - } + params: t.Dict[str, t.Any] = {} + if is_optional: + params["default"] = None if column._meta.db_column_name != column._meta.name: params["alias"] = column._meta.db_column_name @@ -250,6 +245,8 @@ def create_pydantic_model( extra = { "help_text": column._meta.help_text, "choices": column._meta.get_choices_dict(), + "secret": column._meta.secret, + "nullable": column._meta.null, } if isinstance(column, ForeignKey): @@ -283,41 +280,46 @@ def create_pydantic_model( tablename = ( column._foreign_key_meta.resolved_references._meta.tablename ) - field = pydantic.Field( - extra={ - "foreign_key": True, - "to": tablename, - "target_column": column._foreign_key_meta.resolved_target_column._meta.name, # noqa: E501 - **extra, - }, - **params, + target_column = ( + column._foreign_key_meta.resolved_target_column._meta.name ) + extra["foreign_key"] = { + "to": tablename, + "target_column": target_column, + } + if include_readable: columns[f"{column_name}_readable"] = (str, None) - elif isinstance(column, Text): - field = pydantic.Field(format="text-area", extra=extra, **params) - elif isinstance(column, (JSON, JSONB)): - field = pydantic.Field(format="json", extra=extra, **params) - elif isinstance(column, Secret): - field = pydantic.Field(extra={"secret": True, **extra}, **params) else: - field = pydantic.Field(extra=extra, **params) + # This is used to tell Piccolo Admin that we want to display these + # values using a specific widget. + if isinstance(column, Text): + extra["widget"] = "text-area" + elif isinstance(column, (JSON, JSONB)): + extra["widget"] = "json" + + field = pydantic.Field( + json_schema_extra={"extra": extra}, + **params, + ) columns[column_name] = (_type, field) - base_classes: t.List[t.Type[pydantic.BaseConfig]] = [Config] - if pydantic_config_class: - base_classes.append(pydantic_config_class) + pydantic_config = ( + pydantic_config.copy() + if pydantic_config + else pydantic.config.ConfigDict() + ) + pydantic_config["arbitrary_types_allowed"] = True + + json_schema_extra_ = defaultdict(dict, **(json_schema_extra or {})) + json_schema_extra_["extra"]["help_text"] = table._meta.help_text - class CustomConfig(*base_classes): # type: ignore - schema_extra = { - "help_text": table._meta.help_text, - **schema_extra_kwargs, - } + pydantic_config["json_schema_extra"] = dict(json_schema_extra_) model = pydantic.create_model( # type: ignore model_name, - __config__=CustomConfig, + __config__=pydantic_config, __validators__=validators, **columns, ) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index dc6479810..0a5ee6244 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -4,4 +4,4 @@ Jinja2>=2.11.0 targ>=0.3.7 inflection>=0.5.1 typing-extensions>=4.3.0 -pydantic[email]>=1.6,<2.0 +pydantic[email]==2.* diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index 7103e3db6..3ac66afa7 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -1,19 +1,23 @@ import decimal +import typing as t from unittest import TestCase import pydantic +import pydantic_core import pytest from pydantic import ValidationError from piccolo.columns import ( JSON, JSONB, + UUID, Array, Email, Integer, Numeric, Secret, Text, + Time, Varchar, ) from piccolo.columns.column_types import ForeignKey @@ -42,7 +46,9 @@ class Director(Table): pydantic_model = create_pydantic_model(table=Director) self.assertEqual( - pydantic_model.schema()["properties"]["email"]["format"], + pydantic_model.model_json_schema()["properties"]["email"]["anyOf"][ + 0 + ]["format"], "email", ) @@ -97,9 +103,9 @@ class TopSecret(Table): pydantic_model = create_pydantic_model(table=TopSecret) self.assertEqual( - pydantic_model.schema()["properties"]["confidential"]["extra"][ - "secret" - ], + pydantic_model.model_json_schema()["properties"]["confidential"][ + "extra" + ]["secret"], True, ) @@ -112,7 +118,9 @@ class Band(Table): pydantic_model = create_pydantic_model(table=Band) self.assertEqual( - pydantic_model.schema()["properties"]["members"]["items"]["type"], + pydantic_model.model_json_schema()["properties"]["members"][ + "anyOf" + ][0]["items"]["type"], "string", ) @@ -120,7 +128,8 @@ class Band(Table): class TestForeignKeyColumn(TestCase): def test_target_column(self): """ - Make sure the target_column is correctly set in the Pydantic schema. + Make sure the `target_column` is correctly set in the Pydantic schema + for `ForeignKey` columns. """ class Manager(Table): @@ -136,57 +145,115 @@ class BandC(Table): manager = ForeignKey(Manager) self.assertEqual( - create_pydantic_model(table=BandA).schema()["properties"][ - "manager" - ]["extra"]["target_column"], + create_pydantic_model(table=BandA).model_json_schema()[ + "properties" + ]["manager"]["extra"]["foreign_key"]["target_column"], "name", ) self.assertEqual( - create_pydantic_model(table=BandB).schema()["properties"][ - "manager" - ]["extra"]["target_column"], + create_pydantic_model(table=BandB).model_json_schema()[ + "properties" + ]["manager"]["extra"]["foreign_key"]["target_column"], "name", ) self.assertEqual( - create_pydantic_model(table=BandC).schema()["properties"][ - "manager" - ]["extra"]["target_column"], + create_pydantic_model(table=BandC).model_json_schema()[ + "properties" + ]["manager"]["extra"]["foreign_key"]["target_column"], "id", ) class TestTextColumn(TestCase): - def test_text_format(self): + def test_text_widget(self): + """ + Make sure that we indicate that `Text` columns require a special widget + in Piccolo Admin. + """ + class Band(Table): bio = Text() pydantic_model = create_pydantic_model(table=Band) self.assertEqual( - pydantic_model.schema()["properties"]["bio"]["format"], + pydantic_model.model_json_schema()["properties"]["bio"]["extra"][ + "widget" + ], "text-area", ) +class TestTimeColumn(TestCase): + def test_time_format(self): + class Concert(Table): + start_time = Time() + + pydantic_model = create_pydantic_model(table=Concert) + + self.assertEqual( + pydantic_model.model_json_schema()["properties"]["start_time"][ + "anyOf" + ][0]["format"], + "time", + ) + + +class TestUUIDColumn(TestCase): + class Ticket(Table): + code = UUID() + + def setUp(self): + self.Ticket.create_table().run_sync() + + def tearDown(self): + self.Ticket.alter().drop_table().run_sync() + + def test_uuid_format(self): + class Ticket(Table): + code = UUID() + + pydantic_model = create_pydantic_model(table=Ticket) + + ticket = Ticket() + ticket.save().run_sync() + + # We'll also fetch it from the DB in case the database adapter's UUID + # is used. + ticket_from_db = Ticket.objects().first().run_sync() + + for ticket_ in (ticket, ticket_from_db): + json = pydantic_model(**ticket_.to_dict()).model_dump_json() + self.assertEqual(json, '{"code":"' + str(ticket_.code) + '"}') + + self.assertEqual( + pydantic_model.model_json_schema()["properties"]["code"]["anyOf"][ + 0 + ]["format"], + "uuid", + ) + + class TestColumnHelpText(TestCase): """ Make sure that columns with `help_text` attribute defined have the relevant text appear in the schema. """ - def test_help_text_present(self): + def test_column_help_text_present(self): help_text = "In millions of US dollars." class Movie(Table): box_office = Numeric(digits=(5, 1), help_text=help_text) pydantic_model = create_pydantic_model(table=Movie) + self.assertEqual( - pydantic_model.schema()["properties"]["box_office"]["extra"][ - "help_text" - ], + pydantic_model.model_json_schema()["properties"]["box_office"][ + "extra" + ]["help_text"], help_text, ) @@ -197,15 +264,16 @@ class TestTableHelpText(TestCase): relevant text appear in the schema. """ - def test_help_text_present(self): + def test_table_help_text_present(self): help_text = "Movies which were released in cinemas." class Movie(Table, help_text=help_text): name = Varchar() pydantic_model = create_pydantic_model(table=Movie) + self.assertEqual( - pydantic_model.schema()["help_text"], + pydantic_model.model_json_schema()["extra"]["help_text"], help_text, ) @@ -255,14 +323,21 @@ class Movie(Table): with self.assertRaises(pydantic.ValidationError): pydantic_model(meta=json_string, meta_b=json_string) - def test_json_format(self): + def test_json_widget(self): + """ + Make sure that we indicate that `JSON` / `JSONB` columns require a + special widget in Piccolo Admin. + """ + class Movie(Table): features = JSON() pydantic_model = create_pydantic_model(table=Movie) self.assertEqual( - pydantic_model.schema()["properties"]["features"]["format"], + pydantic_model.model_json_schema()["properties"]["features"][ + "extra" + ]["widget"], "json", ) @@ -286,7 +361,7 @@ class Computer(Table): pydantic_model = create_pydantic_model(Computer, exclude_columns=()) - properties = pydantic_model.schema()["properties"] + properties = pydantic_model.model_json_schema()["properties"] self.assertIsInstance(properties["GPU"], dict) self.assertIsInstance(properties["CPU"], dict) @@ -300,7 +375,7 @@ class Computer(Table): exclude_columns=(Computer.CPU,), ) - properties = pydantic_model.schema()["properties"] + properties = pydantic_model.model_json_schema()["properties"] self.assertIsInstance(properties.get("GPU"), dict) self.assertIsNone(properties.get("CPU")) @@ -314,7 +389,7 @@ class Computer(Table): exclude_columns=(Computer.GPU, Computer.CPU), ) - self.assertEqual(pydantic_model.schema()["properties"], {}) + self.assertEqual(pydantic_model.model_json_schema()["properties"], {}) def test_exclude_all_meta(self): class Computer(Table): @@ -326,7 +401,7 @@ class Computer(Table): exclude_columns=tuple(Computer._meta.columns), ) - self.assertEqual(pydantic_model.schema()["properties"], {}) + self.assertEqual(pydantic_model.model_json_schema()["properties"], {}) def test_invalid_column_str(self): class Computer(Table): @@ -384,7 +459,7 @@ class Band(Table): name="Pythonistas", manager={"name": "Guido"} ) self.assertEqual( - model_instance.dict(), + model_instance.model_dump(), {"name": "Pythonistas", "manager": {"name": "Guido"}}, ) @@ -400,7 +475,7 @@ class Band(Table): include_columns=(Band.name,), ) - properties = pydantic_model.schema()["properties"] + properties = pydantic_model.model_json_schema()["properties"] self.assertIsInstance(properties.get("name"), dict) self.assertIsNone(properties.get("popularity")) @@ -448,7 +523,7 @@ class Band(Table): name="Pythonistas", manager={"name": "Guido"} ) self.assertEqual( - model_instance.dict(), + model_instance.model_dump(), {"name": "Pythonistas", "manager": {"name": "Guido"}}, ) @@ -475,17 +550,19 @@ class Band(Table): ####################################################################### - ManagerModel = BandModel.__fields__["manager"].type_ + ManagerModel = BandModel.model_fields["manager"].annotation self.assertTrue(issubclass(ManagerModel, pydantic.BaseModel)) self.assertEqual( - [i for i in ManagerModel.__fields__.keys()], ["name", "country"] + [i for i in ManagerModel.model_fields.keys()], ["name", "country"] ) ####################################################################### - CountryModel = ManagerModel.__fields__["country"].type_ + CountryModel = ManagerModel.model_fields["country"].annotation self.assertTrue(issubclass(CountryModel, pydantic.BaseModel)) - self.assertEqual([i for i in CountryModel.__fields__.keys()], ["name"]) + self.assertEqual( + [i for i in CountryModel.model_fields.keys()], ["name"] + ) def test_tuple(self): """ @@ -518,15 +595,17 @@ class Concert(Table): BandModel = create_pydantic_model(table=Band, nested=(Band.manager,)) - ManagerModel = BandModel.__fields__["manager"].type_ + ManagerModel = BandModel.model_fields["manager"].annotation self.assertTrue(issubclass(ManagerModel, pydantic.BaseModel)) self.assertEqual( - [i for i in ManagerModel.__fields__.keys()], ["name", "country"] + [i for i in ManagerModel.model_fields.keys()], ["name", "country"] ) self.assertEqual(ManagerModel.__qualname__, "Band.manager") - AssistantManagerType = BandModel.__fields__["assistant_manager"].type_ - self.assertIs(AssistantManagerType, int) + AssistantManagerType = BandModel.model_fields[ + "assistant_manager" + ].annotation + self.assertIs(AssistantManagerType, t.Optional[int]) ####################################################################### # Test two levels deep @@ -535,19 +614,23 @@ class Concert(Table): table=Band, nested=(Band.manager.country,) ) - ManagerModel = BandModel.__fields__["manager"].type_ + ManagerModel = BandModel.model_fields["manager"].annotation self.assertTrue(issubclass(ManagerModel, pydantic.BaseModel)) self.assertEqual( - [i for i in ManagerModel.__fields__.keys()], ["name", "country"] + [i for i in ManagerModel.model_fields.keys()], ["name", "country"] ) self.assertEqual(ManagerModel.__qualname__, "Band.manager") - AssistantManagerType = BandModel.__fields__["assistant_manager"].type_ - self.assertIs(AssistantManagerType, int) + AssistantManagerType = BandModel.model_fields[ + "assistant_manager" + ].annotation + self.assertIs(AssistantManagerType, t.Optional[int]) - CountryModel = ManagerModel.__fields__["country"].type_ + CountryModel = ManagerModel.model_fields["country"].annotation self.assertTrue(issubclass(CountryModel, pydantic.BaseModel)) - self.assertEqual([i for i in CountryModel.__fields__.keys()], ["name"]) + self.assertEqual( + [i for i in CountryModel.model_fields.keys()], ["name"] + ) self.assertEqual(CountryModel.__qualname__, "Band.manager.country") ####################################################################### @@ -557,30 +640,32 @@ class Concert(Table): Concert, nested=(Concert.band_1.manager,) ) - VenueModel = ConcertModel.__fields__["venue"].type_ - self.assertIs(VenueModel, int) + VenueModel = ConcertModel.model_fields["venue"].annotation + self.assertIs(VenueModel, t.Optional[int]) - BandModel = ConcertModel.__fields__["band_1"].type_ + BandModel = ConcertModel.model_fields["band_1"].annotation self.assertTrue(issubclass(BandModel, pydantic.BaseModel)) self.assertEqual( - [i for i in BandModel.__fields__.keys()], + [i for i in BandModel.model_fields.keys()], ["name", "manager", "assistant_manager"], ) self.assertEqual(BandModel.__qualname__, "Concert.band_1") - ManagerModel = BandModel.__fields__["manager"].type_ + ManagerModel = BandModel.model_fields["manager"].annotation self.assertTrue(issubclass(ManagerModel, pydantic.BaseModel)) self.assertEqual( - [i for i in ManagerModel.__fields__.keys()], + [i for i in ManagerModel.model_fields.keys()], ["name", "country"], ) self.assertEqual(ManagerModel.__qualname__, "Concert.band_1.manager") - AssistantManagerType = BandModel.__fields__["assistant_manager"].type_ - self.assertIs(AssistantManagerType, int) + AssistantManagerType = BandModel.model_fields[ + "assistant_manager" + ].annotation + self.assertIs(AssistantManagerType, t.Optional[int]) - CountryModel = ManagerModel.__fields__["country"].type_ - self.assertIs(CountryModel, int) + CountryModel = ManagerModel.model_fields["country"].annotation + self.assertIs(CountryModel, t.Optional[int]) ####################################################################### # Test with `model_name` arg @@ -591,10 +676,10 @@ class Concert(Table): model_name="MyConcertModel", ) - BandModel = MyConcertModel.__fields__["band_1"].type_ + BandModel = MyConcertModel.model_fields["band_1"].annotation self.assertEqual(BandModel.__qualname__, "MyConcertModel.band_1") - ManagerModel = BandModel.__fields__["manager"].type_ + ManagerModel = BandModel.model_fields["manager"].annotation self.assertEqual( ManagerModel.__qualname__, "MyConcertModel.band_1.manager" ) @@ -620,17 +705,17 @@ class Band(Table): table=Band, nested=True, include_default_columns=True ) - ManagerModel = BandModel.__fields__["manager"].type_ + ManagerModel = BandModel.model_fields["manager"].annotation self.assertTrue(issubclass(ManagerModel, pydantic.BaseModel)) self.assertEqual( - [i for i in ManagerModel.__fields__.keys()], + [i for i in ManagerModel.model_fields.keys()], ["id", "name", "country"], ) - CountryModel = ManagerModel.__fields__["country"].type_ + CountryModel = ManagerModel.model_fields["country"].annotation self.assertTrue(issubclass(CountryModel, pydantic.BaseModel)) self.assertEqual( - [i for i in CountryModel.__fields__.keys()], ["id", "name"] + [i for i in CountryModel.model_fields.keys()], ["id", "name"] ) @@ -659,18 +744,18 @@ class Concert(Table): table=Concert, nested=True, max_recursion_depth=2 ) - VenueModel = ConcertModel.__fields__["venue"].type_ + VenueModel = ConcertModel.model_fields["venue"].annotation self.assertTrue(issubclass(VenueModel, pydantic.BaseModel)) - BandModel = ConcertModel.__fields__["band"].type_ + BandModel = ConcertModel.model_fields["band"].annotation self.assertTrue(issubclass(BandModel, pydantic.BaseModel)) - ManagerModel = BandModel.__fields__["manager"].type_ + ManagerModel = BandModel.model_fields["manager"].annotation self.assertTrue(issubclass(ManagerModel, pydantic.BaseModel)) # We should have hit the recursion depth: - CountryModel = ManagerModel.__fields__["country"].type_ - self.assertIs(CountryModel, int) + CountryModel = ManagerModel.model_fields["country"].annotation + self.assertIs(CountryModel, t.Optional[int]) class TestDBColumnName(TestCase): @@ -690,18 +775,22 @@ class Band(Table): self.assertEqual(model.name, "test") -class TestSchemaExtraKwargs(TestCase): - def test_schema_extra_kwargs(self): +class TestJSONSchemaExtra(TestCase): + def test_json_schema_extra(self): """ - Make sure that the ``schema_extra_kwargs`` arguments are reflected in + Make sure that the ``json_schema_extra`` arguments are reflected in Pydantic model's schema. """ class Band(Table): name = Varchar() - model = create_pydantic_model(Band, visible_columns=("name",)) - self.assertEqual(model.schema()["visible_columns"], ("name",)) + model = create_pydantic_model( + Band, json_schema_extra={"extra": {"visible_columns": ("name",)}} + ) + self.assertEqual( + model.model_json_schema()["extra"]["visible_columns"], ("name",) + ) class TestPydanticExtraFields(TestCase): @@ -714,13 +803,10 @@ def test_pydantic_extra_fields(self): class Band(Table): name = Varchar() - for v in ["ignore", "allow", "forbid"]: - - class MyConfig(pydantic.BaseConfig): - extra = v + config: pydantic.config.ConfigDict = {"extra": "forbid"} + model = create_pydantic_model(Band, pydantic_config=config) - model = create_pydantic_model(Band, pydantic_config_class=MyConfig) - self.assertEqual(model.Config.extra, v) + self.assertEqual(model.model_config["extra"], "forbid") def test_pydantic_invalid_extra_fields(self): """ @@ -731,8 +817,7 @@ def test_pydantic_invalid_extra_fields(self): class Band(Table): name = Varchar() - class MyConfig(pydantic.BaseConfig): - extra = "foobar" + config: pydantic.config.ConfigDict = {"extra": "foobar"} - with pytest.raises(ValueError): - create_pydantic_model(Band, pydantic_config_class=MyConfig) + with pytest.raises(pydantic_core._pydantic_core.SchemaError): + create_pydantic_model(Band, pydantic_config=config) From 28fe3a4fe3266d17c7a37406dc7fdbd92223d5f6 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 20 Oct 2023 04:37:55 +0100 Subject: [PATCH 513/727] bumped version --- CHANGES.rst | 15 +++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 4c49f3ec9..6e19aac3c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,21 @@ Changes ======= +1.0.0 +----- + +Piccolo v1 is now available! + +We migrated to Pydantic v2, and also migrated Piccolo Admin to Vue 3, which +puts the project in a good place moving forward. + +We don't anticipate any major issues for people who are upgrading. If you +encounter any bugs let us know. + +Make sure you have v1 of Piccolo, Piccolo API, and Piccolo Admin. + +------------------------------------------------------------------------------- + 1.0a3 ----- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 2a87c5b23..d0e42e66d 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.0a3" +__VERSION__ = "1.0.0" From febe5bbdc76cbcba50a994187cf9d953d52a00d3 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 3 Nov 2023 14:05:56 +0000 Subject: [PATCH 514/727] JSON schema - indicate that a column is `timestamptz` (#897) * indicate that a property is `timestamptz` * add a test --- piccolo/utils/pydantic.py | 3 +++ tests/utils/test_pydantic.py | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/piccolo/utils/pydantic.py b/piccolo/utils/pydantic.py index 7a456db6a..9f88ce35a 100644 --- a/piccolo/utils/pydantic.py +++ b/piccolo/utils/pydantic.py @@ -18,6 +18,7 @@ ForeignKey, Numeric, Text, + Timestamptz, Varchar, ) from piccolo.table import Table @@ -297,6 +298,8 @@ def create_pydantic_model( extra["widget"] = "text-area" elif isinstance(column, (JSON, JSONB)): extra["widget"] = "json" + elif isinstance(column, Timestamptz): + extra["widget"] = "timestamptz" field = pydantic.Field( json_schema_extra={"extra": extra}, diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index 3ac66afa7..35aff0fa4 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -18,6 +18,8 @@ Secret, Text, Time, + Timestamp, + Timestamptz, Varchar, ) from piccolo.columns.column_types import ForeignKey @@ -201,6 +203,29 @@ class Concert(Table): ) +class TestTimestamptzColumn(TestCase): + def test_timestamptz_widget(self): + """ + Make sure that we indicate that `Timestamptz` columns require a special + widget in Piccolo Admin. + """ + + class Concert(Table): + starts_on_1 = Timestamptz() + starts_on_2 = Timestamp() + + pydantic_model = create_pydantic_model(table=Concert) + + properties = pydantic_model.model_json_schema()["properties"] + + self.assertEqual( + properties["starts_on_1"]["extra"]["widget"], + "timestamptz", + ) + + self.assertIsNone(properties["starts_on_2"]["extra"].get("widget")) + + class TestUUIDColumn(TestCase): class Ticket(Table): code = UUID() From 6cb005bac644e24fbcdb923cc8a51d146e08b8fc Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 3 Nov 2023 14:58:35 +0000 Subject: [PATCH 515/727] add Python 3.12 support (#888) * add Python 3.12 support * update flake8 version * fix flake8 warnings * remove space around f-string variable * update slotscheck * skip slotscheck for Python 3.12 for now --- .github/workflows/release.yaml | 2 +- .github/workflows/tests.yaml | 14 +++++++------- piccolo/apps/migrations/auto/schema_differ.py | 2 +- piccolo/query/mixins.py | 8 ++++---- piccolo/querystring.py | 2 +- piccolo/table.py | 2 +- requirements/dev-requirements.txt | 4 ++-- scripts/lint.sh | 10 +++++++++- setup.py | 1 + tests/columns/m2m/test_m2m.py | 4 ++-- 10 files changed, 29 insertions(+), 20 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 806b1de20..60f0b83b1 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -14,7 +14,7 @@ jobs: - uses: "actions/checkout@v3" - uses: "actions/setup-python@v1" with: - python-version: 3.11 + python-version: 3.12 - name: "Install dependencies" run: "pip install -r requirements/dev-requirements.txt" - name: "Publish to PyPI" diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 42a82aa10..d71d78e7e 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -16,7 +16,7 @@ jobs: timeout-minutes: 30 strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v3 @@ -85,7 +85,7 @@ jobs: timeout-minutes: 30 strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] postgres-version: [11, 12, 13, 14, 15, 16] # Service containers to run with `container-job` @@ -134,14 +134,14 @@ jobs: PG_PASSWORD: postgres - name: Upload coverage uses: codecov/codecov-action@v1 - if: matrix.python-version == '3.11' + if: matrix.python-version == '3.12' cockroach: runs-on: ubuntu-latest timeout-minutes: 30 strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] cockroachdb-version: ["v22.2.0"] steps: - uses: actions/checkout@v3 @@ -168,14 +168,14 @@ jobs: PG_DATABASE: piccolo - name: Upload coverage uses: codecov/codecov-action@v1 - if: matrix.python-version == '3.11' + if: matrix.python-version == '3.12' sqlite: runs-on: ubuntu-latest timeout-minutes: 30 strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v3 @@ -193,4 +193,4 @@ jobs: run: ./scripts/test-sqlite.sh - name: Upload coverage uses: codecov/codecov-action@v1 - if: matrix.python-version == '3.11' + if: matrix.python-version == '3.12' diff --git a/piccolo/apps/migrations/auto/schema_differ.py b/piccolo/apps/migrations/auto/schema_differ.py index a8deb2e57..2ee4bd3e1 100644 --- a/piccolo/apps/migrations/auto/schema_differ.py +++ b/piccolo/apps/migrations/auto/schema_differ.py @@ -278,7 +278,7 @@ def check_renamed_columns(self) -> RenameColumnCollection: user_response = self.auto_input or input( f"Did you rename the `{drop_column.db_column_name}` " # noqa: E501 f"column to `{add_column.db_column_name}` on the " - f"`{ add_column.table_class_name }` table? (y/N)" + f"`{add_column.table_class_name}` table? (y/N)" ) if user_response.lower() == "y": used_drop_column_names.append(drop_column.column_name) diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index ac474b906..56be35a8f 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -90,7 +90,7 @@ class Limit: number: int def __post_init__(self): - if type(self.number) != int: + if not isinstance(self.number, int): raise TypeError("Limit must be an integer") @property @@ -111,7 +111,7 @@ class AsOf: interval: str def __post_init__(self): - if type(self.interval) != str: + if not isinstance(self.interval, str): raise TypeError("As Of must be a string. Example: '-1s'") @property @@ -129,8 +129,8 @@ class Offset: number: int def __post_init__(self): - if type(self.number) != int: - raise TypeError("Limit must be an integer") + if not isinstance(self.number, int): + raise TypeError("Offset must be an integer") @property def querystring(self) -> QueryString: diff --git a/piccolo/querystring.py b/piccolo/querystring.py index 7470deb15..3c23d86dc 100644 --- a/piccolo/querystring.py +++ b/piccolo/querystring.py @@ -143,7 +143,7 @@ def bundle( fragment.no_arg = True bundled.append(fragment) else: - if type(value) == self.__class__: + if isinstance(value, self.__class__): fragment.no_arg = True bundled.append(fragment) diff --git a/piccolo/table.py b/piccolo/table.py index ddbe04fa6..92590b2c2 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -794,7 +794,7 @@ def querystring(self) -> QueryString: args_dict[column_name] = value def is_unquoted(arg): - return type(arg) == Unquoted + return isinstance(arg, Unquoted) # Strip out any args which are unquoted. filtered_args = [i for i in args_dict.values() if not is_unquoted(i)] diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt index bd8a17e15..2a72559a7 100644 --- a/requirements/dev-requirements.txt +++ b/requirements/dev-requirements.txt @@ -1,9 +1,9 @@ black==22.3.0 ipdb==0.13.9 ipython>=7.31.1 -flake8==4.0.1 +flake8==6.1.0 isort==5.10.1 -slotscheck==0.14.0 +slotscheck==0.17.0 twine==3.8.0 mypy==0.961 pip-upgrader==1.4.15 diff --git a/scripts/lint.sh b/scripts/lint.sh index 14582f903..e328a6d8b 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -21,7 +21,15 @@ mypy $SOURCES echo "-----" echo "Running slotscheck..." -python -m slotscheck $MODULES +# Currently doesn't work for Python 3.12 - so skipping until we can get a proper fix. +pythonVersion=$(python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') +if [ "$pythonVersion" == '3.12' ] + then + echo "Skipping Python 3.12 for now" + else + python -m slotscheck $MODULES +fi + echo "-----" echo "All passed!" diff --git a/setup.py b/setup.py index 155768811..0996a1297 100644 --- a/setup.py +++ b/setup.py @@ -88,6 +88,7 @@ def extras_require() -> t.Dict[str, t.List[str]]: "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: CPython", "Framework :: AsyncIO", "Typing :: Typed", diff --git a/tests/columns/m2m/test_m2m.py b/tests/columns/m2m/test_m2m.py index e928a251f..aaec9fc84 100644 --- a/tests/columns/m2m/test_m2m.py +++ b/tests/columns/m2m/test_m2m.py @@ -395,7 +395,7 @@ def test_select_single(self): original_value = getattr(self.mega_table, column_name) returned_value = data[column_name] - if type(column) == UUID: + if isinstance(column, UUID): self.assertIn(type(returned_value), (uuid.UUID, asyncpgUUID)) else: self.assertEqual( @@ -419,7 +419,7 @@ def test_select_single(self): original_value = getattr(self.mega_table, column_name) returned_value = response[0]["mega_rows"][0] - if type(column) == UUID: + if isinstance(column, UUID): self.assertIn(type(returned_value), (uuid.UUID, asyncpgUUID)) self.assertEqual(str(original_value), str(returned_value)) else: From d4cf588b6a263659bf5fd3cf8544b6764ada3b4d Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 3 Nov 2023 15:10:31 +0000 Subject: [PATCH 516/727] bumped version --- CHANGES.rst | 11 +++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 6e19aac3c..d126244dd 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,17 @@ Changes ======= +1.1.0 +----- + +Added support for Python 3.12. + +Modified ``create_pydantic_model``, so additional information is returned in +the JSON schema to distinguish between ``Timestamp`` and ``Timestamptz`` +columns. This will be used for future Piccolo Admin enhancements. + +------------------------------------------------------------------------------- + 1.0.0 ----- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index d0e42e66d..51f5487c5 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.0.0" +__VERSION__ = "1.1.0" From bfb6acd6256dd6c569ad5929b1ef2e6e2d45dbdd Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 4 Nov 2023 14:00:49 +0000 Subject: [PATCH 517/727] upgrade `slotscheck` (#901) --- requirements/dev-requirements.txt | 2 +- scripts/lint.sh | 10 +--------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt index 2a72559a7..e8c4f5445 100644 --- a/requirements/dev-requirements.txt +++ b/requirements/dev-requirements.txt @@ -3,7 +3,7 @@ ipdb==0.13.9 ipython>=7.31.1 flake8==6.1.0 isort==5.10.1 -slotscheck==0.17.0 +slotscheck==0.17.1 twine==3.8.0 mypy==0.961 pip-upgrader==1.4.15 diff --git a/scripts/lint.sh b/scripts/lint.sh index e328a6d8b..14582f903 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -21,15 +21,7 @@ mypy $SOURCES echo "-----" echo "Running slotscheck..." -# Currently doesn't work for Python 3.12 - so skipping until we can get a proper fix. -pythonVersion=$(python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') -if [ "$pythonVersion" == '3.12' ] - then - echo "Skipping Python 3.12 for now" - else - python -m slotscheck $MODULES -fi - +python -m slotscheck $MODULES echo "-----" echo "All passed!" From e4946d858e1aae1a9a4cf7714957d07983d94cde Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 7 Nov 2023 18:23:10 +0000 Subject: [PATCH 518/727] add docs about avoiding type warnings for pydantic models which are variables (#902) --- docs/src/piccolo/serialization/index.rst | 34 ++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/src/piccolo/serialization/index.rst b/docs/src/piccolo/serialization/index.rst index c68f10772..446e6f20f 100644 --- a/docs/src/piccolo/serialization/index.rst +++ b/docs/src/piccolo/serialization/index.rst @@ -241,6 +241,40 @@ add additional fields, and to override existing fields. >>> CustomBandModel(name="Pythonistas", genre="Rock") +Or even simpler still: + +.. code-block:: python + + class BandModel(create_pydantic_model(Band)): + genre: str + + +Avoiding type warnings +~~~~~~~~~~~~~~~~~~~~~~ + +Some linters will complain if you use variables in type annotations: + +.. code-block:: python + + BandModel = create_pydantic_model(Band) + + + def my_function(band: BandModel): # Variable not allowed in type expression! + ... + + +The fix is really simple: + +.. code-block:: python + + # We now have a class instead of a variable: + class BandModel(create_pydantic_model(Band)): + ... + + + def my_function(band: BandModel): + ... + Source ~~~~~~ From 82679eb8cd1449cf31d87c9914a072e70168b6eb Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 10 Nov 2023 17:20:01 +0000 Subject: [PATCH 519/727] Merge pull request from GHSA-xq59-7jf3-rjc6 Co-authored-by: skelmis --- piccolo/engine/base.py | 15 +++++++++++++++ piccolo/engine/postgres.py | 5 ++++- piccolo/engine/sqlite.py | 5 ++++- tests/engine/test_transaction.py | 13 +++++++++++++ 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/piccolo/engine/base.py b/piccolo/engine/base.py index 77de5ceb1..95d1b8a24 100644 --- a/piccolo/engine/base.py +++ b/piccolo/engine/base.py @@ -3,6 +3,7 @@ import contextvars import logging import pprint +import string import typing as t from abc import ABCMeta, abstractmethod @@ -15,6 +16,20 @@ logger = logging.getLogger(__name__) +# This is a set to speed up lookups from O(n) when +# using str vs O(1) when using set[str] +VALID_SAVEPOINT_CHARACTERS: t.Final[set[str]] = set( + string.ascii_letters + string.digits + "-" + "_" +) + + +def validate_savepoint_name(savepoint_name: str) -> None: + """Validates a save point's name meets the required character set.""" + if not all(i in VALID_SAVEPOINT_CHARACTERS for i in savepoint_name): + raise ValueError( + "Savepoint names can only contain the following characters:" + f" {VALID_SAVEPOINT_CHARACTERS}" + ) class Batch: diff --git a/piccolo/engine/postgres.py b/piccolo/engine/postgres.py index 42a8d6bc2..b5c179703 100644 --- a/piccolo/engine/postgres.py +++ b/piccolo/engine/postgres.py @@ -4,7 +4,7 @@ import typing as t from dataclasses import dataclass -from piccolo.engine.base import Batch, Engine +from piccolo.engine.base import Batch, Engine, validate_savepoint_name from piccolo.engine.exceptions import TransactionError from piccolo.query.base import DDL, Query from piccolo.querystring import QueryString @@ -129,11 +129,13 @@ def __init__(self, name: str, transaction: PostgresTransaction): self.transaction = transaction async def rollback_to(self): + validate_savepoint_name(self.name) await self.transaction.connection.execute( f"ROLLBACK TO SAVEPOINT {self.name}" ) async def release(self): + validate_savepoint_name(self.name) await self.transaction.connection.execute( f"RELEASE SAVEPOINT {self.name}" ) @@ -236,6 +238,7 @@ def get_savepoint_id(self) -> int: async def savepoint(self, name: t.Optional[str] = None) -> Savepoint: name = name or f"savepoint_{self.get_savepoint_id()}" + validate_savepoint_name(name) await self.connection.execute(f"SAVEPOINT {name}") return Savepoint(name=name, transaction=self) diff --git a/piccolo/engine/sqlite.py b/piccolo/engine/sqlite.py index 9084baf71..7d0b3eae2 100644 --- a/piccolo/engine/sqlite.py +++ b/piccolo/engine/sqlite.py @@ -10,7 +10,7 @@ from dataclasses import dataclass from decimal import Decimal -from piccolo.engine.base import Batch, Engine +from piccolo.engine.base import Batch, Engine, validate_savepoint_name from piccolo.engine.exceptions import TransactionError from piccolo.query.base import DDL, Query from piccolo.querystring import QueryString @@ -309,11 +309,13 @@ def __init__(self, name: str, transaction: SQLiteTransaction): self.transaction = transaction async def rollback_to(self): + validate_savepoint_name(self.name) await self.transaction.connection.execute( f"ROLLBACK TO SAVEPOINT {self.name}" ) async def release(self): + validate_savepoint_name(self.name) await self.transaction.connection.execute( f"RELEASE SAVEPOINT {self.name}" ) @@ -413,6 +415,7 @@ def get_savepoint_id(self) -> int: async def savepoint(self, name: t.Optional[str] = None) -> Savepoint: name = name or f"savepoint_{self.get_savepoint_id()}" + validate_savepoint_name(name) await self.connection.execute(f"SAVEPOINT {name}") return Savepoint(name=name, transaction=self) diff --git a/tests/engine/test_transaction.py b/tests/engine/test_transaction.py index bf2489617..3cba32c86 100644 --- a/tests/engine/test_transaction.py +++ b/tests/engine/test_transaction.py @@ -2,6 +2,8 @@ import typing as t from unittest import TestCase +import pytest + from piccolo.engine.postgres import Atomic from piccolo.engine.sqlite import SQLiteEngine, TransactionType from piccolo.table import drop_db_tables_sync @@ -296,3 +298,14 @@ async def run_test(): self.assertListEqual( Manager.select(Manager.name).run_sync(), [{"name": "Manager 1"}] ) + + def test_savepoint_sqli_checks(self): + # Added to test the fix for GHSA-xq59-7jf3-rjc6 + async def run_test(): + async with Manager._meta.db.transaction() as transaction: + await transaction.savepoint( + "my_savepoint; SELECT * FROM Manager" + ) + + with pytest.raises(ValueError): + run_sync(run_test()) From bbd2e4ad6378b2080d58fb7c7ed392f0425f0f21 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 10 Nov 2023 17:28:34 +0000 Subject: [PATCH 520/727] bumped version --- CHANGES.rst | 27 +++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index d126244dd..3d257acc4 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,33 @@ Changes ======= +1.1.1 +----- + +Piccolo allows the user to specify savepoint names which are used in +transactions. For example: + +.. code-block:: python + + async with DB.transaction() as transaction: + await Band.insert(Band(name='Pythonistas')) + + # Passing in a savepoint name is optional: + savepoint_1 = await transaction.savepoint('savepoint_1') + + await Band.insert(Band(name='Terrible band')) + + # Oops, I made a mistake! + await savepoint_1.rollback_to() + +Postgres doesn't allow us to parameterise savepoint names, which means there's +a small chance of SQL injection, if for some reason the savepoint names were +generated from end-user input. Even though the likelihood is very low, it's +best to be safe. We now validate the savepoint name, to make sure it can only +contain certain safe characters. Thanks to @Skelmis for making this change. + +------------------------------------------------------------------------------- + 1.1.0 ----- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 51f5487c5..933b81888 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.1.0" +__VERSION__ = "1.1.1" From 908c8f5c2da96a35bfe32c97d946fcde5db15107 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 10 Nov 2023 20:51:30 +0000 Subject: [PATCH 521/727] Create SECURITY.md --- SECURITY.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..882645aa3 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,13 @@ +# Security Policy + +## Supported Versions + +v1 is actively maintained, and any security vunerabilities will be patched. + +v0.X will have any major security vulnerabilities patched. + +## Reporting a Vulnerability + +We recommend opening a security advisory on GitHub, as per the [documentation](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability). + +Alternatively, reach out to the maintainers via email (see [setup.py](https://github.com/piccolo-orm/piccolo/blob/bbd2e4ad6378b2080d58fb7c7ed392f0425f0f21/setup.py#L60) for contact details). From 5f972d0a7af45aeb3b18fc02f61b8c913f03b118 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 10 Nov 2023 20:52:07 +0000 Subject: [PATCH 522/727] fix typo --- SECURITY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SECURITY.md b/SECURITY.md index 882645aa3..ed95cb8a7 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,7 +2,7 @@ ## Supported Versions -v1 is actively maintained, and any security vunerabilities will be patched. +v1 is actively maintained, and any security vulnerabilities will be patched. v0.X will have any major security vulnerabilities patched. From b8026da68112686952a10ff2db3f37f4d0fdf1bd Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 30 Nov 2023 13:05:51 +0000 Subject: [PATCH 523/727] Mypy 1.7.1 (#905) * update mypy * make sure functions have type annotations Otherwise mypy ignores them * explicit optionals * more explicit optionals * make sure functions have type annotations Otherwise MyPy ignores them * use mypy version supported by VSCode extension * fix variable already defined error * fix type annotation * add missing type annotation * white space * fix errors with `engine` in `sql_shell` * fix errors in `shell/commands/run.py` * fix type annotation in `serialisation.py` * fix return type in `MigrationModule` * fix error with default not being defined * fix typo * fix type annotation * fix optional value * fix warnings in `generate.py` * fix errors in `sql_shell/commands/run.py` * fix type annotations in `migration_manager.py` * fix problems with `main.py` * fix remaining mypy errors in `base.py` * reorder `staticmethod` and `abstractmethod` * fix pydantic errors in latest pydantic version * fix warnings in tests about method signatures without types * use mypy==1.7.1 --- .../apps/migrations/auto/diffable_table.py | 3 +- .../apps/migrations/auto/migration_manager.py | 61 +++++++++++-------- piccolo/apps/migrations/auto/schema_differ.py | 12 ++-- piccolo/apps/migrations/auto/serialisation.py | 4 +- piccolo/apps/schema/commands/generate.py | 20 +++--- piccolo/apps/shell/commands/run.py | 11 ++-- piccolo/apps/sql_shell/commands/run.py | 24 ++++---- piccolo/columns/base.py | 1 + piccolo/columns/column_types.py | 6 +- piccolo/columns/m2m.py | 2 +- piccolo/conf/apps.py | 25 ++++---- piccolo/engine/cockroach.py | 2 +- piccolo/engine/postgres.py | 13 ++-- piccolo/engine/sqlite.py | 6 +- piccolo/main.py | 10 +-- piccolo/query/base.py | 24 +++++--- piccolo/query/methods/insert.py | 2 +- piccolo/query/methods/objects.py | 2 +- piccolo/query/methods/select.py | 2 +- piccolo/query/methods/update.py | 5 +- piccolo/table.py | 13 ++-- piccolo/testing/model_builder.py | 7 +-- piccolo/utils/pydantic.py | 9 ++- requirements/dev-requirements.txt | 2 +- .../migrations/auto/test_schema_differ.py | 21 +++---- tests/apps/schema/commands/test_generate.py | 57 ++++++++++------- .../foreign_key/test_attribute_access.py | 4 +- tests/columns/m2m/base.py | 16 +++-- tests/columns/m2m/test_m2m.py | 12 +--- tests/engine/test_pool.py | 9 +-- tests/engine/test_transaction.py | 4 +- tests/example_apps/music/tables.py | 9 +++ tests/query/test_freeze.py | 6 +- tests/table/test_refresh.py | 17 +++--- tests/utils/test_pydantic.py | 8 ++- 35 files changed, 235 insertions(+), 194 deletions(-) diff --git a/piccolo/apps/migrations/auto/diffable_table.py b/piccolo/apps/migrations/auto/diffable_table.py index 89b312018..aa609f041 100644 --- a/piccolo/apps/migrations/auto/diffable_table.py +++ b/piccolo/apps/migrations/auto/diffable_table.py @@ -39,7 +39,6 @@ def compare_dicts( output = {} for key, value in dict_1.items(): - dict_2_value = dict_2.get(key, ...) if ( dict_2_value is not ... @@ -99,7 +98,7 @@ class DiffableTable: columns: t.List[Column] = field(default_factory=list) previous_class_name: t.Optional[str] = None - def __post_init__(self): + def __post_init__(self) -> None: self.columns_map: t.Dict[str, Column] = { i._meta.name: i for i in self.columns } diff --git a/piccolo/apps/migrations/auto/migration_manager.py b/piccolo/apps/migrations/auto/migration_manager.py index 8c2e9548c..5c18cf89f 100644 --- a/piccolo/apps/migrations/auto/migration_manager.py +++ b/piccolo/apps/migrations/auto/migration_manager.py @@ -18,6 +18,7 @@ from piccolo.engine import engine_finder from piccolo.query import Query from piccolo.query.base import DDL +from piccolo.schema import SchemaDDLBase from piccolo.table import Table, create_table_class, sort_table_classes from piccolo.utils.warnings import colored_warning @@ -123,6 +124,9 @@ def table_class_names(self) -> t.List[str]: return list({i.table_class_name for i in self.alter_columns}) +AsyncFunction = t.Callable[[], t.Coroutine] + + @dataclass class MigrationManager: """ @@ -152,8 +156,10 @@ class MigrationManager: alter_columns: AlterColumnCollection = field( default_factory=AlterColumnCollection ) - raw: t.List[t.Union[t.Callable, t.Coroutine]] = field(default_factory=list) - raw_backwards: t.List[t.Union[t.Callable, t.Coroutine]] = field( + raw: t.List[t.Union[t.Callable, AsyncFunction]] = field( + default_factory=list + ) + raw_backwards: t.List[t.Union[t.Callable, AsyncFunction]] = field( default_factory=list ) @@ -227,7 +233,7 @@ def add_column( db_column_name: t.Optional[str] = None, column_class_name: str = "", column_class: t.Optional[t.Type[Column]] = None, - params: t.Dict[str, t.Any] = None, + params: t.Optional[t.Dict[str, t.Any]] = None, schema: t.Optional[str] = None, ): """ @@ -309,8 +315,8 @@ def alter_column( tablename: str, column_name: str, db_column_name: t.Optional[str] = None, - params: t.Dict[str, t.Any] = None, - old_params: t.Dict[str, t.Any] = None, + params: t.Optional[t.Dict[str, t.Any]] = None, + old_params: t.Optional[t.Dict[str, t.Any]] = None, column_class: t.Optional[t.Type[Column]] = None, old_column_class: t.Optional[t.Type[Column]] = None, schema: t.Optional[str] = None, @@ -336,14 +342,14 @@ def alter_column( ) ) - def add_raw(self, raw: t.Union[t.Callable, t.Coroutine]): + def add_raw(self, raw: t.Union[t.Callable, AsyncFunction]): """ A migration manager can execute arbitrary functions or coroutines when run. This is useful if you want to execute raw SQL. """ self.raw.append(raw) - def add_raw_backwards(self, raw: t.Union[t.Callable, t.Coroutine]): + def add_raw_backwards(self, raw: t.Union[t.Callable, AsyncFunction]): """ When reversing a migration, you may want to run extra code to help clean up. @@ -387,13 +393,13 @@ async def get_table_from_snapshot( ########################################################################### @staticmethod - async def _print_query(query: t.Union[DDL, Query]): + async def _print_query(query: t.Union[DDL, Query, SchemaDDLBase]): if isinstance(query, DDL): print("\n", ";".join(query.ddl) + ";") else: print(str(query)) - async def _run_query(self, query: t.Union[DDL, Query]): + async def _run_query(self, query: t.Union[DDL, Query, SchemaDDLBase]): """ If MigrationManager is not in the preview mode, executes the queries. else, prints the query. @@ -403,7 +409,7 @@ async def _run_query(self, query: t.Union[DDL, Query]): else: await query.run() - async def _run_alter_columns(self, backwards=False): + async def _run_alter_columns(self, backwards: bool = False): for table_class_name in self.alter_columns.table_class_names: alter_columns = self.alter_columns.for_table_class_name( table_class_name @@ -421,7 +427,6 @@ async def _run_alter_columns(self, backwards=False): ) for alter_column in alter_columns: - params = ( alter_column.old_params if backwards @@ -622,7 +627,7 @@ async def _run_drop_tables(self, backwards=False): diffable_table.to_table_class().alter().drop_table() ) - async def _run_drop_columns(self, backwards=False): + async def _run_drop_columns(self, backwards: bool = False): if backwards: for drop_column in self.drop_columns.drop_columns: _Table = await self.get_table_from_snapshot( @@ -647,7 +652,7 @@ async def _run_drop_columns(self, backwards=False): if not columns: continue - _Table: t.Type[Table] = create_table_class( + _Table = create_table_class( class_name=table_class_name, class_kwargs={ "tablename": columns[0].tablename, @@ -660,7 +665,7 @@ async def _run_drop_columns(self, backwards=False): _Table.alter().drop_column(column=column.column_name) ) - async def _run_rename_tables(self, backwards=False): + async def _run_rename_tables(self, backwards: bool = False): for rename_table in self.rename_tables: class_name = ( rename_table.new_class_name @@ -690,7 +695,7 @@ async def _run_rename_tables(self, backwards=False): _Table.alter().rename_table(new_name=new_tablename) ) - async def _run_rename_columns(self, backwards=False): + async def _run_rename_columns(self, backwards: bool = False): for table_class_name in self.rename_columns.table_class_names: columns = self.rename_columns.for_table_class_name( table_class_name @@ -726,7 +731,7 @@ async def _run_rename_columns(self, backwards=False): ) ) - async def _run_add_tables(self, backwards=False): + async def _run_add_tables(self, backwards: bool = False): table_classes: t.List[t.Type[Table]] = [] for add_table in self.add_tables: add_columns: t.List[ @@ -755,7 +760,7 @@ async def _run_add_tables(self, backwards=False): for _Table in sorted_table_classes: await self._run_query(_Table.create_table()) - async def _run_add_columns(self, backwards=False): + async def _run_add_columns(self, backwards: bool = False): """ Add columns, which belong to existing tables """ @@ -768,7 +773,7 @@ async def _run_add_columns(self, backwards=False): # be deleted. continue - _Table: t.Type[Table] = create_table_class( + _Table = create_table_class( class_name=add_column.table_class_name, class_kwargs={ "tablename": add_column.tablename, @@ -790,7 +795,7 @@ async def _run_add_columns(self, backwards=False): # Define the table, with the columns, so the metaclass # sets up the columns correctly. - _Table: t.Type[Table] = create_table_class( + _Table = create_table_class( class_name=add_columns[0].table_class_name, class_kwargs={ "tablename": add_columns[0].tablename, @@ -818,7 +823,7 @@ async def _run_add_columns(self, backwards=False): _Table.create_index([add_column.column]) ) - async def _run_change_table_schema(self, backwards=False): + async def _run_change_table_schema(self, backwards: bool = False): from piccolo.schema import SchemaManager schema_manager = SchemaManager() @@ -827,15 +832,19 @@ async def _run_change_table_schema(self, backwards=False): if backwards: # Note, we don't try dropping any schemas we may have created. # It's dangerous to do so, just in case the user manually - # added tables etc to the scheme, and we delete them. + # added tables etc to the schema, and we delete them. - if change_table_schema.old_schema not in (None, "public"): + if ( + change_table_schema.old_schema + and change_table_schema.old_schema != "public" + ): await self._run_query( schema_manager.create_schema( schema_name=change_table_schema.old_schema, if_not_exists=True, ) ) + await self._run_query( schema_manager.move_table( table_name=change_table_schema.tablename, @@ -845,7 +854,10 @@ async def _run_change_table_schema(self, backwards=False): ) else: - if change_table_schema.new_schema not in (None, "public"): + if ( + change_table_schema.new_schema + and change_table_schema.new_schema != "public" + ): await self._run_query( schema_manager.create_schema( schema_name=change_table_schema.new_schema, @@ -861,7 +873,7 @@ async def _run_change_table_schema(self, backwards=False): ) ) - async def run(self, backwards=False): + async def run(self, backwards: bool = False): direction = "backwards" if backwards else "forwards" if self.preview: direction = "preview " + direction @@ -873,7 +885,6 @@ async def run(self, backwards=False): raise Exception("Can't find engine") async with engine.transaction(): - if not self.preview: if direction == "backwards": raw_list = self.raw_backwards diff --git a/piccolo/apps/migrations/auto/schema_differ.py b/piccolo/apps/migrations/auto/schema_differ.py index 2ee4bd3e1..1d095b938 100644 --- a/piccolo/apps/migrations/auto/schema_differ.py +++ b/piccolo/apps/migrations/auto/schema_differ.py @@ -123,7 +123,7 @@ class SchemaDiffer: ########################################################################### - def __post_init__(self): + def __post_init__(self) -> None: self.schema_snapshot_map: t.Dict[str, DiffableTable] = { i.class_name: i for i in self.schema_snapshot } @@ -270,7 +270,6 @@ def check_renamed_columns(self) -> RenameColumnCollection: used_drop_column_names: t.List[str] = [] for add_column in delta.add_columns: - for drop_column in delta.drop_columns: if drop_column.column_name in used_drop_column_names: continue @@ -455,10 +454,11 @@ def _get_snapshot_table( class_name = self.rename_tables_collection.renamed_from( table_class_name ) - snapshot_table = self.schema_snapshot_map.get(class_name) - if snapshot_table: - snapshot_table.class_name = table_class_name - return snapshot_table + if class_name: + snapshot_table = self.schema_snapshot_map.get(class_name) + if snapshot_table: + snapshot_table.class_name = table_class_name + return snapshot_table return None @property diff --git a/piccolo/apps/migrations/auto/serialisation.py b/piccolo/apps/migrations/auto/serialisation.py index 320cf74f7..d1fd5ee47 100644 --- a/piccolo/apps/migrations/auto/serialisation.py +++ b/piccolo/apps/migrations/auto/serialisation.py @@ -347,7 +347,7 @@ def __hash__(self): def __eq__(self, other): return check_equality(self, other) - def __repr__(self): + def __repr__(self) -> str: tablename = self.table_type._meta.tablename # We have to add the primary key column definition too, so foreign @@ -493,7 +493,6 @@ def serialise_params(params: t.Dict[str, t.Any]) -> SerialisedParams: extra_definitions: t.List[Definition] = [] for key, value in params.items(): - # Builtins, such as str, list and dict. if inspect.getmodule(value) == builtins: params[key] = SerialisedBuiltin(builtin=value) @@ -501,7 +500,6 @@ def serialise_params(params: t.Dict[str, t.Any]) -> SerialisedParams: # Column instances if isinstance(value, Column): - # For target_column (which is used by ForeignKey), we can just # serialise it as the column name: if key == "target_column": diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index 2f2fc9596..20aea360d 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -91,7 +91,7 @@ class TableConstraints: tablename: str constraints: t.List[Constraint] - def __post_init__(self): + def __post_init__(self) -> None: foreign_key_constraints: t.List[Constraint] = [] unique_constraints: t.List[Constraint] = [] primary_key_constraints: t.List[Constraint] = [] @@ -127,7 +127,8 @@ def get_foreign_key_constraint_name(self, column_name) -> ConstraintTable: for i in self.foreign_key_constraints: if i.column_name == column_name: return ConstraintTable( - name=i.constraint_name, schema=i.constraint_schema + name=i.constraint_name, + schema=i.constraint_schema or "public", ) raise ValueError("No matching constraint found") @@ -307,7 +308,7 @@ def __add__(self, value: OutputSchema) -> OutputSchema: } # Re-map for Cockroach compatibility. -COLUMN_TYPE_MAP_COCKROACH = { +COLUMN_TYPE_MAP_COCKROACH: t.Dict[str, t.Type[Column]] = { **COLUMN_TYPE_MAP, **{"integer": BigInt, "json": JSONB}, } @@ -374,14 +375,13 @@ def __add__(self, value: OutputSchema) -> OutputSchema: # Re-map for Cockroach compatibility. COLUMN_DEFAULT_PARSER_COCKROACH = { **COLUMN_DEFAULT_PARSER, - **{BigInt: re.compile(r"^(?P-?\d+)$")}, + BigInt: re.compile(r"^(?P-?\d+)$"), } def get_column_default( column_type: t.Type[Column], column_default: str, engine_type: str ) -> t.Any: - if engine_type == "cockroach": pat = COLUMN_DEFAULT_PARSER_COCKROACH.get(column_type) else: @@ -462,6 +462,7 @@ def get_column_default( "gin": IndexMethod.gin, } + # 'Indices' seems old-fashioned and obscure in this context. async def get_indexes( # noqa: E302 table_class: t.Type[Table], tablename: str, schema_name: str = "public" @@ -786,9 +787,12 @@ async def create_table_class_from_db( kwargs["length"] = pg_row_meta.character_maximum_length elif isinstance(column_type, Numeric): radix = pg_row_meta.numeric_precision_radix - precision = int(str(pg_row_meta.numeric_precision), radix) - scale = int(str(pg_row_meta.numeric_scale), radix) - kwargs["digits"] = (precision, scale) + if radix: + precision = int(str(pg_row_meta.numeric_precision), radix) + scale = int(str(pg_row_meta.numeric_scale), radix) + kwargs["digits"] = (precision, scale) + else: + kwargs["digits"] = None if column_default: default_value = get_column_default( diff --git a/piccolo/apps/shell/commands/run.py b/piccolo/apps/shell/commands/run.py index 566c36c29..38cd1af66 100644 --- a/piccolo/apps/shell/commands/run.py +++ b/piccolo/apps/shell/commands/run.py @@ -1,7 +1,7 @@ import sys import typing as t -from piccolo.conf.apps import AppConfig, AppRegistry, Finder +from piccolo.conf.apps import Finder from piccolo.table import Table try: @@ -13,9 +13,7 @@ IPYTHON = False -def start_ipython_shell( - **tables: t.Dict[str, t.Type[Table]] -): # pragma: no cover +def start_ipython_shell(**tables: t.Type[Table]): # pragma: no cover if not IPYTHON: sys.exit( "Install iPython using `pip install ipython` to use this feature." @@ -29,12 +27,12 @@ def start_ipython_shell( IPython.embed(using=_asyncio_runner, colors="neutral") -def run(): +def run() -> None: """ Runs an iPython shell, and automatically imports all of the Table classes from your project. """ - app_registry: AppRegistry = Finder().get_app_registry() + app_registry = Finder().get_app_registry() tables = {} if app_registry.app_configs: @@ -43,7 +41,6 @@ def run(): print(spacer) for app_name, app_config in app_registry.app_configs.items(): - app_config: AppConfig = app_config print(f"Importing {app_name} tables:") if app_config.table_classes: for table_class in sorted( diff --git a/piccolo/apps/sql_shell/commands/run.py b/piccolo/apps/sql_shell/commands/run.py index dd3c09d17..7de03dfcd 100644 --- a/piccolo/apps/sql_shell/commands/run.py +++ b/piccolo/apps/sql_shell/commands/run.py @@ -1,22 +1,20 @@ import os import signal import subprocess +import sys import typing as t from piccolo.engine.finder import engine_finder from piccolo.engine.postgres import PostgresEngine from piccolo.engine.sqlite import SQLiteEngine -if t.TYPE_CHECKING: # pragma: no cover - from piccolo.engine.base import Engine - -def run(): +def run() -> None: """ Launch the SQL shell for the configured engine. For Postgres this will be psql, and for SQLite it will be sqlite3. """ - engine: t.Optional[Engine] = engine_finder() + engine = engine_finder() if engine is None: raise ValueError( @@ -26,7 +24,7 @@ def run(): # Heavily inspired by Django's dbshell command if isinstance(engine, PostgresEngine): - engine: PostgresEngine = engine + engine = t.cast(PostgresEngine, engine) args = ["psql"] @@ -42,7 +40,8 @@ def run(): args += ["-h", host] if port: args += ["-p", str(port)] - args += [database] + if database: + args += [database] sigint_handler = signal.getsignal(signal.SIGINT) subprocess_env = os.environ.copy() @@ -58,8 +57,11 @@ def run(): signal.signal(signal.SIGINT, sigint_handler) elif isinstance(engine, SQLiteEngine): - engine: SQLiteEngine = engine + engine = t.cast(SQLiteEngine, engine) + + database = t.cast(str, engine.connection_kwargs.get("database")) + if not database: + sys.exit("Unable to determine which database to connect to.") + print("Enter .quit to exit") - subprocess.run( - ["sqlite3", engine.connection_kwargs.get("database")], check=True - ) + subprocess.run(["sqlite3", database], check=True) diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 8c5b84064..eb4dbb693 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -469,6 +469,7 @@ class Band(Table): """ value_type: t.Type = int + default: t.Any def __init__( self, diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 9934eebed..4b316eba4 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -1959,7 +1959,7 @@ def copy(self) -> ForeignKey: return column def all_columns( - self, exclude: t.List[t.Union[Column, str]] = None + self, exclude: t.Optional[t.List[t.Union[Column, str]]] = None ) -> t.List[Column]: """ Allow a user to access all of the columns on the related table. This is @@ -2010,7 +2010,7 @@ def all_columns( ] def all_related( - self, exclude: t.List[t.Union[ForeignKey, str]] = None + self, exclude: t.Optional[t.List[t.Union[ForeignKey, str]]] = None ) -> t.List[ForeignKey]: """ Returns each ``ForeignKey`` column on the related table. This is @@ -2065,7 +2065,7 @@ class Tour(Table): if fk_column._meta.name not in excluded_column_names ] - def set_proxy_columns(self): + def set_proxy_columns(self) -> None: """ In order to allow a fluent interface, where tables can be traversed using ForeignKeys (e.g. ``Band.manager.name``), we add attributes to diff --git a/piccolo/columns/m2m.py b/piccolo/columns/m2m.py index 69a647244..0eefd22e7 100644 --- a/piccolo/columns/m2m.py +++ b/piccolo/columns/m2m.py @@ -252,7 +252,7 @@ class M2MAddRelated: rows: t.Sequence[Table] extra_column_values: t.Dict[t.Union[Column, str], t.Any] - def __post_init__(self): + def __post_init__(self) -> None: # Normalise `extra_column_values`, so we just have the column names. self.extra_column_values: t.Dict[str, t.Any] = { i._meta.name if isinstance(i, Column) else i: j diff --git a/piccolo/conf/apps.py b/piccolo/conf/apps.py index b88d50ba2..df8ddad10 100644 --- a/piccolo/conf/apps.py +++ b/piccolo/conf/apps.py @@ -6,10 +6,12 @@ import pathlib import traceback import typing as t +from abc import abstractmethod from dataclasses import dataclass, field from importlib import import_module from types import ModuleType +from piccolo.apps.migrations.auto.migration_manager import MigrationManager from piccolo.engine.base import Engine from piccolo.table import Table from piccolo.utils.graphlib import TopologicalSorter @@ -22,8 +24,9 @@ class MigrationModule(ModuleType): DESCRIPTION: str @staticmethod - async def forwards() -> None: - pass + @abstractmethod + async def forwards() -> MigrationManager: + ... class PiccoloAppModule(ModuleType): @@ -32,8 +35,8 @@ class PiccoloAppModule(ModuleType): def table_finder( modules: t.Sequence[str], - include_tags: t.Sequence[str] = None, - exclude_tags: t.Sequence[str] = None, + include_tags: t.Optional[t.Sequence[str]] = None, + exclude_tags: t.Optional[t.Sequence[str]] = None, exclude_imported: bool = False, ) -> t.List[t.Type[Table]]: """ @@ -151,11 +154,7 @@ class AppConfig: default_factory=list ) - def __post_init__(self): - self.commands = [ - i if isinstance(i, Command) else Command(i) for i in self.commands - ] - + def __post_init__(self) -> None: if isinstance(self.migrations_folder_path, pathlib.Path): self.migrations_folder_path = str(self.migrations_folder_path) @@ -167,6 +166,11 @@ def register_table(self, table_class: t.Type[Table]): self.table_classes.append(table_class) return table_class + def get_commands(self) -> t.List[Command]: + return [ + i if isinstance(i, Command) else Command(i) for i in self.commands + ] + @property def migration_dependency_app_configs(self) -> t.List[AppConfig]: """ @@ -176,7 +180,6 @@ def migration_dependency_app_configs(self) -> t.List[AppConfig]: # We cache the value so it's more efficient, and also so we can set the # underlying value in unit tests for easier mocking. if self._migration_dependency_app_configs is None: - modules: t.List[PiccoloAppModule] = [ t.cast(PiccoloAppModule, import_module(module_path)) for module_path in self.migration_dependencies @@ -214,7 +217,7 @@ class AppRegistry: """ - def __init__(self, apps: t.List[str] = None): + def __init__(self, apps: t.Optional[t.List[str]] = None): self.apps = apps or [] self.app_configs: t.Dict[str, AppConfig] = {} app_names = [] diff --git a/piccolo/engine/cockroach.py b/piccolo/engine/cockroach.py index 6c5019531..ecbb74ad8 100644 --- a/piccolo/engine/cockroach.py +++ b/piccolo/engine/cockroach.py @@ -25,7 +25,7 @@ def __init__( extensions: t.Sequence[str] = (), log_queries: bool = False, log_responses: bool = False, - extra_nodes: t.Dict[str, CockroachEngine] = None, + extra_nodes: t.Optional[t.Dict[str, CockroachEngine]] = None, ) -> None: super().__init__( config=config, diff --git a/piccolo/engine/postgres.py b/piccolo/engine/postgres.py index b5c179703..06b8ffb4b 100644 --- a/piccolo/engine/postgres.py +++ b/piccolo/engine/postgres.py @@ -22,7 +22,6 @@ @dataclass class AsyncBatch(Batch): - connection: Connection query: Query batch_size: int @@ -93,9 +92,9 @@ class Atomic: def __init__(self, engine: PostgresEngine): self.engine = engine - self.queries: t.List[Query] = [] + self.queries: t.List[t.Union[Query, DDL]] = [] - def add(self, *query: Query): + def add(self, *query: t.Union[Query, DDL]): self.queries += list(query) async def run(self): @@ -348,7 +347,7 @@ def __init__( extensions: t.Sequence[str] = ("uuid-ossp",), log_queries: bool = False, log_responses: bool = False, - extra_nodes: t.Mapping[str, PostgresEngine] = None, + extra_nodes: t.Optional[t.Mapping[str, PostgresEngine]] = None, ) -> None: if extra_nodes is None: extra_nodes = {} @@ -489,7 +488,9 @@ async def batch( ########################################################################### - async def _run_in_pool(self, query: str, args: t.Sequence[t.Any] = None): + async def _run_in_pool( + self, query: str, args: t.Optional[t.Sequence[t.Any]] = None + ): if args is None: args = [] if not self.pool: @@ -501,7 +502,7 @@ async def _run_in_pool(self, query: str, args: t.Sequence[t.Any] = None): return response async def _run_in_new_connection( - self, query: str, args: t.Sequence[t.Any] = None + self, query: str, args: t.Optional[t.Sequence[t.Any]] = None ): if args is None: args = [] diff --git a/piccolo/engine/sqlite.py b/piccolo/engine/sqlite.py index 7d0b3eae2..ffc6606b0 100644 --- a/piccolo/engine/sqlite.py +++ b/piccolo/engine/sqlite.py @@ -196,7 +196,6 @@ def convert_M2M_out(value: bytes) -> t.List: @dataclass class AsyncBatch(Batch): - connection: Connection query: Query batch_size: int @@ -448,7 +447,6 @@ def dict_factory(cursor, row) -> t.Dict: class SQLiteEngine(Engine[t.Optional[SQLiteTransaction]]): - __slots__ = ( "connection_kwargs", "current_transaction", @@ -585,7 +583,7 @@ async def _get_inserted_pk(self, cursor, table: t.Type[Table]) -> t.Any: async def _run_in_new_connection( self, query: str, - args: t.List[t.Any] = None, + args: t.Optional[t.List[t.Any]] = None, query_type: str = "generic", table: t.Optional[t.Type[Table]] = None, ): @@ -611,7 +609,7 @@ async def _run_in_existing_connection( self, connection, query: str, - args: t.List[t.Any] = None, + args: t.Optional[t.List[t.Any]] = None, query_type: str = "generic", table: t.Optional[t.Type[Table]] = None, ): diff --git a/piccolo/main.py b/piccolo/main.py index 0766a0aa7..556b86f59 100644 --- a/piccolo/main.py +++ b/piccolo/main.py @@ -34,7 +34,7 @@ def get_diagnose_flag() -> bool: return DIAGNOSE_FLAG in sys.argv -def main(): +def main() -> None: """ The entrypoint to the Piccolo CLI. """ @@ -72,7 +72,7 @@ def main(): tester_config, user_config, ]: - for command in _app_config.commands: + for command in _app_config.get_commands(): cli.register( command.callable, group_name=_app_config.app_name, @@ -92,12 +92,14 @@ def main(): ) else: for app_name, _app_config in APP_REGISTRY.app_configs.items(): - for command in _app_config.commands: + for command in _app_config.get_commands(): if cli.command_exists( - group_name=app_name, command_name=command.callable.__name__ + group_name=app_name, + command_name=command.callable.__name__, ): # Skipping - already registered. continue + cli.register( command.callable, group_name=app_name, diff --git a/piccolo/query/base.py b/piccolo/query/base.py index a7761b8c6..b10d42ee0 100644 --- a/piccolo/query/base.py +++ b/piccolo/query/base.py @@ -26,7 +26,6 @@ def __exit__(self, exception_type, exception, traceback): class Query(t.Generic[TableInstance, QueryResponseType]): - __slots__ = ("table", "_frozen_querystrings") def __init__( @@ -45,7 +44,7 @@ def engine_type(self) -> str: else: raise ValueError("Engine isn't defined.") - async def _process_results(self, results): + async def _process_results(self, results) -> QueryResponseType: if results: keys = results[0].keys() keys = [i.replace("$", ".") for i in keys] @@ -118,14 +117,20 @@ async def _process_results(self, results): if output: if output._output.as_objects: if output._output.nested: - raw = [make_nested_object(row, self.table) for row in raw] + return t.cast( + QueryResponseType, + [make_nested_object(row, self.table) for row in raw], + ) else: - raw = [ - self.table(**columns, _exists_in_db=True) - for columns in raw - ] + return t.cast( + QueryResponseType, + [ + self.table(**columns, _exists_in_db=True) + for columns in raw + ], + ) - return raw + return t.cast(QueryResponseType, raw) def _validate(self): """ @@ -222,7 +227,7 @@ def run_sync( with Timer(): return run_sync(coroutine) - async def response_handler(self, response): + async def response_handler(self, response: t.List) -> t.Any: """ Subclasses can override this to modify the raw response returned by the database driver. @@ -370,7 +375,6 @@ def __str__(self) -> str: class DDL: - __slots__ = ("table",) def __init__(self, table: t.Type[Table], **kwargs): diff --git a/piccolo/query/methods/insert.py b/piccolo/query/methods/insert.py index 5c06169d8..d7c655b80 100644 --- a/piccolo/query/methods/insert.py +++ b/piccolo/query/methods/insert.py @@ -81,7 +81,7 @@ def on_conflict( ########################################################################### - def _raw_response_callback(self, results): + def _raw_response_callback(self, results: t.List): """ Assign the ids of the created rows to the model instances. """ diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index 6892fc95c..5b1c96002 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -280,7 +280,7 @@ def get(self: Self, where: Combinable) -> Get[TableInstance]: def get_or_create( self: Self, where: Combinable, - defaults: t.Dict[Column, t.Any] = None, + defaults: t.Optional[t.Dict[Column, t.Any]] = None, ) -> GetOrCreate[TableInstance]: if defaults is None: defaults = {} diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 85b2ac6b4..a00745e48 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -367,7 +367,7 @@ class Select(Query[TableInstance, t.List[t.Dict[str, t.Any]]]): def __init__( self, table: t.Type[TableInstance], - columns_list: t.Sequence[t.Union[Selectable, str]] = None, + columns_list: t.Optional[t.Sequence[t.Union[Selectable, str]]] = None, exclude_secrets: bool = False, **kwargs, ): diff --git a/piccolo/query/methods/update.py b/piccolo/query/methods/update.py index 0d914b858..ff6a10589 100644 --- a/piccolo/query/methods/update.py +++ b/piccolo/query/methods/update.py @@ -20,7 +20,6 @@ class UpdateError(Exception): class Update(Query[TableInstance, t.List[t.Any]]): - __slots__ = ( "force", "returning_delegate", @@ -41,7 +40,9 @@ def __init__( # Clauses def values( - self, values: t.Dict[t.Union[Column, str], t.Any] = None, **kwargs + self, + values: t.Optional[t.Dict[t.Union[Column, str], t.Any]] = None, + **kwargs, ) -> Update: if values is None: values = {} diff --git a/piccolo/table.py b/piccolo/table.py index 92590b2c2..abae84a52 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -224,7 +224,7 @@ def __init_subclass__( cls, tablename: t.Optional[str] = None, db: t.Optional[Engine] = None, - tags: t.List[str] = None, + tags: t.Optional[t.List[str]] = None, help_text: t.Optional[str] = None, schema: t.Optional[str] = None, ): # sourcery no-metrics @@ -364,7 +364,7 @@ def __init_subclass__( def __init__( self, - _data: t.Dict[Column, t.Any] = None, + _data: t.Optional[t.Dict[Column, t.Any]] = None, _ignore_missing: bool = False, _exists_in_db: bool = False, **kwargs, @@ -826,7 +826,7 @@ def __repr__(self) -> str: @classmethod def all_related( - cls, exclude: t.List[t.Union[str, ForeignKey]] = None + cls, exclude: t.Optional[t.List[t.Union[str, ForeignKey]]] = None ) -> t.List[Column]: """ Used in conjunction with ``objects`` queries. Just as we can use @@ -876,7 +876,7 @@ def all_related( @classmethod def all_columns( - cls, exclude: t.Sequence[t.Union[str, Column]] = None + cls, exclude: t.Optional[t.Sequence[t.Union[str, Column]]] = None ) -> t.List[Column]: """ Used in conjunction with ``select`` queries. Just as we can use @@ -1120,7 +1120,6 @@ def count( column: t.Optional[Column] = None, distinct: t.Optional[t.Sequence[Column]] = None, ) -> Count: - """ Count the number of matching rows:: @@ -1191,7 +1190,7 @@ def table_exists(cls) -> TableExists: @classmethod def update( cls, - values: t.Dict[t.Union[Column, str], t.Any] = None, + values: t.Optional[t.Dict[t.Union[Column, str], t.Any]] = None, force: bool = False, use_auto_update: bool = True, **kwargs, @@ -1303,7 +1302,7 @@ def _get_index_name(cls, column_names: t.List[str]) -> str: @classmethod def _table_str( - cls, abbreviated=False, excluded_params: t.List[str] = None + cls, abbreviated=False, excluded_params: t.Optional[t.List[str]] = None ): """ Returns a basic string representation of the table and its columns. diff --git a/piccolo/testing/model_builder.py b/piccolo/testing/model_builder.py index 8010f2139..15b50416c 100644 --- a/piccolo/testing/model_builder.py +++ b/piccolo/testing/model_builder.py @@ -30,7 +30,7 @@ class ModelBuilder: async def build( cls, table_class: t.Type[TableInstance], - defaults: t.Dict[t.Union[Column, str], t.Any] = None, + defaults: t.Optional[t.Dict[t.Union[Column, str], t.Any]] = None, persist: bool = True, minimal: bool = False, ) -> TableInstance: @@ -81,7 +81,7 @@ async def build( def build_sync( cls, table_class: t.Type[TableInstance], - defaults: t.Dict[t.Union[Column, str], t.Any] = None, + defaults: t.Optional[t.Dict[t.Union[Column, str], t.Any]] = None, persist: bool = True, minimal: bool = False, ) -> TableInstance: @@ -101,7 +101,7 @@ def build_sync( async def _build( cls, table_class: t.Type[TableInstance], - defaults: t.Dict[t.Union[Column, str], t.Any] = None, + defaults: t.Optional[t.Dict[t.Union[Column, str], t.Any]] = None, minimal: bool = False, persist: bool = True, ) -> TableInstance: @@ -115,7 +115,6 @@ async def _build( setattr(model, column._meta.name, value) for column in model._meta.columns: - if column._meta.null and minimal: continue diff --git a/piccolo/utils/pydantic.py b/piccolo/utils/pydantic.py index 9f88ce35a..e245e522d 100644 --- a/piccolo/utils/pydantic.py +++ b/piccolo/utils/pydantic.py @@ -24,6 +24,11 @@ from piccolo.table import Table from piccolo.utils.encoding import load_json +try: + from pydantic.config import JsonDict +except ImportError: + JsonDict = dict # type: ignore + def pydantic_json_validator(value: t.Optional[str], required: bool = True): if value is None: @@ -243,7 +248,7 @@ def create_pydantic_model( if column._meta.db_column_name != column._meta.name: params["alias"] = column._meta.db_column_name - extra = { + extra: JsonDict = { "help_text": column._meta.help_text, "choices": column._meta.get_choices_dict(), "secret": column._meta.secret, @@ -320,7 +325,7 @@ def create_pydantic_model( pydantic_config["json_schema_extra"] = dict(json_schema_extra_) - model = pydantic.create_model( # type: ignore + model = pydantic.create_model( model_name, __config__=pydantic_config, __validators__=validators, diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt index e8c4f5445..853726ee9 100644 --- a/requirements/dev-requirements.txt +++ b/requirements/dev-requirements.txt @@ -5,6 +5,6 @@ flake8==6.1.0 isort==5.10.1 slotscheck==0.17.1 twine==3.8.0 -mypy==0.961 +mypy==1.7.1 pip-upgrader==1.4.15 wheel==0.38.1 diff --git a/tests/apps/migrations/auto/test_schema_differ.py b/tests/apps/migrations/auto/test_schema_differ.py index cf621a916..9cf6d26f2 100644 --- a/tests/apps/migrations/auto/test_schema_differ.py +++ b/tests/apps/migrations/auto/test_schema_differ.py @@ -16,10 +16,9 @@ class TestSchemaDiffer(TestCase): - maxDiff = None - def test_add_table(self): + def test_add_table(self) -> None: """ Test adding a new table. """ @@ -49,7 +48,7 @@ def test_add_table(self): "manager.add_column(table_class_name='Band', tablename='band', column_name='name', db_column_name='name', column_class_name='Varchar', column_class=Varchar, params={'length': 255, 'default': '', 'null': False, 'primary_key': False, 'unique': False, 'index': False, 'index_method': IndexMethod.btree, 'choices': None, 'db_column_name': None, 'secret': False}, schema=None)", # noqa ) - def test_drop_table(self): + def test_drop_table(self) -> None: """ Test dropping an existing table. """ @@ -67,7 +66,7 @@ def test_drop_table(self): "manager.drop_table(class_name='Band', tablename='band', schema=None)", # noqa: E501 ) - def test_rename_table(self): + def test_rename_table(self) -> None: """ Test renaming a table. """ @@ -98,7 +97,7 @@ def test_rename_table(self): self.assertEqual(schema_differ.create_tables.statements, []) self.assertEqual(schema_differ.drop_tables.statements, []) - def test_change_schema(self): + def test_change_schema(self) -> None: """ Testing changing the schema. """ @@ -133,7 +132,7 @@ def test_change_schema(self): self.assertListEqual(schema_differ.create_tables.statements, []) self.assertListEqual(schema_differ.drop_tables.statements, []) - def test_add_column(self): + def test_add_column(self) -> None: """ Test adding a column to an existing table. """ @@ -168,7 +167,7 @@ def test_add_column(self): "manager.add_column(table_class_name='Band', tablename='band', column_name='genre', db_column_name='genre', column_class_name='Varchar', column_class=Varchar, params={'length': 255, 'default': '', 'null': False, 'primary_key': False, 'unique': False, 'index': False, 'index_method': IndexMethod.btree, 'choices': None, 'db_column_name': None, 'secret': False}, schema=None)", # noqa: E501 ) - def test_drop_column(self): + def test_drop_column(self) -> None: """ Test dropping a column from an existing table. """ @@ -203,7 +202,7 @@ def test_drop_column(self): "manager.drop_column(table_class_name='Band', tablename='band', column_name='genre', db_column_name='genre', schema=None)", # noqa: E501 ) - def test_rename_column(self): + def test_rename_column(self) -> None: """ Test renaming a column in an existing table. """ @@ -261,7 +260,7 @@ def test_rename_column(self): self.assertTrue(schema_differ.rename_columns.statements == []) @patch("piccolo.apps.migrations.auto.schema_differ.input") - def test_rename_multiple_columns(self, input: MagicMock): + def test_rename_multiple_columns(self, input: MagicMock) -> None: """ Make sure renaming columns works when several columns have been renamed. @@ -419,7 +418,7 @@ def mock_input(value: str): ], ) - def test_alter_column_precision(self): + def test_alter_column_precision(self) -> None: price_1 = Numeric(digits=(4, 2)) price_1._meta.name = "price" @@ -451,7 +450,7 @@ def test_alter_column_precision(self): "manager.alter_column(table_class_name='Ticket', tablename='ticket', column_name='price', db_column_name='price', params={'digits': (4, 2)}, old_params={'digits': (5, 2)}, column_class=Numeric, old_column_class=Numeric, schema=None)", # noqa ) - def test_db_column_name(self): + def test_db_column_name(self) -> None: """ Make sure alter statements use the ``db_column_name`` if provided. diff --git a/tests/apps/schema/commands/test_generate.py b/tests/apps/schema/commands/test_generate.py index a08521038..29784fa8a 100644 --- a/tests/apps/schema/commands/test_generate.py +++ b/tests/apps/schema/commands/test_generate.py @@ -64,7 +64,7 @@ def _compare_table_columns( # Make sure the unique constraint is the same self.assertEqual(col_1._meta.unique, col_2._meta.unique) - def test_get_output_schema(self): + def test_get_output_schema(self) -> None: """ Make sure that the a Piccolo schema can be generated from the database. """ @@ -75,9 +75,11 @@ def test_get_output_schema(self): self.assertTrue(len(output_schema.imports) > 0) MegaTable_ = output_schema.get_table_with_name("MegaTable") + assert MegaTable_ is not None self._compare_table_columns(MegaTable, MegaTable_) SmallTable_ = output_schema.get_table_with_name("SmallTable") + assert SmallTable_ is not None self._compare_table_columns(SmallTable, SmallTable_) @patch("piccolo.apps.schema.commands.generate.print") @@ -94,7 +96,7 @@ def test_generate_command(self, print_: MagicMock): # Cockroach throws FeatureNotSupportedError, which does not pass this test. @engines_skip("cockroach") - def test_unknown_column_type(self): + def test_unknown_column_type(self) -> None: """ Make sure unknown column types are handled gracefully. """ @@ -119,11 +121,13 @@ class Box(Column): for table in output_schema.tables: if table.__name__ == "MegaTable": self.assertEqual( - output_schema.tables[1].my_column.__class__.__name__, + output_schema.tables[1] + ._meta.get_column_by_name("my_column") + .__class__.__name__, "Column", ) - def test_generate_required_tables(self): + def test_generate_required_tables(self) -> None: """ Make sure only tables passed to `tablenames` are created. """ @@ -132,9 +136,10 @@ def test_generate_required_tables(self): ) self.assertEqual(len(output_schema.tables), 1) SmallTable_ = output_schema.get_table_with_name("SmallTable") + assert SmallTable_ is not None self._compare_table_columns(SmallTable, SmallTable_) - def test_exclude_table(self): + def test_exclude_table(self) -> None: """ Make sure exclude works. """ @@ -143,10 +148,11 @@ def test_exclude_table(self): ) self.assertEqual(len(output_schema.tables), 1) SmallTable_ = output_schema.get_table_with_name("SmallTable") + assert SmallTable_ is not None self._compare_table_columns(SmallTable, SmallTable_) @engines_skip("cockroach") - def test_self_referencing_fk(self): + def test_self_referencing_fk(self) -> None: """ Make sure self-referencing foreign keys are handled correctly. """ @@ -160,12 +166,15 @@ def test_self_referencing_fk(self): # Make sure the 'references' value of the generated column is "self". for table in output_schema.tables: if table.__name__ == "MegaTable": - column: ForeignKey = output_schema.tables[ - 1 - ].self_referencing_fk + column = t.cast( + ForeignKey, + output_schema.tables[1]._meta.get_column_by_name( + "self_referencing_fk" + ), + ) self.assertEqual( - column._foreign_key_meta.references._meta.tablename, + column._foreign_key_meta.resolved_references._meta.tablename, # noqa: E501 MegaTable._meta.tablename, ) self.assertEqual(column._meta.params["references"], "self") @@ -190,23 +199,24 @@ def setUp(self): def tearDown(self): Concert.alter().drop_table(if_exists=True).run_sync() - def test_index(self): + def test_index(self) -> None: """ Make sure that a table with an index is reflected correctly. """ output_schema: OutputSchema = run_sync(get_output_schema()) Concert_ = output_schema.tables[0] - self.assertEqual(Concert_.name._meta.index, True) - self.assertEqual(Concert_.name._meta.index_method, IndexMethod.hash) + name_column = Concert_._meta.get_column_by_name("name") + self.assertTrue(name_column._meta.index) + self.assertEqual(name_column._meta.index_method, IndexMethod.hash) - self.assertEqual(Concert_.time._meta.index, True) - self.assertEqual(Concert_.time._meta.index_method, IndexMethod.btree) + time_column = Concert_._meta.get_column_by_name("time") + self.assertTrue(time_column._meta.index) + self.assertEqual(time_column._meta.index_method, IndexMethod.btree) - self.assertEqual(Concert_.capacity._meta.index, False) - self.assertEqual( - Concert_.capacity._meta.index_method, IndexMethod.btree - ) + capacity_column = Concert_._meta.get_column_by_name("capacity") + self.assertEqual(capacity_column._meta.index, False) + self.assertEqual(capacity_column._meta.index_method, IndexMethod.btree) ############################################################################### @@ -229,7 +239,6 @@ class Book(Table): @engines_only("postgres") class TestGenerateWithSchema(TestCase): - tables = [Publication, Writer, Book] schema_manager = SchemaManager() @@ -250,7 +259,7 @@ def tearDown(self) -> None: schema_name=schema_name, if_exists=True, cascade=True ).run_sync() - def test_reference_to_another_schema(self): + def test_reference_to_another_schema(self) -> None: output_schema: OutputSchema = run_sync(get_output_schema()) self.assertEqual(len(output_schema.tables), 3) publication = output_schema.tables[0] @@ -263,8 +272,10 @@ def test_reference_to_another_schema(self): self.assertEqual(Writer._meta.tablename, writer._meta.tablename) # Make sure foreign key values are correct. - self.assertEqual(writer.publication, publication) - self.assertEqual(book.writer, writer) + self.assertEqual( + writer._meta.get_column_by_name("publication"), publication + ) + self.assertEqual(book._meta.get_column_by_name("writer"), writer) @engines_only("postgres", "cockroach") diff --git a/tests/columns/foreign_key/test_attribute_access.py b/tests/columns/foreign_key/test_attribute_access.py index 87caeb78e..3f9d8afab 100644 --- a/tests/columns/foreign_key/test_attribute_access.py +++ b/tests/columns/foreign_key/test_attribute_access.py @@ -40,7 +40,7 @@ def test_attribute_access(self): for band_table in (BandA, BandB, BandC, BandD): self.assertIsInstance(band_table.manager.name, Varchar) - def test_recursion_limit(self): + def test_recursion_limit(self) -> None: """ When a table has a ForeignKey to itself, an Exception should be raised if the call chain is too large. @@ -51,7 +51,7 @@ def test_recursion_limit(self): self.assertIsInstance(column, Varchar) with self.assertRaises(Exception): - Manager.manager.manager.manager.manager.manager.manager.manager.manager.manager.manager.manager.name # noqa + Manager.manager.manager.manager.manager.manager.manager.manager.manager.manager.manager.manager.name # type: ignore # noqa: E501 def test_recursion_time(self): """ diff --git a/tests/columns/m2m/base.py b/tests/columns/m2m/base.py index f91d8d7d6..a3f282d23 100644 --- a/tests/columns/m2m/base.py +++ b/tests/columns/m2m/base.py @@ -292,7 +292,7 @@ def test_add_m2m(self): Genre = self.genre GenreToBand = self.genre_to_band - band: Band = Band.objects().get(Band.name == "Pythonistas").run_sync() + band = Band.objects().get(Band.name == "Pythonistas").run_sync() band.add_m2m(Genre(name="Punk Rock"), m2m=Band.genres).run_sync() self.assertTrue( @@ -320,7 +320,7 @@ def test_extra_columns_str(self): reason = "Their second album was very punk rock." - band: Band = Band.objects().get(Band.name == "Pythonistas").run_sync() + band = Band.objects().get(Band.name == "Pythonistas").run_sync() band.add_m2m( Genre(name="Punk Rock"), m2m=Band.genres, @@ -351,7 +351,7 @@ def test_extra_columns_class(self): reason = "Their second album was very punk rock." - band: Band = Band.objects().get(Band.name == "Pythonistas").run_sync() + band = Band.objects().get(Band.name == "Pythonistas").run_sync() band.add_m2m( Genre(name="Punk Rock"), m2m=Band.genres, @@ -379,11 +379,9 @@ def test_add_m2m_existing(self): Genre = self.genre GenreToBand = self.genre_to_band - band: Band = Band.objects().get(Band.name == "Pythonistas").run_sync() + band = Band.objects().get(Band.name == "Pythonistas").run_sync() - genre: Genre = ( - Genre.objects().get(Genre.name == "Classical").run_sync() - ) + genre = Genre.objects().get(Genre.name == "Classical").run_sync() band.add_m2m(genre, m2m=Band.genres).run_sync() @@ -408,7 +406,7 @@ def test_get_m2m(self): """ Band = self.band - band: Band = Band.objects().get(Band.name == "Pythonistas").run_sync() + band = Band.objects().get(Band.name == "Pythonistas").run_sync() genres = band.get_m2m(Band.genres).run_sync() @@ -424,7 +422,7 @@ def test_remove_m2m(self): Genre = self.genre GenreToBand = self.genre_to_band - band: Band = Band.objects().get(Band.name == "Pythonistas").run_sync() + band = Band.objects().get(Band.name == "Pythonistas").run_sync() genre = Genre.objects().get(Genre.name == "Rock").run_sync() diff --git a/tests/columns/m2m/test_m2m.py b/tests/columns/m2m/test_m2m.py index aaec9fc84..d897eace6 100644 --- a/tests/columns/m2m/test_m2m.py +++ b/tests/columns/m2m/test_m2m.py @@ -160,9 +160,7 @@ def test_add_m2m(self): """ Make sure we can add items to the joining table. """ - customer: Customer = ( - Customer.objects().get(Customer.name == "Bob").run_sync() - ) + customer = Customer.objects().get(Customer.name == "Bob").run_sync() customer.add_m2m( Concert(name="Jazzfest"), m2m=Customer.concerts ).run_sync() @@ -193,9 +191,7 @@ def test_add_m2m_within_transaction(self): async def add_m2m_in_transaction(): async with engine.transaction(): - customer: Customer = await Customer.objects().get( - Customer.name == "Bob" - ) + customer = await Customer.objects().get(Customer.name == "Bob") await customer.add_m2m( Concert(name="Jazzfest"), m2m=Customer.concerts ) @@ -220,9 +216,7 @@ def test_get_m2m(self): """ Make sure we can get related items via the joining table. """ - customer: Customer = ( - Customer.objects().get(Customer.name == "Bob").run_sync() - ) + customer = Customer.objects().get(Customer.name == "Bob").run_sync() concerts = customer.get_m2m(Customer.concerts).run_sync() diff --git a/tests/engine/test_pool.py b/tests/engine/test_pool.py index adddc6bda..510a77072 100644 --- a/tests/engine/test_pool.py +++ b/tests/engine/test_pool.py @@ -1,6 +1,7 @@ import asyncio import os import tempfile +import typing as t from unittest import TestCase from unittest.mock import call, patch @@ -12,8 +13,8 @@ @engines_only("postgres", "cockroach") class TestPool(DBTestCase): - async def _create_pool(self): - engine: PostgresEngine = Manager._meta.db + async def _create_pool(self) -> None: + engine = t.cast(PostgresEngine, Manager._meta.db) await engine.start_connection_pool() assert engine.pool is not None @@ -70,8 +71,8 @@ def test_many_queries(self): @engines_only("postgres", "cockroach") class TestPoolProxyMethods(DBTestCase): - async def _create_pool(self): - engine: PostgresEngine = Manager._meta.db + async def _create_pool(self) -> None: + engine = t.cast(PostgresEngine, Manager._meta.db) # Deliberate typo ('nnn'): await engine.start_connnection_pool() diff --git a/tests/engine/test_transaction.py b/tests/engine/test_transaction.py index 3cba32c86..4b47f8759 100644 --- a/tests/engine/test_transaction.py +++ b/tests/engine/test_transaction.py @@ -45,12 +45,12 @@ def test_succeeds(self): drop_db_tables_sync(Band, Manager) @engines_only("postgres", "cockroach") - def test_pool(self): + def test_pool(self) -> None: """ Make sure atomic works correctly when a connection pool is active. """ - async def run(): + async def run() -> None: """ We have to run this async function, so we can use a connection pool. diff --git a/tests/example_apps/music/tables.py b/tests/example_apps/music/tables.py index 8e2870ea1..dff416c2f 100644 --- a/tests/example_apps/music/tables.py +++ b/tests/example_apps/music/tables.py @@ -7,6 +7,7 @@ ForeignKey, Integer, Numeric, + Serial, Text, Varchar, ) @@ -21,6 +22,7 @@ class Manager(Table): + id: Serial name = Varchar(length=50) @classmethod @@ -29,6 +31,7 @@ def get_readable(cls) -> Readable: class Band(Table): + id: Serial name = Varchar(length=50) manager = ForeignKey(Manager, null=True) popularity = ( @@ -47,6 +50,7 @@ def get_readable(cls) -> Readable: class Venue(Table): + id: Serial name = Varchar(length=100) capacity = Integer(default=0, secret=True) @@ -56,6 +60,7 @@ def get_readable(cls) -> Readable: class Concert(Table): + id: Serial band_1 = ForeignKey(Band) band_2 = ForeignKey(Band) venue = ForeignKey(Venue) @@ -74,6 +79,7 @@ def get_readable(cls) -> Readable: class Ticket(Table): + id: Serial concert = ForeignKey(Concert) price = Numeric(digits=(5, 2)) @@ -83,6 +89,7 @@ class Poster(Table, tags=["special"]): Has tags for tests which need it. """ + id: Serial content = Text() @@ -96,6 +103,7 @@ class Size(str, Enum): medium = "m" large = "l" + id: Serial size = Varchar(length=1, choices=Size, default=Size.large) @@ -104,5 +112,6 @@ class RecordingStudio(Table): Used for testing JSON and JSONB columns. """ + id: Serial facilities = JSON() facilities_b = JSONB() diff --git a/tests/query/test_freeze.py b/tests/query/test_freeze.py index 29cb5271f..ca916ba5e 100644 --- a/tests/query/test_freeze.py +++ b/tests/query/test_freeze.py @@ -4,7 +4,7 @@ from unittest import mock from piccolo.columns import Integer, Varchar -from piccolo.query.base import Query +from piccolo.query.base import FrozenQuery, Query from piccolo.table import Table from tests.base import AsyncMock, DBTestCase, sqlite_only from tests.example_apps.music.tables import Band @@ -12,12 +12,12 @@ @dataclass class QueryResponse: - query: Query + query: t.Union[Query, FrozenQuery] response: t.Any class TestFreeze(DBTestCase): - def test_frozen_select_queries(self): + def test_frozen_select_queries(self) -> None: """ Make sure a variety of select queries work as expected when frozen. """ diff --git a/tests/table/test_refresh.py b/tests/table/test_refresh.py index 69deb9a73..ffb78ddb8 100644 --- a/tests/table/test_refresh.py +++ b/tests/table/test_refresh.py @@ -7,12 +7,13 @@ def setUp(self): super().setUp() self.insert_rows() - def test_refresh(self): + def test_refresh(self) -> None: """ Make sure ``refresh`` works, with no columns specified. """ # Fetch an instance from the database. - band: Band = Band.objects().get(Band.name == "Pythonistas").run_sync() + band = Band.objects().get(Band.name == "Pythonistas").run_sync() + assert band is not None initial_data = band.to_dict() # Modify the data in the database. @@ -27,12 +28,13 @@ def test_refresh(self): self.assertTrue(band.popularity == 8000) self.assertTrue(band.id == initial_data["id"]) - def test_columns(self): + def test_columns(self) -> None: """ Make sure ``refresh`` works, when columns are specified. """ # Fetch an instance from the database. - band: Band = Band.objects().get(Band.name == "Pythonistas").run_sync() + band = Band.objects().get(Band.name == "Pythonistas").run_sync() + assert band is not None initial_data = band.to_dict() # Modify the data in the database. @@ -52,7 +54,7 @@ def test_columns(self): self.assertTrue(band.popularity == initial_data["popularity"]) self.assertTrue(band.id == initial_data["id"]) - def test_error_when_not_in_db(self): + def test_error_when_not_in_db(self) -> None: """ Make sure we can't refresh an instance which hasn't been saved in the database. @@ -67,12 +69,13 @@ def test_error_when_not_in_db(self): str(manager.exception), ) - def test_error_when_pk_in_none(self): + def test_error_when_pk_in_none(self) -> None: """ Make sure we can't refresh an instance when the primary key value isn't set. """ - band: Band = Band.objects().first().run_sync() + band = Band.objects().first().run_sync() + assert band is not None band.id = None with self.assertRaises(ValueError) as manager: diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index 35aff0fa4..a6e003f7f 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -819,7 +819,7 @@ class Band(Table): class TestPydanticExtraFields(TestCase): - def test_pydantic_extra_fields(self): + def test_pydantic_extra_fields(self) -> None: """ Make sure that the value of ``extra`` in the config class is correctly propagated to the generated model. @@ -833,7 +833,7 @@ class Band(Table): self.assertEqual(model.model_config["extra"], "forbid") - def test_pydantic_invalid_extra_fields(self): + def test_pydantic_invalid_extra_fields(self) -> None: """ Make sure that invalid values for ``extra`` in the config class are rejected. @@ -842,7 +842,9 @@ def test_pydantic_invalid_extra_fields(self): class Band(Table): name = Varchar() - config: pydantic.config.ConfigDict = {"extra": "foobar"} + config: pydantic.config.ConfigDict = { + "extra": "foobar" # type: ignore + } with pytest.raises(pydantic_core._pydantic_core.SchemaError): create_pydantic_model(Band, pydantic_config=config) From 8580fb7879ca633351b2fb18364f436523c98207 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 1 Dec 2023 18:59:04 +0000 Subject: [PATCH 524/727] fix typo in URL for Piccolo Admin docs (#908) --- docs/src/piccolo/getting_started/what_is_piccolo.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/piccolo/getting_started/what_is_piccolo.rst b/docs/src/piccolo/getting_started/what_is_piccolo.rst index 5e2cf8db5..d9f68b680 100644 --- a/docs/src/piccolo/getting_started/what_is_piccolo.rst +++ b/docs/src/piccolo/getting_started/what_is_piccolo.rst @@ -25,7 +25,7 @@ Python around the time Piccolo was started. A really important thing when working at a design agency is having a **great admin interface**. A huge amount of effort has gone into -`Piccolo Admin `_ +`Piccolo Admin `_ to make something you'd be proud to give to a client. A lot of batteries are included because Piccolo is a pragmatic framework From 4fb169d81551c122f353f1e9018974f5f077b3bd Mon Sep 17 00:00:00 2001 From: Tiago Silva Date: Tue, 12 Dec 2023 11:15:00 +0000 Subject: [PATCH 525/727] Add support for Esmerald (#907) * Add support for Esmerald * Add missing self to Esmerald routes * Adding missing Esmerald reference command * Update piccolo/apps/asgi/commands/templates/app/home/_esmerald_endpoints.py.jinja Co-authored-by: sinisaos * Update piccolo/apps/asgi/commands/templates/app/_esmerald_app.py.jinja Co-authored-by: sinisaos * Remove piccolo admin from esmerald routes * Add esmerald to template * Update _esmerald_app.py.jinja with cleat paths * Update piccolo/apps/asgi/commands/templates/app/_esmerald_app.py.jinja Co-authored-by: sinisaos --------- Co-authored-by: sinisaos --- docs/src/index.rst | 4 +- docs/src/piccolo/asgi/index.rst | 8 +- piccolo/apps/asgi/commands/new.py | 2 +- .../templates/app/_esmerald_app.py.jinja | 100 ++++++++++++++++++ .../asgi/commands/templates/app/app.py.jinja | 2 + .../app/home/_esmerald_endpoints.py.jinja | 20 ++++ .../templates/app/home/endpoints.py.jinja | 2 + .../app/home/templates/home.html.jinja_raw | 5 + 8 files changed, 136 insertions(+), 7 deletions(-) create mode 100644 piccolo/apps/asgi/commands/templates/app/_esmerald_app.py.jinja create mode 100644 piccolo/apps/asgi/commands/templates/app/home/_esmerald_endpoints.py.jinja diff --git a/docs/src/index.rst b/docs/src/index.rst index 0609dfd0d..577cb7620 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -59,8 +59,8 @@ Give me an ASGI web app! piccolo asgi new -FastAPI, Starlette, BlackSheep, and Litestar are currently supported, with more -coming soon. +FastAPI, Starlette, BlackSheep, Litestar and Esmerald are currently supported, +with more coming soon. ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/asgi/index.rst b/docs/src/piccolo/asgi/index.rst index 67bf87027..57f1553c4 100644 --- a/docs/src/piccolo/asgi/index.rst +++ b/docs/src/piccolo/asgi/index.rst @@ -21,8 +21,8 @@ Routing frameworks ****************** Currently, `Starlette `_, `FastAPI `_, -`BlackSheep `_, and -`Litestar `_ are supported. +`BlackSheep `_, +`Litestar `_ and `Esmerald `_ are supported. Other great ASGI routing frameworks exist, and may be supported in the future (`Quart `_ , @@ -32,8 +32,8 @@ Other great ASGI routing frameworks exist, and may be supported in the future Which to use? ============= -All are great choices. FastAPI is built on top of Starlette, so they're -very similar. FastAPI and BlackSheep are great if you want to document a REST +All are great choices. FastAPI and Esmerald are built on top of Starlette, so they're +very similar. FastAPI, BlackSheep and Esmerald are great if you want to document a REST API, as they have built-in OpenAPI support. ------------------------------------------------------------------------------- diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index 0618a6814..38a056295 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -10,7 +10,7 @@ TEMPLATE_DIR = os.path.join(os.path.dirname(__file__), "templates/app/") SERVERS = ["uvicorn", "Hypercorn"] -ROUTERS = ["starlette", "fastapi", "blacksheep", "litestar"] +ROUTERS = ["starlette", "fastapi", "blacksheep", "litestar", "esmerald"] ROUTER_DEPENDENCIES = { "fastapi": ["fastapi>=0.100.0"], } diff --git a/piccolo/apps/asgi/commands/templates/app/_esmerald_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_esmerald_app.py.jinja new file mode 100644 index 000000000..a69df7706 --- /dev/null +++ b/piccolo/apps/asgi/commands/templates/app/_esmerald_app.py.jinja @@ -0,0 +1,100 @@ +import typing as t + +from pathlib import Path + +from piccolo.utils.pydantic import create_pydantic_model +from piccolo.engine import engine_finder + +from esmerald import ( + Esmerald, + Include, + Gateway, + JSONResponse, + APIView, + get, + post, + put, + delete +) +from esmerald.config import StaticFilesConfig + +from home.endpoints import home +from home.piccolo_app import APP_CONFIG +from home.tables import Task + + +async def open_database_connection_pool(): + try: + engine = engine_finder() + await engine.start_connection_pool() + except Exception: + print("Unable to connect to the database") + + +async def close_database_connection_pool(): + try: + engine = engine_finder() + await engine.close_connection_pool() + except Exception: + print("Unable to connect to the database") + + +TaskModelIn: t.Any = create_pydantic_model( + table=Task, + model_name='TaskModelIn' +) +TaskModelOut: t.Any = create_pydantic_model( + table=Task, + include_default_columns=True, + model_name='TaskModelOut' +) + + +class TaskAPIView(APIView): + path: str = "/" + tags: str = ["Task"] + + @get("/") + async def tasks(self) -> t.List[TaskModelOut]: + return await Task.select().order_by(Task.id) + + + @post('/') + async def create_task(self, payload: TaskModelIn) -> TaskModelOut: + task = Task(**payload.dict()) + await task.save() + return task.to_dict() + + + @put('/{task_id}') + async def update_task(self, payload: TaskModelIn, task_id: int) -> TaskModelOut: + task = await Task.objects().get(Task.id == task_id) + if not task: + return JSONResponse({}, status_code=404) + + for key, value in payload.dict().items(): + setattr(task, key, value) + + await task.save() + + return task.to_dict() + + + @delete('/{task_id}') + async def delete_task(self, task_id: int) -> None: + task = await Task.objects().get(Task.id == task_id) + if not task: + return JSONResponse({}, status_code=404) + + await task.remove() + + +app = Esmerald( + routes=[ + Gateway("/", handler=home), + Gateway("/tasks", handler=TaskAPIView) + ], + static_files_config=StaticFilesConfig(path="/static", directory=Path("static")), + on_startup=[open_database_connection_pool], + on_shutdown=[close_database_connection_pool], +) diff --git a/piccolo/apps/asgi/commands/templates/app/app.py.jinja b/piccolo/apps/asgi/commands/templates/app/app.py.jinja index 2a8de80da..234286a3b 100644 --- a/piccolo/apps/asgi/commands/templates/app/app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/app.py.jinja @@ -6,4 +6,6 @@ {% include '_blacksheep_app.py.jinja' %} {% elif router == 'litestar' %} {% include '_litestar_app.py.jinja' %} +{% elif router == 'esmerald' %} + {% include '_esmerald_app.py.jinja' %} {% endif %} diff --git a/piccolo/apps/asgi/commands/templates/app/home/_esmerald_endpoints.py.jinja b/piccolo/apps/asgi/commands/templates/app/home/_esmerald_endpoints.py.jinja new file mode 100644 index 000000000..a8c9bfdff --- /dev/null +++ b/piccolo/apps/asgi/commands/templates/app/home/_esmerald_endpoints.py.jinja @@ -0,0 +1,20 @@ +import os + +import jinja2 +from esmerald import Request, Response, get +from esmerald.responses import HTMLResponse + +ENVIRONMENT = jinja2.Environment( + loader=jinja2.FileSystemLoader( + searchpath=os.path.join(os.path.dirname(__file__), "templates") + ) +) + + +@get(path="/", include_in_schema=False) +def home(request: Request) -> HTMLResponse: + template = ENVIRONMENT.get_template("home.html.jinja") + + content = template.render(title="Piccolo + ASGI",) + + return HTMLResponse(content) diff --git a/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja b/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja index fc5011b24..cbce94e2f 100644 --- a/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja @@ -4,4 +4,6 @@ {% include '_blacksheep_endpoints.py.jinja' %} {% elif router == 'litestar' %} {% include '_litestar_endpoints.py.jinja' %} +{% elif router == 'esmerald' %} + {% include '_esmerald_endpoints.py.jinja' %} {% endif %} diff --git a/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw b/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw index ab175a6b0..644bc4265 100644 --- a/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw +++ b/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw @@ -56,6 +56,11 @@
    • Admin
    • Swagger API
    +

    Esmerald

    + {% endblock content %} From d05c2da77794178acebb32b9dd86bedce6cc1a16 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 22 Dec 2023 19:25:15 +0000 Subject: [PATCH 526/727] `ForeignKey` static typing (#914) * foreign key improvements * fix formatting * fix linter errors * add docs * added a unit test for the new join syntax --- docs/src/piccolo/query_types/joins.rst | 40 +++++++- piccolo/columns/base.py | 13 ++- piccolo/columns/column_types.py | 93 +++++++++++++++++-- piccolo/table.py | 23 +++-- .../foreign_key/test_attribute_access.py | 10 +- .../test_foreign_key_references.py | 2 +- .../foreign_key/test_foreign_key_self.py | 2 +- .../foreign_key/test_foreign_key_string.py | 8 +- tests/columns/foreign_key/test_value_type.py | 2 +- tests/table/instance/test_get_related.py | 10 +- tests/table/test_join.py | 24 ++++- tests/testing/test_model_builder.py | 2 +- tests/type_checking.py | 18 +++- 13 files changed, 209 insertions(+), 38 deletions(-) diff --git a/docs/src/piccolo/query_types/joins.rst b/docs/src/piccolo/query_types/joins.rst index d7727162e..3d89c6da2 100644 --- a/docs/src/piccolo/query_types/joins.rst +++ b/docs/src/piccolo/query_types/joins.rst @@ -26,8 +26,44 @@ And a ``where`` clause which uses joins: Left joins are used. -join_on -------- +Improved static typing +---------------------- + +You can optionally modify the above queries slightly for powerful static typing +support from tools like Mypy and Pylance: + +.. code-block:: python + + await Band.select(Band.name, Band.manager._.name) + +Notice how we use ``._.`` instead of ``.`` after each foreign key. An easy way +to remember this is ``._.`` looks a bit like a connector in a diagram. + +Static type checkers now know that we're referencing the ``name`` column on the +``Manager`` table, which has many advantages: + +* Autocompletion of column names. +* Easier code navigation (command + click on column names to navigate to the + column definition). +* Most importantly, the detection of typos in column names. + +This works, no matter how many joins are performed. For example: + +.. code-block:: python + + await Concert.select( + Concert.band_1._.name, + Concert.band_1._.manager._.name, + ) + +.. note:: You may wonder why this syntax is required. We're operating within + the limits of Python's typing support, which is still fairly young. In the + future we will hopefully be able to offer identical static typing support + for ``Band.manager.name`` and ``Band.manager._.name``. But even then, + the ``._.`` syntax will still be supported. + +``join_on`` +----------- Joins are usually performed using ``ForeignKey`` columns, though there may be situations where you want to join using a column which isn't a ``ForeignKey``. diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index eb4dbb693..d477dc992 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -77,9 +77,12 @@ def __repr__(self): return self.__str__() +ReferencedTable = t.TypeVar("ReferencedTable", bound="Table") + + @dataclass -class ForeignKeyMeta: - references: t.Union[t.Type[Table], LazyTableReference] +class ForeignKeyMeta(t.Generic[ReferencedTable]): + references: t.Union[t.Type[ReferencedTable], LazyTableReference] on_delete: OnDelete on_update: OnUpdate target_column: t.Union[Column, str, None] @@ -121,15 +124,15 @@ def resolved_target_column(self) -> Column: else: raise ValueError("Unable to resolve target_column.") - def copy(self) -> ForeignKeyMeta: + def copy(self) -> ForeignKeyMeta[ReferencedTable]: kwargs = self.__dict__.copy() kwargs.update(proxy_columns=self.proxy_columns.copy()) return self.__class__(**kwargs) - def __copy__(self) -> ForeignKeyMeta: + def __copy__(self) -> ForeignKeyMeta[ReferencedTable]: return self.copy() - def __deepcopy__(self, memo) -> ForeignKeyMeta: + def __deepcopy__(self, memo) -> ForeignKeyMeta[ReferencedTable]: """ We override deepcopy, as it's too slow if it has to recreate everything. diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 4b316eba4..33c0f261f 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -35,7 +35,13 @@ class Band(Table): from datetime import date, datetime, time, timedelta from enum import Enum -from piccolo.columns.base import Column, ForeignKeyMeta, OnDelete, OnUpdate +from piccolo.columns.base import ( + Column, + ForeignKeyMeta, + OnDelete, + OnUpdate, + ReferencedTable, +) from piccolo.columns.combination import Where from piccolo.columns.defaults.date import DateArg, DateCustom, DateNow from piccolo.columns.defaults.interval import IntervalArg, IntervalCustom @@ -1635,7 +1641,7 @@ class ForeignKeySetupResponse: is_lazy: bool -class ForeignKey(Column): +class ForeignKey(Column, t.Generic[ReferencedTable]): """ Used to reference another table. Uses the same type as the primary key column on the table it references. @@ -1846,9 +1852,48 @@ def value_type(self): target_column = self._foreign_key_meta.resolved_target_column return target_column.value_type + @t.overload + def __init__( + self, + references: t.Type[ReferencedTable], + default: t.Any = None, + null: bool = True, + on_delete: OnDelete = OnDelete.cascade, + on_update: OnUpdate = OnUpdate.cascade, + target_column: t.Union[str, Column, None] = None, + **kwargs, + ) -> None: + ... + + @t.overload + def __init__( + self, + references: LazyTableReference, + default: t.Any = None, + null: bool = True, + on_delete: OnDelete = OnDelete.cascade, + on_update: OnUpdate = OnUpdate.cascade, + target_column: t.Union[str, Column, None] = None, + **kwargs, + ) -> None: + ... + + @t.overload def __init__( self, - references: t.Union[t.Type[Table], LazyTableReference, str], + references: str, + default: t.Any = None, + null: bool = True, + on_delete: OnDelete = OnDelete.cascade, + on_update: OnUpdate = OnUpdate.cascade, + target_column: t.Union[str, Column, None] = None, + **kwargs, + ) -> None: + ... + + def __init__( + self, + references: t.Union[t.Type[ReferencedTable], LazyTableReference, str], default: t.Any = None, null: bool = True, on_delete: OnDelete = OnDelete.cascade, @@ -1859,7 +1904,6 @@ def __init__( from piccolo.table import Table if inspect.isclass(references): - references = t.cast(t.Type, references) if issubclass(references, Table): # Using this to validate the default value - will raise a # ValueError if incorrect. @@ -2078,6 +2122,40 @@ def set_proxy_columns(self) -> None: setattr(self, _column._meta.name, _column) _fk_meta.proxy_columns.append(_column) + @property + def _(self) -> t.Type[ReferencedTable]: + """ + This allows us specify joins in a way which is friendly to static type + checkers like Mypy and Pyright. + + Whilst this works:: + + # Fetch the band's name, and their manager's name via a foreign + # key: + await Band.select(Band.name, Band.manager.name) + + There currently isn't a 100% reliable way to tell static type checkers + that ``Band.manager.name`` refers to a ``name`` column on the + ``Manager`` table. + + However, by using the ``_`` property, it works perfectly. Instead + of ``Band.manager.name`` we use ``Band.manager._.name``:: + + await Band.select(Band.name, Band.manager._.name) + + So when doing joins, after every foreign key we use ``._.`` instead of + ``.``. An easy way to remember this is ``._.`` looks a bit like a + connector in a diagram. + + As Python's typing support increases, we'd love ``Band.manager.name`` + to have the same static typing as ``Band.manager._.name`` (using some + kind of ``Proxy`` type), but for now this is the best solution, and is + a huge improvement in developer experience, as static type checkers + easily know if any of your joins contain typos. + + """ + return t.cast(t.Type[ReferencedTable], self) + def __getattribute__(self, name: str) -> t.Union[Column, t.Any]: """ Returns attributes unmodified unless they're Column instances, in which @@ -2085,7 +2163,7 @@ def __getattribute__(self, name: str) -> t.Union[Column, t.Any]: joins required). """ # If the ForeignKey is using a lazy reference, we need to set the - # attributes here. Attributes starting with a double underscore are + # attributes here. Attributes starting with an underscore are # unlikely to be column names. if not name.startswith("__"): try: @@ -2105,6 +2183,9 @@ def __getattribute__(self, name: str) -> t.Union[Column, t.Any]: except AttributeError: raise AttributeError + if name == "_": + return value + foreignkey_class: t.Type[ForeignKey] = object.__getattribute__( self, "__class__" ) @@ -2162,7 +2243,7 @@ def __get__(self, obj: Table, objtype=None) -> t.Any: ... @t.overload - def __get__(self, obj: None, objtype=None) -> ForeignKey: + def __get__(self, obj: None, objtype=None) -> ForeignKey[ReferencedTable]: ... @t.overload diff --git a/piccolo/table.py b/piccolo/table.py index abae84a52..64fb66ea9 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -14,6 +14,7 @@ Array, Email, ForeignKey, + ReferencedTable, Secret, Serial, ) @@ -558,9 +559,19 @@ def refresh( """ return Refresh(instance=self, columns=columns) + @t.overload def get_related( - self: TableInstance, foreign_key: t.Union[ForeignKey, str] - ) -> First[Table]: + self, foreign_key: ForeignKey[ReferencedTable] + ) -> First[ReferencedTable]: + ... + + @t.overload + def get_related(self, foreign_key: str) -> First[Table]: + ... + + def get_related( + self, foreign_key: t.Union[str, ForeignKey[ReferencedTable]] + ) -> t.Union[First[Table], First[ReferencedTable]]: """ Used to fetch a ``Table`` instance, for the target of a foreign key. @@ -588,16 +599,12 @@ def get_related( column_name = foreign_key._meta.name - references: t.Type[ - Table - ] = foreign_key._foreign_key_meta.resolved_references + references = foreign_key._foreign_key_meta.resolved_references return ( references.objects() .where( - references._meta.get_column_by_name( - self._meta.primary_key._meta.name - ) + foreign_key._foreign_key_meta.resolved_target_column == getattr(self, column_name) ) .first() diff --git a/tests/columns/foreign_key/test_attribute_access.py b/tests/columns/foreign_key/test_attribute_access.py index 3f9d8afab..ccfc77818 100644 --- a/tests/columns/foreign_key/test_attribute_access.py +++ b/tests/columns/foreign_key/test_attribute_access.py @@ -7,7 +7,7 @@ class Manager(Table): name = Varchar() - manager = ForeignKey("self") + manager: ForeignKey["Manager"] = ForeignKey("self") class BandA(Table): @@ -15,11 +15,11 @@ class BandA(Table): class BandB(Table): - manager = ForeignKey(references="Manager") + manager: ForeignKey["Manager"] = ForeignKey(references="Manager") class BandC(Table): - manager = ForeignKey( + manager: ForeignKey["Manager"] = ForeignKey( references=LazyTableReference( table_class_name="Manager", module_path=__name__, @@ -28,7 +28,9 @@ class BandC(Table): class BandD(Table): - manager = ForeignKey(references=f"{__name__}.Manager") + manager: ForeignKey["Manager"] = ForeignKey( + references=f"{__name__}.Manager" + ) class TestAttributeAccess(TestCase): diff --git a/tests/columns/foreign_key/test_foreign_key_references.py b/tests/columns/foreign_key/test_foreign_key_references.py index 5e38a3ec2..76b5f2f39 100644 --- a/tests/columns/foreign_key/test_foreign_key_references.py +++ b/tests/columns/foreign_key/test_foreign_key_references.py @@ -13,7 +13,7 @@ class BandA(Table): class BandB(Table): - manager = ForeignKey(references="Manager") + manager: ForeignKey["Manager"] = ForeignKey(references="Manager") class TestReferences(TestCase): diff --git a/tests/columns/foreign_key/test_foreign_key_self.py b/tests/columns/foreign_key/test_foreign_key_self.py index a594379de..830c147ce 100644 --- a/tests/columns/foreign_key/test_foreign_key_self.py +++ b/tests/columns/foreign_key/test_foreign_key_self.py @@ -6,7 +6,7 @@ class Manager(Table, tablename="manager"): name = Varchar() - manager = ForeignKey("self", null=True) + manager: ForeignKey["Manager"] = ForeignKey("self", null=True) class TestForeignKeySelf(TestCase): diff --git a/tests/columns/foreign_key/test_foreign_key_string.py b/tests/columns/foreign_key/test_foreign_key_string.py index 1dd8e3aee..e37298734 100644 --- a/tests/columns/foreign_key/test_foreign_key_string.py +++ b/tests/columns/foreign_key/test_foreign_key_string.py @@ -9,11 +9,11 @@ class Manager(Table): class BandA(Table): - manager = ForeignKey(references="Manager") + manager: ForeignKey["Manager"] = ForeignKey(references="Manager") class BandB(Table): - manager = ForeignKey( + manager: ForeignKey["Manager"] = ForeignKey( references=LazyTableReference( table_class_name="Manager", module_path=__name__, @@ -22,7 +22,9 @@ class BandB(Table): class BandC(Table, tablename="band"): - manager = ForeignKey(references=f"{__name__}.Manager") + manager: ForeignKey["Manager"] = ForeignKey( + references=f"{__name__}.Manager" + ) class TestForeignKeyString(TestCase): diff --git a/tests/columns/foreign_key/test_value_type.py b/tests/columns/foreign_key/test_value_type.py index b1ca9808f..1c858d425 100644 --- a/tests/columns/foreign_key/test_value_type.py +++ b/tests/columns/foreign_key/test_value_type.py @@ -7,7 +7,7 @@ class Manager(Table): name = Varchar() - manager = ForeignKey("self", null=True) + manager: ForeignKey["Manager"] = ForeignKey("self", null=True) class Band(Table): diff --git a/tests/table/instance/test_get_related.py b/tests/table/instance/test_get_related.py index e68fdc57d..28c572314 100644 --- a/tests/table/instance/test_get_related.py +++ b/tests/table/instance/test_get_related.py @@ -1,3 +1,4 @@ +import typing as t from unittest import TestCase from tests.example_apps.music.tables import Band, Manager @@ -14,7 +15,7 @@ def tearDown(self): for table in reversed(TABLES): table.alter().drop_table().run_sync() - def test_get_related(self): + def test_get_related(self) -> None: """ Make sure you can get a related object from another object instance. """ @@ -25,15 +26,16 @@ def test_get_related(self): band.save().run_sync() _manager = band.get_related(Band.manager).run_sync() + assert _manager is not None self.assertTrue(_manager.name == "Guido") # Test non-ForeignKey with self.assertRaises(ValueError): - band.get_related(Band.name) + band.get_related(Band.name) # type: ignore # Make sure it also works using a string - _manager = band.get_related("manager").run_sync() - self.assertTrue(_manager.name == "Guido") + _manager_2 = t.cast(Manager, band.get_related("manager").run_sync()) + self.assertTrue(_manager_2.name == "Guido") # Test an invalid string with self.assertRaises(ValueError): diff --git a/tests/table/test_join.py b/tests/table/test_join.py index 72651c198..b5ebc867d 100644 --- a/tests/table/test_join.py +++ b/tests/table/test_join.py @@ -23,7 +23,6 @@ def test_create_join(self): class TestJoin(TestCase): - tables = [Manager, Band, Venue, Concert, Ticket] def setUp(self): @@ -98,6 +97,29 @@ def test_join(self): response = select_query.run_sync() self.assertEqual(response, [{"band_1.manager.name": "Guido"}]) + def test_underscore_syntax(self): + """ + Make sure that queries work with the ``._.`` syntax for joins. + """ + response = Concert.select( + Concert.band_1._.name, + Concert.band_1._.manager._.name, + Concert.band_2._.name, + Concert.band_2._.manager._.name, + ).run_sync() + + self.assertListEqual( + response, + [ + { + "band_1.name": "Pythonistas", + "band_1.manager.name": "Guido", + "band_2.name": "Rustaceans", + "band_2.manager.name": "Graydon", + } + ], + ) + def test_select_all_columns(self): """ Make sure you can retrieve all columns from a related table, without diff --git a/tests/testing/test_model_builder.py b/tests/testing/test_model_builder.py index 8e6e05434..4eead76bf 100644 --- a/tests/testing/test_model_builder.py +++ b/tests/testing/test_model_builder.py @@ -42,7 +42,7 @@ class TableWithDecimal(Table): class BandWithLazyReference(Table): - manager = ForeignKey( + manager: ForeignKey["Manager"] = ForeignKey( references=LazyTableReference( "Manager", module_path="tests.example_apps.music.tables" ) diff --git a/tests/type_checking.py b/tests/type_checking.py index 11f2cef6f..d1e9d96ca 100644 --- a/tests/type_checking.py +++ b/tests/type_checking.py @@ -9,9 +9,10 @@ from typing_extensions import assert_type +from piccolo.columns import ForeignKey, Varchar from piccolo.testing.model_builder import ModelBuilder -from .example_apps.music.tables import Band, Manager +from .example_apps.music.tables import Band, Concert, Manager if t.TYPE_CHECKING: @@ -33,6 +34,21 @@ async def get() -> None: assert_type(await query.run(), t.Optional[Band]) assert_type(query.run_sync(), t.Optional[Band]) + async def foreign_key_reference() -> None: + assert_type(Band.manager, ForeignKey[Manager]) + + async def foreign_key_traversal() -> None: + # Single level + assert_type(Band.manager._.name, Varchar) + # Multi level + assert_type(Concert.band_1._.manager._.name, Varchar) + + async def get_related() -> None: + band = await Band.objects().get(Band.name == "Pythonistas") + assert band is not None + manager = await band.get_related(Band.manager) + assert_type(manager, t.Optional[Manager]) + async def get_or_create() -> None: query = Band.objects().get_or_create(Band.name == "Pythonistas") assert_type(await query, Band) From 17e9448c987b0f320634f5c41e50ba2d1358573b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 23 Dec 2023 12:37:26 +0000 Subject: [PATCH 527/727] bumped version --- CHANGES.rst | 34 ++++++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 3d257acc4..fcb9a8fe4 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,40 @@ Changes ======= +1.2.0 +----- + +There's now an alternative syntax for joins, which works really well with +static type checkers like Mypy and Pylance. + +The traditional syntax (which continues to work as before): + +.. code-block:: python + + # Get the band name, and the manager's name from a related table + await Band.select(Band.name, Band.manager.name) + +The alternative syntax is as follows: + +.. code-block:: python + + await Band.select(Band.name, Band.manager._.name) + +Note how we use ``._.`` instead of ``.`` after a ``ForeignKey``. + +This offers a considerably better static typing experience. In the above +example, type checkers know that ``Band.manager._.name`` refers to the ``name`` +column on the ``Manager`` table. This means typos can be detected, and code +navigation is easier. + +Other changes +~~~~~~~~~~~~~ + +* Improve static typing for ``get_related``. +* Added support for the ``esmerald`` ASGI framework. + +------------------------------------------------------------------------------- + 1.1.1 ----- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 933b81888..55f78263a 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.1.1" +__VERSION__ = "1.2.0" From b888cef1318998109743211fa96e60fcf40af71e Mon Sep 17 00:00:00 2001 From: waldner Date: Mon, 22 Jan 2024 21:30:24 +0000 Subject: [PATCH 528/727] add "piccolo user list" command (#921) --- docs/src/piccolo/authentication/baseuser.rst | 5 +++ piccolo/apps/user/commands/list.py | 15 ++++++++ piccolo/apps/user/piccolo_app.py | 2 ++ piccolo/utils/printing.py | 36 ++++++++++++++++++++ 4 files changed, 58 insertions(+) create mode 100644 piccolo/apps/user/commands/list.py diff --git a/docs/src/piccolo/authentication/baseuser.rst b/docs/src/piccolo/authentication/baseuser.rst index ae687ad5b..1919f5cec 100644 --- a/docs/src/piccolo/authentication/baseuser.rst +++ b/docs/src/piccolo/authentication/baseuser.rst @@ -44,6 +44,11 @@ script), you can pass all of the arguments in as follows: If you choose this approach then be careful, as the password will be in the shell's history. +list +~~~~ + +List existing users. + change_password ~~~~~~~~~~~~~~~ diff --git a/piccolo/apps/user/commands/list.py b/piccolo/apps/user/commands/list.py new file mode 100644 index 000000000..de57c3614 --- /dev/null +++ b/piccolo/apps/user/commands/list.py @@ -0,0 +1,15 @@ +from piccolo.apps.user.tables import BaseUser +from piccolo.utils.printing import print_dict_table + + +def list(): + """ + List existing users. + """ + users = ( + BaseUser.select(BaseUser.all_columns(exclude=[BaseUser.password])) + .order_by(BaseUser.username) + .run_sync() + ) + + print_dict_table(users) diff --git a/piccolo/apps/user/piccolo_app.py b/piccolo/apps/user/piccolo_app.py index b523b1b72..635f65143 100644 --- a/piccolo/apps/user/piccolo_app.py +++ b/piccolo/apps/user/piccolo_app.py @@ -5,6 +5,7 @@ from .commands.change_password import change_password from .commands.change_permissions import change_permissions from .commands.create import create +from .commands.list import list from .tables import BaseUser CURRENT_DIRECTORY = os.path.dirname(os.path.abspath(__file__)) @@ -19,6 +20,7 @@ migration_dependencies=[], commands=[ Command(callable=create, aliases=["new"]), + Command(callable=list, aliases=["ls"]), Command(callable=change_password, aliases=["password", "pass"]), Command(callable=change_permissions, aliases=["perm", "perms"]), ], diff --git a/piccolo/utils/printing.py b/piccolo/utils/printing.py index c6adc76dd..03972b32d 100644 --- a/piccolo/utils/printing.py +++ b/piccolo/utils/printing.py @@ -1,3 +1,6 @@ +from typing import List + + def get_fixed_length_string(string: str, length=20) -> str: """ Add spacing to the end of the string so it's a fixed length, or truncate @@ -16,3 +19,36 @@ def print_heading(string: str, width: int = 64) -> None: """ print(f"\n{string.upper():^{width}}") print("-" * width) + + +def print_dict_table(data: List[dict], header_separator: bool = False) -> None: + """ + Prints out a list of dictionaries in tabular form. + Uses the first list element to extract the + column names and their order within the row. + """ + + if len(data) < 1: + print("No data") + return + + ref_order = [column for column in data[0]] + width = {column: len(str(column)) for column in ref_order} + + for item in data: + for column in ref_order: + if len(str(item[column])) > width[column]: + width[column] = len(str(item[column])) + + format_string = " | ".join([f"{{:<{width[w]}}}" for w in ref_order]) + + print(format_string.format(*[str(w) for w in ref_order])) + + if header_separator: + format_string_sep = "-+-".join( + [f"{{:<{width[w]}}}" for w in ref_order] + ) + print(format_string_sep.format(*["-" * width[w] for w in ref_order])) + + for item in data: + print(format_string.format(*[str(item[w]) for w in ref_order])) From bade80fe55dfbf9950d17dec6cd447fa05c19a27 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 22 Jan 2024 22:45:26 +0000 Subject: [PATCH 529/727] add a test for the `piccolo user list` command (#922) * add a test for the `piccolo user list` command * add a limit * minor tweaks to `print_dict_table` --- piccolo/apps/user/commands/list.py | 15 ++++++++--- piccolo/apps/user/piccolo_app.py | 4 +-- piccolo/conf/apps.py | 11 ++++++++ piccolo/main.py | 1 + piccolo/utils/printing.py | 37 +++++++++++++++------------ tests/apps/user/commands/test_list.py | 31 ++++++++++++++++++++++ 6 files changed, 77 insertions(+), 22 deletions(-) create mode 100644 tests/apps/user/commands/test_list.py diff --git a/piccolo/apps/user/commands/list.py b/piccolo/apps/user/commands/list.py index de57c3614..e78a12222 100644 --- a/piccolo/apps/user/commands/list.py +++ b/piccolo/apps/user/commands/list.py @@ -2,14 +2,23 @@ from piccolo.utils.printing import print_dict_table -def list(): +def list_users(limit: int = 20): """ List existing users. + + :param limit: + The maximum number of users to list. + """ users = ( - BaseUser.select(BaseUser.all_columns(exclude=[BaseUser.password])) + BaseUser.select(*BaseUser.all_columns(exclude=[BaseUser.password])) .order_by(BaseUser.username) + .limit(limit) .run_sync() ) - print_dict_table(users) + if len(users) == 0: + print("No data") + return + + print_dict_table(users, header_separator=True) diff --git a/piccolo/apps/user/piccolo_app.py b/piccolo/apps/user/piccolo_app.py index 635f65143..efa08d934 100644 --- a/piccolo/apps/user/piccolo_app.py +++ b/piccolo/apps/user/piccolo_app.py @@ -5,7 +5,7 @@ from .commands.change_password import change_password from .commands.change_permissions import change_permissions from .commands.create import create -from .commands.list import list +from .commands.list import list_users from .tables import BaseUser CURRENT_DIRECTORY = os.path.dirname(os.path.abspath(__file__)) @@ -20,7 +20,7 @@ migration_dependencies=[], commands=[ Command(callable=create, aliases=["new"]), - Command(callable=list, aliases=["ls"]), + Command(callable=list_users, command_name="list", aliases=["ls"]), Command(callable=change_password, aliases=["password", "pass"]), Command(callable=change_permissions, aliases=["perm", "perms"]), ], diff --git a/piccolo/conf/apps.py b/piccolo/conf/apps.py index df8ddad10..c311e1569 100644 --- a/piccolo/conf/apps.py +++ b/piccolo/conf/apps.py @@ -118,7 +118,18 @@ class Task(Table): # included @dataclass class Command: + """ + :param callable: + The function or method to be called. + :param command_name: + If not specified, the name of the ``callable`` is used. + :param aliases: + Alternative ways to refer to this command in the CLI. + + """ + callable: t.Callable + command_name: t.Optional[str] = None aliases: t.List[str] = field(default_factory=list) diff --git a/piccolo/main.py b/piccolo/main.py index 556b86f59..6b6a5dc27 100644 --- a/piccolo/main.py +++ b/piccolo/main.py @@ -75,6 +75,7 @@ def main() -> None: for command in _app_config.get_commands(): cli.register( command.callable, + command_name=command.command_name, group_name=_app_config.app_name, aliases=command.aliases, ) diff --git a/piccolo/utils/printing.py b/piccolo/utils/printing.py index 03972b32d..d1a0bd538 100644 --- a/piccolo/utils/printing.py +++ b/piccolo/utils/printing.py @@ -1,7 +1,7 @@ from typing import List -def get_fixed_length_string(string: str, length=20) -> str: +def get_fixed_length_string(string: str, length: int = 20) -> str: """ Add spacing to the end of the string so it's a fixed length, or truncate if it's too long. @@ -24,31 +24,34 @@ def print_heading(string: str, width: int = 64) -> None: def print_dict_table(data: List[dict], header_separator: bool = False) -> None: """ Prints out a list of dictionaries in tabular form. - Uses the first list element to extract the - column names and their order within the row. - """ - if len(data) < 1: - print("No data") - return + Uses the first list element to extract the column names and their order + within the row. + + """ + if len(data) == 0: + raise ValueError("The data must have at least one element.") - ref_order = [column for column in data[0]] - width = {column: len(str(column)) for column in ref_order} + column_names = data[0].keys() + widths = {column_name: len(column_name) for column_name in column_names} for item in data: - for column in ref_order: - if len(str(item[column])) > width[column]: - width[column] = len(str(item[column])) + for column in column_names: + width = len(str(item[column])) + if width > widths[column]: + widths[column] = width - format_string = " | ".join([f"{{:<{width[w]}}}" for w in ref_order]) + format_string = " | ".join(f"{{:<{widths[w]}}}" for w in column_names) - print(format_string.format(*[str(w) for w in ref_order])) + print(format_string.format(*[str(w) for w in column_names])) if header_separator: format_string_sep = "-+-".join( - [f"{{:<{width[w]}}}" for w in ref_order] + [f"{{:<{widths[w]}}}" for w in column_names] + ) + print( + format_string_sep.format(*["-" * widths[w] for w in column_names]) ) - print(format_string_sep.format(*["-" * width[w] for w in ref_order])) for item in data: - print(format_string.format(*[str(item[w]) for w in ref_order])) + print(format_string.format(*[str(item[w]) for w in column_names])) diff --git a/tests/apps/user/commands/test_list.py b/tests/apps/user/commands/test_list.py new file mode 100644 index 000000000..9c5f8875d --- /dev/null +++ b/tests/apps/user/commands/test_list.py @@ -0,0 +1,31 @@ +from unittest import TestCase +from unittest.mock import MagicMock, patch + +from piccolo.apps.user.commands.list import list_users +from piccolo.apps.user.tables import BaseUser + + +class TestList(TestCase): + def setUp(self): + BaseUser.create_table(if_not_exists=True).run_sync() + self.username = "test_user" + self.password = "abc123XYZ" + self.user = BaseUser.create_user_sync( + username=self.username, password=self.password + ) + + def tearDown(self): + BaseUser.alter().drop_table().run_sync() + + @patch("piccolo.utils.printing.print") + def test_list(self, print_mock: MagicMock): + """ + Make sure the user information is listed, excluding the password. + """ + list_users() + + output = "\n".join(i.args[0] for i in print_mock.call_args_list) + + assert self.username in output + assert self.password not in output + assert self.user.password not in output From 44940bbd99ed68a4babf145327236b3d97e6baf2 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 23 Jan 2024 12:18:13 +0000 Subject: [PATCH 530/727] make sure `ModeBuilder` generates tz aware values for `Timestamptz` (#923) --- piccolo/columns/column_types.py | 5 +++++ piccolo/testing/model_builder.py | 13 ++++++++----- piccolo/testing/random_builder.py | 19 ++++++++++--------- tests/testing/test_model_builder.py | 21 +++++++++++++++++++++ 4 files changed, 44 insertions(+), 14 deletions(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 33c0f261f..75634bdf3 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -1005,6 +1005,11 @@ class Concert(Table): """ value_type = datetime + + # Currently just used by ModelBuilder, to know that we want a timezone + # aware datetime. + tz_aware = True + timedelta_delegate = TimedeltaDelegate() def __init__( diff --git a/piccolo/testing/model_builder.py b/piccolo/testing/model_builder.py index 15b50416c..2994b3ec3 100644 --- a/piccolo/testing/model_builder.py +++ b/piccolo/testing/model_builder.py @@ -1,8 +1,8 @@ from __future__ import annotations +import datetime import json import typing as t -from datetime import date, datetime, time, timedelta from decimal import Decimal from uuid import UUID @@ -16,13 +16,13 @@ class ModelBuilder: __DEFAULT_MAPPER: t.Dict[t.Type, t.Callable] = { bool: RandomBuilder.next_bool, bytes: RandomBuilder.next_bytes, - date: RandomBuilder.next_date, - datetime: RandomBuilder.next_datetime, + datetime.date: RandomBuilder.next_date, + datetime.datetime: RandomBuilder.next_datetime, float: RandomBuilder.next_float, int: RandomBuilder.next_int, str: RandomBuilder.next_str, - time: RandomBuilder.next_time, - timedelta: RandomBuilder.next_timedelta, + datetime.time: RandomBuilder.next_time, + datetime.timedelta: RandomBuilder.next_timedelta, UUID: RandomBuilder.next_uuid, } @@ -155,6 +155,9 @@ def _randomize_attribute(cls, column: Column) -> t.Any: random_value = RandomBuilder.next_float( maximum=10 ** (precision - scale), scale=scale ) + elif column.value_type == datetime.datetime: + tz_aware = getattr(column, "tz_aware", False) + random_value = RandomBuilder.next_datetime(tz_aware=tz_aware) elif column.value_type == list: length = RandomBuilder.next_int(maximum=10) base_type = t.cast(Array, column).base_column.value_type diff --git a/piccolo/testing/random_builder.py b/piccolo/testing/random_builder.py index dfc46a9f2..bca29a7f2 100644 --- a/piccolo/testing/random_builder.py +++ b/piccolo/testing/random_builder.py @@ -1,9 +1,9 @@ +import datetime import enum import random import string import typing as t import uuid -from datetime import date, datetime, time, timedelta class RandomBuilder: @@ -16,22 +16,23 @@ def next_bytes(cls, length=8) -> bytes: return random.getrandbits(length * 8).to_bytes(length, "little") @classmethod - def next_date(cls) -> date: - return date( + def next_date(cls) -> datetime.date: + return datetime.date( year=random.randint(2000, 2050), month=random.randint(1, 12), day=random.randint(1, 28), ) @classmethod - def next_datetime(cls) -> datetime: - return datetime( + def next_datetime(cls, tz_aware: bool = False) -> datetime.datetime: + return datetime.datetime( year=random.randint(2000, 2050), month=random.randint(1, 12), day=random.randint(1, 28), hour=random.randint(0, 23), minute=random.randint(0, 59), second=random.randint(0, 59), + tzinfo=datetime.timezone.utc if tz_aware else None, ) @classmethod @@ -53,16 +54,16 @@ def next_str(cls, length=16) -> str: ) @classmethod - def next_time(cls) -> time: - return time( + def next_time(cls) -> datetime.time: + return datetime.time( hour=random.randint(0, 23), minute=random.randint(0, 59), second=random.randint(0, 59), ) @classmethod - def next_timedelta(cls) -> timedelta: - return timedelta( + def next_timedelta(cls) -> datetime.timedelta: + return datetime.timedelta( days=random.randint(1, 7), hours=random.randint(1, 23), minutes=random.randint(0, 59), diff --git a/tests/testing/test_model_builder.py b/tests/testing/test_model_builder.py index 4eead76bf..f56fcf956 100644 --- a/tests/testing/test_model_builder.py +++ b/tests/testing/test_model_builder.py @@ -11,6 +11,8 @@ LazyTableReference, Numeric, Real, + Timestamp, + Timestamptz, Varchar, ) from piccolo.table import Table, create_db_tables_sync, drop_db_tables_sync @@ -97,6 +99,25 @@ def test_choices(self): ["s", "l", "m"], ) + def test_datetime(self): + """ + Make sure that ``ModelBuilder`` generates timezone aware datetime + objects for ``Timestamptz`` columns, and timezone naive datetime + objects for ``Timestamp`` columns. + """ + + class Table1(Table): + starts = Timestamptz() + + class Table2(Table): + starts = Timestamp() + + model_1 = ModelBuilder.build_sync(Table1, persist=False) + assert model_1.starts.tzinfo is not None + + model_2 = ModelBuilder.build_sync(Table2, persist=False) + assert model_2.starts.tzinfo is None + def test_foreign_key(self): model = ModelBuilder.build_sync(Band, persist=True) From 1b0c521547d8d39046f3c165423db6defe62102b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 23 Jan 2024 13:25:53 +0000 Subject: [PATCH 531/727] add a `page` arg to `piccolo user list` command (#924) --- piccolo/apps/user/commands/list.py | 11 ++++++++++- tests/apps/user/commands/test_list.py | 18 ++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/piccolo/apps/user/commands/list.py b/piccolo/apps/user/commands/list.py index e78a12222..881f0397d 100644 --- a/piccolo/apps/user/commands/list.py +++ b/piccolo/apps/user/commands/list.py @@ -2,18 +2,27 @@ from piccolo.utils.printing import print_dict_table -def list_users(limit: int = 20): +def list_users(limit: int = 20, page: int = 1): """ List existing users. :param limit: The maximum number of users to list. + :param page: + Lets you paginate through the list of users. """ + if page < 1: + raise ValueError("The page number must > 0.") + + if limit < 1: + raise ValueError("The limit number must be > 0.") + users = ( BaseUser.select(*BaseUser.all_columns(exclude=[BaseUser.password])) .order_by(BaseUser.username) .limit(limit) + .offset(limit * (page - 1)) .run_sync() ) diff --git a/tests/apps/user/commands/test_list.py b/tests/apps/user/commands/test_list.py index 9c5f8875d..2a3c58bab 100644 --- a/tests/apps/user/commands/test_list.py +++ b/tests/apps/user/commands/test_list.py @@ -29,3 +29,21 @@ def test_list(self, print_mock: MagicMock): assert self.username in output assert self.password not in output assert self.user.password not in output + + +class TestListArgs(TestCase): + def test_limit(self): + """ + Make sure non-positive `limit` values are rejected. + """ + for value in (0, -1): + with self.assertRaises(ValueError): + list_users(page=value) + + def test_page(self): + """ + Make sure non-positive `page` values are rejected. + """ + for value in (0, -1): + with self.assertRaises(ValueError): + list_users(limit=value) From 6ec06e7272f9968d635547c284aa9b6d2fa20384 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 23 Jan 2024 21:23:47 +0000 Subject: [PATCH 532/727] Update ASGI requirements (#911) * remove `jinja2` Piccolo has `jinja2` as a direct dependency, so this shouldn't be needed. * remove `requeste` I think this was needed for Starlette at some point, but it has since transitioned to `httpx`, so I don't think it's needed any more. * move `piccolo_admin` to `ROUTER_DEPENDENCIES` We don't use it for `esmerald` so don't include it by default for all routers. * no longer omit `piccolo_admin` for `esmerald` * add `piccolo_admin` to `_esmerald_app.py.jinja` --- piccolo/apps/asgi/commands/new.py | 6 +++++- .../commands/templates/app/_esmerald_app.py.jinja | 11 ++++++++++- .../commands/templates/app/requirements.txt.jinja | 6 ++---- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index 38a056295..70ecefdc8 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -10,10 +10,14 @@ TEMPLATE_DIR = os.path.join(os.path.dirname(__file__), "templates/app/") SERVERS = ["uvicorn", "Hypercorn"] -ROUTERS = ["starlette", "fastapi", "blacksheep", "litestar", "esmerald"] ROUTER_DEPENDENCIES = { + "starlette": ["starlette"], "fastapi": ["fastapi>=0.100.0"], + "blacksheep": ["blacksheep"], + "litestar": ["litestar"], + "esmerald": ["esmerald"], } +ROUTERS = list(ROUTER_DEPENDENCIES.keys()) def print_instruction(message: str): diff --git a/piccolo/apps/asgi/commands/templates/app/_esmerald_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_esmerald_app.py.jinja index a69df7706..60560b980 100644 --- a/piccolo/apps/asgi/commands/templates/app/_esmerald_app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/_esmerald_app.py.jinja @@ -4,6 +4,7 @@ from pathlib import Path from piccolo.utils.pydantic import create_pydantic_model from piccolo.engine import engine_finder +from piccolo_admin.endpoints import create_admin from esmerald import ( Esmerald, @@ -92,7 +93,15 @@ class TaskAPIView(APIView): app = Esmerald( routes=[ Gateway("/", handler=home), - Gateway("/tasks", handler=TaskAPIView) + Gateway("/tasks", handler=TaskAPIView), + Include( + "/admin/", + create_admin( + tables=APP_CONFIG.table_classes, + # Required when running under HTTPS: + # allowed_hosts=['my_site.com'] + ), + ), ], static_files_config=StaticFilesConfig(path="/static", directory=Path("static")), on_startup=[open_database_connection_pool], diff --git a/piccolo/apps/asgi/commands/templates/app/requirements.txt.jinja b/piccolo/apps/asgi/commands/templates/app/requirements.txt.jinja index 7bbb8ba53..828796740 100644 --- a/piccolo/apps/asgi/commands/templates/app/requirements.txt.jinja +++ b/piccolo/apps/asgi/commands/templates/app/requirements.txt.jinja @@ -2,7 +2,5 @@ {{ router_dependency }} {% endfor -%} {{ server }} -jinja2 -piccolo[postgres] -piccolo_admin -requests +piccolo[postgres]>=1.0.0 +piccolo_admin>=1.0.0 \ No newline at end of file From bd8ec507e3a56188fdd3505bb3e708daf3f1a610 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 23 Jan 2024 22:25:17 +0000 Subject: [PATCH 533/727] add `order_by` argument to `piccolo user list` command (#925) --- piccolo/apps/user/commands/list.py | 52 +++++++++++++++++++--- tests/apps/user/commands/test_list.py | 62 ++++++++++++++++++++++++--- 2 files changed, 100 insertions(+), 14 deletions(-) diff --git a/piccolo/apps/user/commands/list.py b/piccolo/apps/user/commands/list.py index 881f0397d..5a39f6961 100644 --- a/piccolo/apps/user/commands/list.py +++ b/piccolo/apps/user/commands/list.py @@ -1,8 +1,33 @@ +import typing as t + from piccolo.apps.user.tables import BaseUser +from piccolo.columns import Column from piccolo.utils.printing import print_dict_table +ORDER_BY_COLUMN_NAMES = [ + i._meta.name for i in BaseUser.all_columns(exclude=[BaseUser.password]) +] + + +async def get_users( + order_by: Column, ascending: bool, limit: int, page: int +) -> t.List[t.Dict[str, t.Any]]: + return ( + await BaseUser.select( + *BaseUser.all_columns(exclude=[BaseUser.password]) + ) + .order_by( + order_by, + ascending=ascending, + ) + .limit(limit) + .offset(limit * (page - 1)) + ) -def list_users(limit: int = 20, page: int = 1): + +async def list_users( + limit: int = 20, page: int = 1, order_by: str = "username" +): """ List existing users. @@ -10,6 +35,9 @@ def list_users(limit: int = 20, page: int = 1): The maximum number of users to list. :param page: Lets you paginate through the list of users. + :param order_by: + The column used to order the results. Prefix with '-' for descending + order. """ if page < 1: @@ -18,12 +46,22 @@ def list_users(limit: int = 20, page: int = 1): if limit < 1: raise ValueError("The limit number must be > 0.") - users = ( - BaseUser.select(*BaseUser.all_columns(exclude=[BaseUser.password])) - .order_by(BaseUser.username) - .limit(limit) - .offset(limit * (page - 1)) - .run_sync() + ascending = True + if order_by.startswith("-"): + ascending = False + order_by = order_by[1:] + + if order_by not in ORDER_BY_COLUMN_NAMES: + raise ValueError( + "The order_by argument must be one of the following: " + + ", ".join(ORDER_BY_COLUMN_NAMES) + ) + + users = await get_users( + order_by=BaseUser._meta.get_column_by_name(order_by), + ascending=ascending, + limit=limit, + page=page, ) if len(users) == 0: diff --git a/tests/apps/user/commands/test_list.py b/tests/apps/user/commands/test_list.py index 2a3c58bab..2b35a4f65 100644 --- a/tests/apps/user/commands/test_list.py +++ b/tests/apps/user/commands/test_list.py @@ -1,8 +1,9 @@ from unittest import TestCase -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, patch from piccolo.apps.user.commands.list import list_users from piccolo.apps.user.tables import BaseUser +from piccolo.utils.sync import run_sync class TestList(TestCase): @@ -22,7 +23,7 @@ def test_list(self, print_mock: MagicMock): """ Make sure the user information is listed, excluding the password. """ - list_users() + run_sync(list_users()) output = "\n".join(i.args[0] for i in print_mock.call_args_list) @@ -31,19 +32,66 @@ def test_list(self, print_mock: MagicMock): assert self.user.password not in output -class TestListArgs(TestCase): - def test_limit(self): +class TestLimit(TestCase): + def test_non_positive(self): """ Make sure non-positive `limit` values are rejected. """ for value in (0, -1): with self.assertRaises(ValueError): - list_users(page=value) + run_sync(list_users(page=value)) - def test_page(self): + +class TestPage(TestCase): + def test_non_positive(self): """ Make sure non-positive `page` values are rejected. """ for value in (0, -1): with self.assertRaises(ValueError): - list_users(limit=value) + run_sync(list_users(limit=value)) + + +class TestOrder(TestCase): + @patch("piccolo.apps.user.commands.list.get_users") + def test_order(self, get_users: AsyncMock): + """ + Make sure valid column names are accepted. + """ + get_users.return_value = [] + run_sync(list_users(order_by="email")) + + self.assertDictEqual( + get_users.call_args.kwargs, + { + "order_by": BaseUser.email, + "ascending": True, + "limit": 20, + "page": 1, + }, + ) + + @patch("piccolo.apps.user.commands.list.get_users") + def test_descending(self, get_users: AsyncMock): + """ + Make sure a colume name prefixed with '-' works. + """ + get_users.return_value = [] + run_sync(list_users(order_by="-email")) + + self.assertDictEqual( + get_users.call_args.kwargs, + { + "order_by": BaseUser.email, + "ascending": False, + "limit": 20, + "page": 1, + }, + ) + + def test_unrecognised_column(self): + """ + Make sure invalid column names are rejected. + """ + with self.assertRaises(ValueError): + run_sync(list_users(order_by="abc123")) From 553e68872a60713daf0a7456f76d1b877a56235f Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 23 Jan 2024 22:55:11 +0000 Subject: [PATCH 534/727] allow SQLite auto migrations (#927) * allow SQLite auto migrations * simplify warning message --- piccolo/apps/migrations/commands/new.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piccolo/apps/migrations/commands/new.py b/piccolo/apps/migrations/commands/new.py index 09f6ac9e8..ff123aaa2 100644 --- a/piccolo/apps/migrations/commands/new.py +++ b/piccolo/apps/migrations/commands/new.py @@ -3,7 +3,6 @@ import datetime import os import string -import sys import typing as t from dataclasses import dataclass from itertools import chain @@ -22,6 +21,7 @@ from piccolo.conf.apps import AppConfig, Finder from piccolo.engine import SQLiteEngine from piccolo.utils.printing import print_heading +from piccolo.utils.warnings import colored_warning from .base import BaseMigrationManager @@ -232,7 +232,7 @@ async def new( """ engine = Finder().get_engine() if auto and isinstance(engine, SQLiteEngine): - sys.exit("Auto migrations aren't currently supported by SQLite.") + colored_warning("Auto migrations aren't fully supported by SQLite.") if app_name == "all" and not auto: raise ValueError( From 46e72f439ba5e9e441c57e3942622d0d6d1024bf Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 23 Jan 2024 23:03:35 +0000 Subject: [PATCH 535/727] bumped version --- CHANGES.rst | 17 +++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index fcb9a8fe4..13457e15b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,23 @@ Changes ======= +1.3.0 +----- + +Added the ``piccolo user list`` command - a quick and convenient way of listing +Piccolo Admin users from the command line. + +``ModelBuilder`` now creates timezone aware ``datetime`` objects for +``Timestamptz`` columns. + +Updated the ASGI templates. + +SQLite auto migrations are now allowed. We used to raise an exception, but +now we output a warning instead. While SQLite auto migrations aren't as feature +rich as Postgres, they work fine for simple use cases. + +------------------------------------------------------------------------------- + 1.2.0 ----- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 55f78263a..0b0ad2a01 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.2.0" +__VERSION__ = "1.3.0" From 74681b7d170fc6110e0db06f96f46dd35af2cc4e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 25 Feb 2024 22:37:08 +0000 Subject: [PATCH 536/727] fix type of bigserial foreign keys (#934) --- piccolo/columns/column_types.py | 18 +++-- tests/columns/foreign_key/test_column_type.py | 69 +++++++++++++++++++ 2 files changed, 80 insertions(+), 7 deletions(-) create mode 100644 tests/columns/foreign_key/test_column_type.py diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 75634bdf3..3e25a53dc 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -682,9 +682,7 @@ class Band(Table): """ - @property - def column_type(self): - engine_type = self._meta.engine_type + def _get_column_type(self, engine_type: str): if engine_type == "postgres": return "BIGINT" elif engine_type == "cockroach": @@ -693,6 +691,10 @@ def column_type(self): return "INTEGER" raise Exception("Unrecognized engine type") + @property + def column_type(self): + return self._get_column_type(engine_type=self._meta.engine_type) + ########################################################################### # Descriptors @@ -1841,10 +1843,12 @@ def column_type(self): column of the table being referenced. """ target_column = self._foreign_key_meta.resolved_target_column - engine_type = self._meta.engine_type - if engine_type == "cockroach": - return target_column.column_type - if isinstance(target_column, Serial): + + if isinstance(target_column, BigSerial): + return BigInt()._get_column_type( + engine_type=self._meta.engine_type + ) + elif isinstance(target_column, Serial): return Integer().column_type else: return target_column.column_type diff --git a/tests/columns/foreign_key/test_column_type.py b/tests/columns/foreign_key/test_column_type.py new file mode 100644 index 000000000..68f3d6b00 --- /dev/null +++ b/tests/columns/foreign_key/test_column_type.py @@ -0,0 +1,69 @@ +from unittest import TestCase + +from piccolo.columns import ( + UUID, + BigInt, + BigSerial, + ForeignKey, + Integer, + Serial, + Varchar, +) +from piccolo.table import Table + + +class TestColumnType(TestCase): + """ + The `column_type` of the `ForeignKey` should depend on the `PrimaryKey` of + the referenced table. + """ + + def test_serial(self): + class Manager(Table): + id = Serial(primary_key=True) + + class Band(Table): + manager = ForeignKey(Manager) + + self.assertEqual( + Band.manager.column_type, + Integer().column_type, + ) + + def test_bigserial(self): + class Manager(Table): + id = BigSerial(primary_key=True) + + class Band(Table): + manager = ForeignKey(Manager) + + self.assertEqual( + Band.manager.column_type, + BigInt()._get_column_type( + engine_type=Band.manager._meta.engine_type + ), + ) + + def test_uuid(self): + class Manager(Table): + id = UUID(primary_key=True) + + class Band(Table): + manager = ForeignKey(Manager) + + self.assertEqual( + Band.manager.column_type, + Manager.id.column_type, + ) + + def test_varchar(self): + class Manager(Table): + id = Varchar(primary_key=True) + + class Band(Table): + manager = ForeignKey(Manager) + + self.assertEqual( + Band.manager.column_type, + Manager.id.column_type, + ) From 4f66bd89e6add8a2d78392b69a0d3e4cecd88222 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 25 Feb 2024 22:40:28 +0000 Subject: [PATCH 537/727] bumped version --- CHANGES.rst | 8 ++++++++ piccolo/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 13457e15b..e30974b3d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,14 @@ Changes ======= +1.3.1 +----- + +Fixed a bug with foreign keys which reference ``BigSerial`` primary keys. +Thanks to @Abdelhadi92 for reporting this issue. + +------------------------------------------------------------------------------- + 1.3.0 ----- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 0b0ad2a01..579e98fd8 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.3.0" +__VERSION__ = "1.3.1" From 894efe56879971c2e154d6f34514a9e5d833f616 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 4 Mar 2024 11:58:04 +0000 Subject: [PATCH 538/727] 936 Handle setup of nested arrays containing `BigInt` (#937) * handle setup of nested arrays * added a test for nested arrays --- piccolo/columns/column_types.py | 9 +++++ piccolo/engine/sqlite.py | 4 +-- piccolo/table.py | 2 +- tests/columns/test_array.py | 60 ++++++++++++++++++++++++++------- 4 files changed, 60 insertions(+), 15 deletions(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 3e25a53dc..7012355d7 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -2588,6 +2588,15 @@ def column_type(self): return "ARRAY" raise Exception("Unrecognized engine type") + def _setup_base_column(self, table_class: t.Type[Table]): + """ + Called from the ``Table.__init_subclass__`` - makes sure + that the ``base_column`` has a reference to the parent table. + """ + self.base_column._meta._table = table_class + if isinstance(self.base_column, Array): + self.base_column._setup_base_column(table_class=table_class) + def __getitem__(self, value: int) -> Array: """ Allows queries which retrieve an item from the array. The index starts diff --git a/piccolo/engine/sqlite.py b/piccolo/engine/sqlite.py index ffc6606b0..862afaa71 100644 --- a/piccolo/engine/sqlite.py +++ b/piccolo/engine/sqlite.py @@ -85,8 +85,8 @@ def convert_array_in(value: list): """ Converts a list value into a string. """ - if value and type(value[0]) not in [str, int, float]: - raise ValueError("Can only serialise str, int and float.") + if value and type(value[0]) not in [str, int, float, list]: + raise ValueError("Can only serialise str, int, float, and list.") return dump_json(value) diff --git a/piccolo/table.py b/piccolo/table.py index 64fb66ea9..be2296597 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -303,7 +303,7 @@ def __init_subclass__( column._meta._table = cls if isinstance(column, Array): - column.base_column._meta._table = cls + column._setup_base_column(table_class=cls) if isinstance(column, Email): email_columns.append(column) diff --git a/tests/columns/test_array.py b/tests/columns/test_array.py index 0e3b24a6b..8db64c17d 100644 --- a/tests/columns/test_array.py +++ b/tests/columns/test_array.py @@ -1,6 +1,6 @@ from unittest import TestCase -from piccolo.columns.column_types import Array, Integer +from piccolo.columns.column_types import Array, BigInt, Integer from piccolo.table import Table from tests.base import engines_only, sqlite_only @@ -22,7 +22,7 @@ def test_array_default(self): class TestArray(TestCase): """ - Make sure an Array column can be created. + Make sure an Array column can be created, and work correctly. """ def setUp(self): @@ -35,9 +35,9 @@ def tearDown(self): def test_storage(self): """ Make sure data can be stored and retrieved. - """ - """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + """ # noqa: E501 MyTable(value=[1, 2, 3]).save().run_sync() @@ -48,9 +48,9 @@ def test_storage(self): def test_index(self): """ Indexes should allow individual array elements to be queried. - """ - """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + """ # noqa: E501 MyTable(value=[1, 2, 3]).save().run_sync() @@ -63,9 +63,9 @@ def test_all(self): """ Make sure rows can be retrieved where all items in an array match a given value. - """ - """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + """ # noqa: E501 MyTable(value=[1, 1, 1]).save().run_sync() @@ -90,9 +90,9 @@ def test_any(self): """ Make sure rows can be retrieved where any items in an array match a given value. - """ - """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + """ # noqa: E501 MyTable(value=[1, 2, 3]).save().run_sync() @@ -116,9 +116,9 @@ def test_any(self): def test_cat(self): """ Make sure values can be appended to an array. - """ - """ + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + """ # noqa: E501 MyTable(value=[1, 1, 1]).save().run_sync() @@ -163,3 +163,39 @@ def test_cat_sqlite(self): str(manager.exception), "Only Postgres and Cockroach support array appending.", ) + + +class NestedArrayTable(Table): + value = Array(base_column=Array(base_column=BigInt())) + + +class TestNestedArray(TestCase): + """ + Make sure that tables with nested arrays can be created, and work + correctly. + + Related to this bug, with nested array columns containing ``BigInt``: + + https://github.com/piccolo-orm/piccolo/issues/936 + + """ + + def setUp(self): + NestedArrayTable.create_table().run_sync() + + def tearDown(self): + NestedArrayTable.alter().drop_table().run_sync() + + @engines_only("postgres", "sqlite") + def test_storage(self): + """ + Make sure data can be stored and retrieved. + + 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + + """ # noqa: E501 + NestedArrayTable(value=[[1, 2, 3], [4, 5, 6]]).save().run_sync() + + row = NestedArrayTable.objects().first().run_sync() + assert row is not None + self.assertEqual(row.value, [[1, 2, 3], [4, 5, 6]]) From 2322a075562f8256487ded97bdb3559ffb878eb3 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 4 Mar 2024 12:02:01 +0000 Subject: [PATCH 539/727] bumped version --- CHANGES.rst | 14 ++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index e30974b3d..dbfae8610 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,20 @@ Changes ======= +1.3.2 +----- + +Fixed a bug with nested array columns containing ``BigInt``. For example: + +.. code-block:: python + + class MyTable(Table): + my_column = Array(Array(BigInt)) + +Thanks to @AmazingAkai for reporting this issue. + +------------------------------------------------------------------------------- + 1.3.1 ----- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 579e98fd8..922bcbf79 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.3.1" +__VERSION__ = "1.3.2" From 0e2ec8a892630427b0ca76a9a950a8677e5d5233 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 4 Mar 2024 17:45:17 +0000 Subject: [PATCH 540/727] 938 Improve `create_pydantic_model` for multidimensional arrays (#939) * handle multidimensional arrays in pydantic * make sure the inner type is also correct for example, if it's `Array(Array(Varchar()))`, the inner `Varchar` should be `constr`. * ignore type warning --- piccolo/utils/pydantic.py | 51 +++++++++++++++++++++++++++--------- tests/utils/test_pydantic.py | 26 ++++++++++++++++++ 2 files changed, 65 insertions(+), 12 deletions(-) diff --git a/piccolo/utils/pydantic.py b/piccolo/utils/pydantic.py index e245e522d..a11b88752 100644 --- a/piccolo/utils/pydantic.py +++ b/piccolo/utils/pydantic.py @@ -74,6 +74,43 @@ def validate_columns( ) +def get_array_value_type( + column: Array, inner: t.Optional[t.Type] = None +) -> t.Type: + """ + Gets the correct type for an ``Array`` column (which might be + multidimensional). + """ + if isinstance(column.base_column, Array): + inner_type = get_array_value_type(column.base_column, inner=inner) + else: + inner_type = get_pydantic_value_type(column.base_column) + + return t.List[inner_type] # type: ignore + + +def get_pydantic_value_type(column: Column) -> t.Type: + """ + Map the Piccolo ``Column`` to a Pydantic type. + """ + value_type: t.Type + + if isinstance(column, (Decimal, Numeric)): + value_type = pydantic.condecimal( + max_digits=column.precision, decimal_places=column.scale + ) + elif isinstance(column, Email): + value_type = pydantic.EmailStr # type: ignore + elif isinstance(column, Varchar): + value_type = pydantic.constr(max_length=column.length) + elif isinstance(column, Array): + value_type = get_array_value_type(column=column) + else: + value_type = column.value_type + + return value_type + + def create_pydantic_model( table: t.Type[Table], nested: t.Union[bool, t.Tuple[ForeignKey, ...]] = False, @@ -211,17 +248,7 @@ def create_pydantic_model( ####################################################################### # Work out the column type - if isinstance(column, (Decimal, Numeric)): - value_type: t.Type = pydantic.condecimal( - max_digits=column.precision, decimal_places=column.scale - ) - elif isinstance(column, Email): - value_type = pydantic.EmailStr - elif isinstance(column, Varchar): - value_type = pydantic.constr(max_length=column.length) - elif isinstance(column, Array): - value_type = t.List[column.base_column.value_type] # type: ignore - elif isinstance(column, (JSON, JSONB)): + if isinstance(column, (JSON, JSONB)): if deserialize_json: value_type = pydantic.Json else: @@ -235,7 +262,7 @@ def create_pydantic_model( validator # type: ignore ) else: - value_type = column.value_type + value_type = get_pydantic_value_type(column=column) _type = t.Optional[value_type] if is_optional else value_type diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index a6e003f7f..2c773b8de 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -126,6 +126,32 @@ class Band(Table): "string", ) + def test_multidimensional_array(self): + """ + Make sure that multidimensional arrays have the correct type. + """ + + class Band(Table): + members = Array(Array(Varchar(length=255)), required=True) + + pydantic_model = create_pydantic_model(table=Band) + + self.assertEqual( + pydantic_model.model_fields["members"].annotation, + t.List[t.List[pydantic.constr(max_length=255)]], + ) + + # Should not raise a validation error: + pydantic_model( + members=[ + ["Alice", "Bob", "Francis"], + ["Alan", "Georgia", "Sue"], + ] + ) + + with self.assertRaises(ValueError): + pydantic_model(members=["Bob"]) + class TestForeignKeyColumn(TestCase): def test_target_column(self): From 6d50b0aac6d51367176bd837e4b66e25a5bfa952 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 11 Mar 2024 20:25:58 +0000 Subject: [PATCH 541/727] fix pylance warnings in tests (#947) --- tests/apps/app/commands/test_new.py | 4 +- .../apps/fixtures/commands/test_dump_load.py | 2 - tests/apps/fixtures/commands/test_shared.py | 4 +- .../migrations/auto/test_migration_manager.py | 15 +-- tests/apps/user/test_tables.py | 16 ++- tests/base.py | 18 +++- tests/columns/foreign_key/test_all_columns.py | 8 +- tests/columns/foreign_key/test_all_related.py | 8 +- .../foreign_key/test_attribute_access.py | 2 +- .../foreign_key/test_foreign_key_self.py | 3 +- tests/columns/foreign_key/test_schema.py | 2 +- tests/columns/m2m/base.py | 99 +++++++++++-------- tests/columns/m2m/test_m2m.py | 25 +---- tests/columns/m2m/test_m2m_schema.py | 36 +------ tests/columns/test_array.py | 1 + tests/columns/test_base.py | 2 +- tests/columns/test_boolean.py | 16 ++- tests/columns/test_bytea.py | 2 +- tests/columns/test_choices.py | 1 + tests/columns/test_date.py | 2 + tests/columns/test_db_column_name.py | 5 +- tests/columns/test_defaults.py | 16 +-- tests/columns/test_double_precision.py | 1 + tests/columns/test_interval.py | 2 + tests/columns/test_json.py | 32 +++--- tests/columns/test_jsonb.py | 3 + tests/columns/test_numeric.py | 1 + tests/columns/test_primary_key.py | 10 +- tests/columns/test_real.py | 1 + tests/columns/test_time.py | 2 + tests/columns/test_timestamp.py | 2 + tests/columns/test_timestamptz.py | 2 + tests/conftest.py | 2 +- tests/engine/test_extra_nodes.py | 4 + tests/query/mixins/test_columns_delegate.py | 20 ++-- tests/query/test_slots.py | 2 +- .../instance/test_get_related_readable.py | 2 +- tests/table/instance/test_to_dict.py | 5 + tests/table/test_alter.py | 23 +++-- tests/table/test_batch.py | 1 + tests/table/test_inheritance.py | 2 + tests/table/test_insert.py | 3 +- tests/table/test_join.py | 64 +++++++----- tests/table/test_join_on.py | 6 +- tests/table/test_objects.py | 18 ++-- tests/table/test_output.py | 4 +- tests/table/test_repr.py | 1 + tests/table/test_select.py | 34 ++++++- tests/table/test_update.py | 15 ++- tests/test_schema.py | 9 +- tests/testing/test_model_builder.py | 3 + 51 files changed, 328 insertions(+), 233 deletions(-) diff --git a/tests/apps/app/commands/test_new.py b/tests/apps/app/commands/test_new.py index efdacdb5f..fe2addd88 100644 --- a/tests/apps/app/commands/test_new.py +++ b/tests/apps/app/commands/test_new.py @@ -39,5 +39,7 @@ def test_new_with_clashing_name(self): exception = context.exception self.assertTrue( - exception.code.startswith("A module called sys already exists") + str(exception.code).startswith( + "A module called sys already exists" + ) ) diff --git a/tests/apps/fixtures/commands/test_dump_load.py b/tests/apps/fixtures/commands/test_dump_load.py index 407bd4504..59c4d04a2 100644 --- a/tests/apps/fixtures/commands/test_dump_load.py +++ b/tests/apps/fixtures/commands/test_dump_load.py @@ -276,5 +276,3 @@ def test_on_conflict(self): run_sync(load(path=json_file_path, on_conflict="DO NOTHING")) run_sync(load(path=json_file_path, on_conflict="DO UPDATE")) - run_sync(load(path=json_file_path, on_conflict="do nothing")) - run_sync(load(path=json_file_path, on_conflict="do update")) diff --git a/tests/apps/fixtures/commands/test_shared.py b/tests/apps/fixtures/commands/test_shared.py index b2246aa71..34e2af4ee 100644 --- a/tests/apps/fixtures/commands/test_shared.py +++ b/tests/apps/fixtures/commands/test_shared.py @@ -56,5 +56,5 @@ def test_shared(self): } model = pydantic_model(**data) - self.assertEqual(model.mega.SmallTable[0].id, 1) - self.assertEqual(model.mega.MegaTable[0].id, 1) + self.assertEqual(model.mega.SmallTable[0].id, 1) # type: ignore + self.assertEqual(model.mega.MegaTable[0].id, 1) # type: ignore diff --git a/tests/apps/migrations/auto/test_migration_manager.py b/tests/apps/migrations/auto/test_migration_manager.py index 8d58bb8ce..a1988a029 100644 --- a/tests/apps/migrations/auto/test_migration_manager.py +++ b/tests/apps/migrations/auto/test_migration_manager.py @@ -1,5 +1,6 @@ import asyncio import random +import typing as t from io import StringIO from unittest import TestCase from unittest.mock import MagicMock, patch @@ -267,7 +268,7 @@ def test_add_table(self, get_app_config: MagicMock): self.assertEqual(self.table_exists("musician"), False) @engines_only("postgres", "cockroach") - def test_add_column(self): + def test_add_column(self) -> None: """ Test adding a column to a MigrationManager. """ @@ -304,21 +305,21 @@ def test_add_column(self): response = self.run_sync("SELECT * FROM manager;") self.assertEqual(response, [{"id": 1, "name": "Dave"}]) - id = 0 + row_id: t.Optional[int] = None if engine_is("cockroach"): - id = self.run_sync( + row_id = self.run_sync( "INSERT INTO manager VALUES (default, 'Dave', 'dave@me.com') RETURNING id;" # noqa: E501 - ) + )[0]["id"] response = self.run_sync("SELECT * FROM manager;") self.assertEqual( response, - [{"id": id[0]["id"], "name": "Dave", "email": "dave@me.com"}], + [{"id": row_id, "name": "Dave", "email": "dave@me.com"}], ) # Reverse asyncio.run(manager.run(backwards=True)) response = self.run_sync("SELECT * FROM manager;") - self.assertEqual(response, [{"id": id[0]["id"], "name": "Dave"}]) + self.assertEqual(response, [{"id": row_id, "name": "Dave"}]) # Preview manager.preview = True @@ -333,7 +334,7 @@ def test_add_column(self): if engine_is("postgres"): self.assertEqual(response, [{"id": 1, "name": "Dave"}]) if engine_is("cockroach"): - self.assertEqual(response, [{"id": id[0]["id"], "name": "Dave"}]) + self.assertEqual(response, [{"id": row_id, "name": "Dave"}]) @engines_only("postgres", "cockroach") def test_add_column_with_index(self): diff --git a/tests/apps/user/test_tables.py b/tests/apps/user/test_tables.py index d48e2cc75..59d274905 100644 --- a/tests/apps/user/test_tables.py +++ b/tests/apps/user/test_tables.py @@ -221,7 +221,10 @@ def test_long_password_error(self): def test_no_username_error(self): with self.assertRaises(ValueError) as manager: - BaseUser.create_user_sync(username=None, password="abc123") + BaseUser.create_user_sync( + username=None, # type: ignore + password="abc123", + ) self.assertEqual( manager.exception.__str__(), "A username must be provided." @@ -229,7 +232,10 @@ def test_no_username_error(self): def test_no_password_error(self): with self.assertRaises(ValueError) as manager: - BaseUser.create_user_sync(username="bob", password=None) + BaseUser.create_user_sync( + username="bob", + password=None, # type: ignore + ) self.assertEqual( manager.exception.__str__(), "A password must be provided." @@ -272,12 +278,14 @@ def test_hash_update(self): BaseUser.login_sync(username=username, password=password) ) - hashed_password = ( + user_data = ( BaseUser.select(BaseUser.password) .where(BaseUser.id == user.id) .first() - .run_sync()["password"] + .run_sync() ) + assert user_data is not None + hashed_password = user_data["password"] algorithm, iterations_, salt, hashed = BaseUser.split_stored_password( hashed_password diff --git a/tests/base.py b/tests/base.py index 6f9bfbb55..b05f85622 100644 --- a/tests/base.py +++ b/tests/base.py @@ -19,19 +19,19 @@ ENGINE = engine_finder() -def engine_version_lt(version: float): - return ENGINE and run_sync(ENGINE.get_version()) < version +def engine_version_lt(version: float) -> bool: + return ENGINE is not None and run_sync(ENGINE.get_version()) < version -def is_running_postgres(): +def is_running_postgres() -> bool: return type(ENGINE) is PostgresEngine -def is_running_sqlite(): +def is_running_sqlite() -> bool: return type(ENGINE) is SQLiteEngine -def is_running_cockroach(): +def is_running_cockroach() -> bool: return type(ENGINE) is CockroachEngine @@ -228,6 +228,8 @@ def get_postgres_varchar_length( ########################################################################### def create_tables(self): + assert ENGINE is not None + if ENGINE.engine_type in ("postgres", "cockroach"): self.run_sync( """ @@ -308,6 +310,8 @@ def create_tables(self): raise Exception("Unrecognised engine") def insert_row(self): + assert ENGINE is not None + if ENGINE.engine_type == "cockroach": id = self.run_sync( """ @@ -352,6 +356,8 @@ def insert_row(self): ) def insert_rows(self): + assert ENGINE is not None + if ENGINE.engine_type == "cockroach": id = self.run_sync( """ @@ -428,6 +434,8 @@ def insert_many_rows(self, row_count=10000): self.run_sync(f"INSERT INTO manager (name) VALUES {values_string};") def drop_tables(self): + assert ENGINE is not None + if ENGINE.engine_type in ("postgres", "cockroach"): self.run_sync("DROP TABLE IF EXISTS band CASCADE;") self.run_sync("DROP TABLE IF EXISTS manager CASCADE;") diff --git a/tests/columns/foreign_key/test_all_columns.py b/tests/columns/foreign_key/test_all_columns.py index e2718ce5b..0d6828ddf 100644 --- a/tests/columns/foreign_key/test_all_columns.py +++ b/tests/columns/foreign_key/test_all_columns.py @@ -24,17 +24,17 @@ def test_all_columns_deep(self): """ Make sure ``all_columns`` works when the joins are several layers deep. """ - all_columns = Concert.band_1.manager.all_columns() - self.assertEqual(all_columns, [Band.manager.id, Band.manager.name]) + all_columns = Concert.band_1._.manager.all_columns() + self.assertEqual(all_columns, [Band.manager._.id, Band.manager._.name]) # Make sure the call chains are also correct. self.assertEqual( all_columns[0]._meta.call_chain, - Concert.band_1.manager.id._meta.call_chain, + Concert.band_1._.manager._.id._meta.call_chain, ) self.assertEqual( all_columns[1]._meta.call_chain, - Concert.band_1.manager.name._meta.call_chain, + Concert.band_1._.manager._.name._meta.call_chain, ) def test_all_columns_exclude(self): diff --git a/tests/columns/foreign_key/test_all_related.py b/tests/columns/foreign_key/test_all_related.py index 737ad1924..94ebf7dc2 100644 --- a/tests/columns/foreign_key/test_all_related.py +++ b/tests/columns/foreign_key/test_all_related.py @@ -38,13 +38,13 @@ def test_all_related_deep(self): """ Make sure ``all_related`` works when the joins are several layers deep. """ - all_related = Ticket.concert.band_1.all_related() - self.assertEqual(all_related, [Ticket.concert.band_1.manager]) + all_related = Ticket.concert._.band_1.all_related() + self.assertEqual(all_related, [Ticket.concert._.band_1._.manager]) # Make sure the call chains are also correct. self.assertEqual( all_related[0]._meta.call_chain, - Ticket.concert.band_1.manager._meta.call_chain, + Ticket.concert._.band_1._.manager._meta.call_chain, ) def test_all_related_exclude(self): @@ -57,6 +57,6 @@ def test_all_related_exclude(self): ) self.assertEqual( - Ticket.concert.all_related(exclude=[Ticket.concert.venue]), + Ticket.concert.all_related(exclude=[Ticket.concert._.venue]), [Ticket.concert.band_1, Ticket.concert.band_2], ) diff --git a/tests/columns/foreign_key/test_attribute_access.py b/tests/columns/foreign_key/test_attribute_access.py index ccfc77818..597b33bd6 100644 --- a/tests/columns/foreign_key/test_attribute_access.py +++ b/tests/columns/foreign_key/test_attribute_access.py @@ -60,6 +60,6 @@ def test_recursion_time(self): Make sure that a really large call chain doesn't take too long. """ start = time.time() - Manager.manager.manager.manager.manager.manager.manager.name + Manager.manager._.manager._.manager._.manager._.manager._.manager._.name # noqa: E501 end = time.time() self.assertLess(end - start, 1.0) diff --git a/tests/columns/foreign_key/test_foreign_key_self.py b/tests/columns/foreign_key/test_foreign_key_self.py index 830c147ce..18c35e337 100644 --- a/tests/columns/foreign_key/test_foreign_key_self.py +++ b/tests/columns/foreign_key/test_foreign_key_self.py @@ -1,10 +1,11 @@ from unittest import TestCase -from piccolo.columns import ForeignKey, Varchar +from piccolo.columns import ForeignKey, Serial, Varchar from piccolo.table import Table class Manager(Table, tablename="manager"): + id: Serial name = Varchar() manager: ForeignKey["Manager"] = ForeignKey("self", null=True) diff --git a/tests/columns/foreign_key/test_schema.py b/tests/columns/foreign_key/test_schema.py index 121e32ebd..7e6b45c18 100644 --- a/tests/columns/foreign_key/test_schema.py +++ b/tests/columns/foreign_key/test_schema.py @@ -84,7 +84,7 @@ def test_with_schema(self): query = Concert.select( Concert.start_date, Concert.band.name.as_alias("band_name"), - Concert.band.manager.name.as_alias("manager_name"), + Concert.band._.manager._.name.as_alias("manager_name"), ) self.assertIn('"schema_1"."concert"', query.__str__()) self.assertIn('"schema_1"."band"', query.__str__()) diff --git a/tests/columns/m2m/base.py b/tests/columns/m2m/base.py index a3f282d23..6386ffcaf 100644 --- a/tests/columns/m2m/base.py +++ b/tests/columns/m2m/base.py @@ -1,27 +1,53 @@ import typing as t +from piccolo.columns.column_types import ( + ForeignKey, + LazyTableReference, + Serial, + Text, + Varchar, +) +from piccolo.columns.m2m import M2M from piccolo.engine.finder import engine_finder +from piccolo.schema import SchemaManager from piccolo.table import Table, create_db_tables_sync, drop_db_tables_sync from tests.base import engine_is, engines_skip engine = engine_finder() +class Band(Table): + id: Serial + name = Varchar() + genres = M2M(LazyTableReference("GenreToBand", module_path=__name__)) + + +class Genre(Table): + id: Serial + name = Varchar() + bands = M2M(LazyTableReference("GenreToBand", module_path=__name__)) + + +class GenreToBand(Table): + id: Serial + band = ForeignKey(Band) + genre = ForeignKey(Genre) + reason = Text(help_text="For testing additional columns on join tables.") + + class M2MBase: """ This allows us to test M2M when the tables are in different schemas (public vs non-public). """ - band: t.Type[Table] - genre: t.Type[Table] - genre_to_band: t.Type[Table] - all_tables: t.List[t.Type[Table]] + def _setUp(self, schema: t.Optional[str] = None): + self.schema = schema - def setUp(self): - Band = self.band - Genre = self.genre - GenreToBand = self.genre_to_band + for table_class in (Band, Genre, GenreToBand): + table_class._meta.schema = schema + + self.all_tables = [Band, Genre, GenreToBand] create_db_tables_sync(*self.all_tables, if_not_exists=True) @@ -77,14 +103,22 @@ def setUp(self): def tearDown(self): drop_db_tables_sync(*self.all_tables) + if self.schema: + SchemaManager().drop_schema( + schema_name="schema_1", cascade=True + ).run_sync() + + def assertEqual(self, first, second, msg=None): + assert first == second + + def assertTrue(self, first, msg=None): + assert first is True + @engines_skip("cockroach") def test_select_name(self): """ 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg """ # noqa: E501 - Band = self.band - Genre = self.genre - response = Band.select( Band.name, Band.genres(Genre.name, as_list=True) ).run_sync() @@ -118,9 +152,6 @@ def test_no_related(self): """ 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg """ # noqa: E501 - Band = self.band - Genre = self.genre - GenreToBand = self.genre_to_band GenreToBand.delete(force=True).run_sync() @@ -156,8 +187,6 @@ def test_select_multiple(self): """ 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg """ # noqa: E501 - Band = self.band - Genre = self.genre response = Band.select( Band.name, Band.genres(Genre.id, Genre.name) @@ -218,8 +247,6 @@ def test_select_id(self): """ 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg """ # noqa: E501 - Band = self.band - Genre = self.genre response = Band.select( Band.name, Band.genres(Genre.id, as_list=True) @@ -257,8 +284,6 @@ def test_select_all_columns(self): 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg """ # noqa: E501 - Band = self.band - Genre = self.genre response = Band.select( Band.name, Band.genres(Genre.all_columns(exclude=(Genre.id,))) @@ -288,11 +313,9 @@ def test_add_m2m(self): """ Make sure we can add items to the joining table. """ - Band = self.band - Genre = self.genre - GenreToBand = self.genre_to_band band = Band.objects().get(Band.name == "Pythonistas").run_sync() + assert band is not None band.add_m2m(Genre(name="Punk Rock"), m2m=Band.genres).run_sync() self.assertTrue( @@ -314,13 +337,11 @@ def test_extra_columns_str(self): Make sure the ``extra_column_values`` parameter for ``add_m2m`` works correctly when the dictionary keys are strings. """ - Band = self.band - Genre = self.genre - GenreToBand = self.genre_to_band reason = "Their second album was very punk rock." band = Band.objects().get(Band.name == "Pythonistas").run_sync() + assert band is not None band.add_m2m( Genre(name="Punk Rock"), m2m=Band.genres, @@ -329,7 +350,7 @@ def test_extra_columns_str(self): }, ).run_sync() - genre_to_band = ( + Genreto_band = ( GenreToBand.objects() .get( (GenreToBand.band.name == "Pythonistas") @@ -337,21 +358,20 @@ def test_extra_columns_str(self): ) .run_sync() ) + assert Genreto_band is not None - self.assertEqual(genre_to_band.reason, reason) + self.assertEqual(Genreto_band.reason, reason) def test_extra_columns_class(self): """ Make sure the ``extra_column_values`` parameter for ``add_m2m`` works correctly when the dictionary keys are ``Column`` classes. """ - Band = self.band - Genre = self.genre - GenreToBand = self.genre_to_band reason = "Their second album was very punk rock." band = Band.objects().get(Band.name == "Pythonistas").run_sync() + assert band is not None band.add_m2m( Genre(name="Punk Rock"), m2m=Band.genres, @@ -360,7 +380,7 @@ def test_extra_columns_class(self): }, ).run_sync() - genre_to_band = ( + Genreto_band = ( GenreToBand.objects() .get( (GenreToBand.band.name == "Pythonistas") @@ -368,20 +388,20 @@ def test_extra_columns_class(self): ) .run_sync() ) + assert Genreto_band is not None - self.assertEqual(genre_to_band.reason, reason) + self.assertEqual(Genreto_band.reason, reason) def test_add_m2m_existing(self): """ Make sure we can add an existing element to the joining table. """ - Band = self.band - Genre = self.genre - GenreToBand = self.genre_to_band band = Band.objects().get(Band.name == "Pythonistas").run_sync() + assert band is not None genre = Genre.objects().get(Genre.name == "Classical").run_sync() + assert genre is not None band.add_m2m(genre, m2m=Band.genres).run_sync() @@ -404,9 +424,9 @@ def test_get_m2m(self): """ Make sure we can get related items via the joining table. """ - Band = self.band band = Band.objects().get(Band.name == "Pythonistas").run_sync() + assert band is not None genres = band.get_m2m(Band.genres).run_sync() @@ -418,13 +438,12 @@ def test_remove_m2m(self): """ Make sure we can remove related items via the joining table. """ - Band = self.band - Genre = self.genre - GenreToBand = self.genre_to_band band = Band.objects().get(Band.name == "Pythonistas").run_sync() + assert band is not None genre = Genre.objects().get(Genre.name == "Rock").run_sync() + assert genre is not None band.remove_m2m(genre, m2m=Band.genres).run_sync() diff --git a/tests/columns/m2m/test_m2m.py b/tests/columns/m2m/test_m2m.py index d897eace6..85731a715 100644 --- a/tests/columns/m2m/test_m2m.py +++ b/tests/columns/m2m/test_m2m.py @@ -44,27 +44,9 @@ engine = engine_finder() -class Band(Table): - name = Varchar() - genres = M2M(LazyTableReference("GenreToBand", module_path=__name__)) - - -class Genre(Table): - name = Varchar() - bands = M2M(LazyTableReference("GenreToBand", module_path=__name__)) - - -class GenreToBand(Table): - band = ForeignKey(Band) - genre = ForeignKey(Genre) - reason = Text(help_text="For testing additional columns on join tables.") - - class TestM2M(M2MBase, TestCase): - band = Band - genre = Genre - genre_to_band = GenreToBand - all_tables = [Band, Genre, GenreToBand] + def setUp(self): + return self._setUp(schema=None) ############################################################################### @@ -161,6 +143,7 @@ def test_add_m2m(self): Make sure we can add items to the joining table. """ customer = Customer.objects().get(Customer.name == "Bob").run_sync() + assert customer is not None customer.add_m2m( Concert(name="Jazzfest"), m2m=Customer.concerts ).run_sync() @@ -192,6 +175,7 @@ def test_add_m2m_within_transaction(self): async def add_m2m_in_transaction(): async with engine.transaction(): customer = await Customer.objects().get(Customer.name == "Bob") + assert customer is not None await customer.add_m2m( Concert(name="Jazzfest"), m2m=Customer.concerts ) @@ -217,6 +201,7 @@ def test_get_m2m(self): Make sure we can get related items via the joining table. """ customer = Customer.objects().get(Customer.name == "Bob").run_sync() + assert customer is not None concerts = customer.get_m2m(Customer.concerts).run_sync() diff --git a/tests/columns/m2m/test_m2m_schema.py b/tests/columns/m2m/test_m2m_schema.py index f9b958838..01ed90681 100644 --- a/tests/columns/m2m/test_m2m_schema.py +++ b/tests/columns/m2m/test_m2m_schema.py @@ -1,35 +1,10 @@ from unittest import TestCase -from piccolo.columns.column_types import ( - ForeignKey, - LazyTableReference, - Text, - Varchar, -) -from piccolo.columns.m2m import M2M -from piccolo.schema import SchemaManager -from piccolo.table import Table from tests.base import engines_skip from .base import M2MBase -class Band(Table, schema="schema_1"): - name = Varchar() - genres = M2M(LazyTableReference("GenreToBand", module_path=__name__)) - - -class Genre(Table, schema="schema_1"): - name = Varchar() - bands = M2M(LazyTableReference("GenreToBand", module_path=__name__)) - - -class GenreToBand(Table, schema="schema_1"): - band = ForeignKey(Band) - genre = ForeignKey(Genre) - reason = Text(help_text="For testing additional columns on join tables.") - - @engines_skip("sqlite") class TestM2MWithSchema(M2MBase, TestCase): """ @@ -37,12 +12,5 @@ class TestM2MWithSchema(M2MBase, TestCase): works. """ - band = Band - genre = Genre - genre_to_band = GenreToBand - all_tables = [Band, Genre, GenreToBand] - - def tearDown(self): - SchemaManager().drop_schema( - schema_name="schema_1", cascade=True - ).run_sync() + def setUp(self): + return self._setUp(schema="schema_1") diff --git a/tests/columns/test_array.py b/tests/columns/test_array.py index 8db64c17d..bc3c863db 100644 --- a/tests/columns/test_array.py +++ b/tests/columns/test_array.py @@ -42,6 +42,7 @@ def test_storage(self): MyTable(value=[1, 2, 3]).save().run_sync() row = MyTable.objects().first().run_sync() + assert row is not None self.assertEqual(row.value, [1, 2, 3]) @engines_only("postgres") diff --git a/tests/columns/test_base.py b/tests/columns/test_base.py index db111f6f1..2ef4c2a8c 100644 --- a/tests/columns/test_base.py +++ b/tests/columns/test_base.py @@ -121,7 +121,7 @@ def test_non_column(self): Make sure non-column values don't match. """ for value in (1, "abc", None): - self.assertFalse(Manager.name._equals(value)) + self.assertFalse(Manager.name._equals(value)) # type: ignore def test_equals(self): """ diff --git a/tests/columns/test_boolean.py b/tests/columns/test_boolean.py index eea3df8d0..4c67ef6db 100644 --- a/tests/columns/test_boolean.py +++ b/tests/columns/test_boolean.py @@ -1,3 +1,4 @@ +import typing as t from unittest import TestCase from piccolo.columns.column_types import Boolean @@ -15,23 +16,30 @@ def setUp(self): def tearDown(self): MyTable.alter().drop_table().run_sync() - def test_return_type(self): + def test_return_type(self) -> None: for value in (True, False, None, ...): - kwargs = {} if value is ... else {"boolean": value} + kwargs: t.Dict[str, t.Any] = ( + {} if value is ... else {"boolean": value} + ) expected = MyTable.boolean.default if value is ... else value row = MyTable(**kwargs) row.save().run_sync() self.assertEqual(row.boolean, expected) - self.assertEqual( + row_from_db = ( MyTable.select(MyTable.boolean) .where( MyTable._meta.primary_key == getattr(row, MyTable._meta.primary_key._meta.name) ) .first() - .run_sync()["boolean"], + .run_sync() + ) + assert row_from_db is not None + + self.assertEqual( + row_from_db["boolean"], expected, ) diff --git a/tests/columns/test_bytea.py b/tests/columns/test_bytea.py index 666a5a8d6..6976a8840 100644 --- a/tests/columns/test_bytea.py +++ b/tests/columns/test_bytea.py @@ -59,4 +59,4 @@ def test_json_default(self): def test_invalid_default(self): with self.assertRaises(ValueError): for value in ("a", 1, ("x", "y", "z")): - Bytea(default=value) + Bytea(default=value) # type: ignore diff --git a/tests/columns/test_choices.py b/tests/columns/test_choices.py index ccea6b97f..d127a87d0 100644 --- a/tests/columns/test_choices.py +++ b/tests/columns/test_choices.py @@ -34,6 +34,7 @@ def test_default(self): """ Shirt().save().run_sync() shirt = Shirt.objects().first().run_sync() + assert shirt is not None self.assertEqual(shirt.size, "l") def test_update(self): diff --git a/tests/columns/test_date.py b/tests/columns/test_date.py index 3169d8bf6..5c1016211 100644 --- a/tests/columns/test_date.py +++ b/tests/columns/test_date.py @@ -27,6 +27,7 @@ def test_timestamp(self): row.save().run_sync() result = MyTable.objects().first().run_sync() + assert result is not None self.assertEqual(result.created_on, created_on) @@ -43,4 +44,5 @@ def test_timestamp(self): row.save().run_sync() result = MyTableDefault.objects().first().run_sync() + assert result is not None self.assertEqual(result.created_on, created_on) diff --git a/tests/columns/test_db_column_name.py b/tests/columns/test_db_column_name.py index 24d9bd9eb..33beffed9 100644 --- a/tests/columns/test_db_column_name.py +++ b/tests/columns/test_db_column_name.py @@ -1,9 +1,10 @@ -from piccolo.columns.column_types import Integer, Varchar +from piccolo.columns.column_types import Integer, Serial, Varchar from piccolo.table import Table from tests.base import DBTestCase, engine_is, engines_only, engines_skip class Band(Table): + id: Serial name = Varchar(db_column_name="regrettable_column_name") popularity = Integer() @@ -48,6 +49,7 @@ def test_save(self): band.save().run_sync() band_from_db = Band.objects().first().run_sync() + assert band_from_db is not None self.assertEqual(band_from_db.name, "Pythonistas") def test_create(self): @@ -62,6 +64,7 @@ def test_create(self): self.assertEqual(band.name, "Pythonistas") band_from_db = Band.objects().first().run_sync() + assert band_from_db is not None self.assertEqual(band_from_db.name, "Pythonistas") def test_select(self): diff --git a/tests/columns/test_defaults.py b/tests/columns/test_defaults.py index db95b165c..77df731bf 100644 --- a/tests/columns/test_defaults.py +++ b/tests/columns/test_defaults.py @@ -35,32 +35,32 @@ def test_int(self): _type(default=0) _type(default=None, null=True) with self.assertRaises(ValueError): - _type(default="hello world") + _type(default="hello world") # type: ignore def test_text(self): for _type in (Text, Varchar): _type(default="") _type(default=None, null=True) with self.assertRaises(ValueError): - _type(default=123) + _type(default=123) # type: ignore def test_real(self): Real(default=0.0) Real(default=None, null=True) with self.assertRaises(ValueError): - Real(default="hello world") + Real(default="hello world") # type: ignore def test_double_precision(self): DoublePrecision(default=0.0) DoublePrecision(default=None, null=True) with self.assertRaises(ValueError): - DoublePrecision(default="hello world") + DoublePrecision(default="hello world") # type: ignore def test_numeric(self): Numeric(default=decimal.Decimal(1.0)) Numeric(default=None, null=True) with self.assertRaises(ValueError): - Numeric(default="hello world") + Numeric(default="hello world") # type: ignore def test_uuid(self): UUID(default=None, null=True) @@ -74,21 +74,21 @@ def test_time(self): Time(default=TimeNow()) Time(default=datetime.datetime.now().time()) with self.assertRaises(ValueError): - Time(default="hello world") + Time(default="hello world") # type: ignore def test_date(self): Date(default=None, null=True) Date(default=DateNow()) Date(default=datetime.datetime.now().date()) with self.assertRaises(ValueError): - Date(default="hello world") + Date(default="hello world") # type: ignore def test_timestamp(self): Timestamp(default=None, null=True) Timestamp(default=TimestampNow()) Timestamp(default=datetime.datetime.now()) with self.assertRaises(ValueError): - Timestamp(default="hello world") + Timestamp(default="hello world") # type: ignore def test_foreignkey(self): class MyTable(Table): diff --git a/tests/columns/test_double_precision.py b/tests/columns/test_double_precision.py index 20d63331b..99e411f11 100644 --- a/tests/columns/test_double_precision.py +++ b/tests/columns/test_double_precision.py @@ -20,5 +20,6 @@ def test_creation(self): row.save().run_sync() _row = MyTable.objects().first().run_sync() + assert _row is not None self.assertEqual(type(_row.column_a), float) self.assertAlmostEqual(_row.column_a, 1.23) diff --git a/tests/columns/test_interval.py b/tests/columns/test_interval.py index 11f30c670..d9e7c6f10 100644 --- a/tests/columns/test_interval.py +++ b/tests/columns/test_interval.py @@ -48,6 +48,7 @@ def test_interval(self): .first() .run_sync() ) + assert result is not None self.assertEqual(result.interval, interval) def test_interval_where_clause(self): @@ -102,4 +103,5 @@ def test_interval(self): row.save().run_sync() result = MyTableDefault.objects().first().run_sync() + assert result is not None self.assertEqual(result.interval.days, 1) diff --git a/tests/columns/test_json.py b/tests/columns/test_json.py index 932c244d3..69808163c 100644 --- a/tests/columns/test_json.py +++ b/tests/columns/test_json.py @@ -34,11 +34,11 @@ def test_json_string(self): row = MyTable(json='{"a": 1}') row.save().run_sync() + row_from_db = MyTable.select(MyTable.json).first().run_sync() + assert row_from_db is not None + self.assertEqual( - MyTable.select(MyTable.json) - .first() - .run_sync()["json"] - .replace(" ", ""), + row_from_db["json"].replace(" ", ""), '{"a":1}', ) @@ -49,11 +49,11 @@ def test_json_object(self): row = MyTable(json={"a": 1}) row.save().run_sync() + row_from_db = MyTable.select(MyTable.json).first().run_sync() + assert row_from_db is not None + self.assertEqual( - MyTable.select(MyTable.json) - .first() - .run_sync()["json"] - .replace(" ", ""), + row_from_db["json"].replace(" ", ""), '{"a":1}', ) @@ -78,7 +78,7 @@ def test_json_default(self): def test_invalid_default(self): with self.assertRaises(ValueError): for value in ("a", 1, ("x", "y", "z")): - JSON(default=value) + JSON(default=value) # type: ignore class TestJSONInsert(TestCase): @@ -89,11 +89,10 @@ def tearDown(self): MyTable.alter().drop_table().run_sync() def check_response(self): + row = MyTable.select(MyTable.json).first().run_sync() + assert row is not None self.assertEqual( - MyTable.select(MyTable.json) - .first() - .run_sync()["json"] - .replace(" ", ""), + row["json"].replace(" ", ""), '{"message":"original"}', ) @@ -125,11 +124,10 @@ def add_row(self): row.save().run_sync() def check_response(self): + row = MyTable.select(MyTable.json).first().run_sync() + assert row is not None self.assertEqual( - MyTable.select(MyTable.json) - .first() - .run_sync()["json"] - .replace(" ", ""), + row["json"].replace(" ", ""), '{"message":"updated"}', ) diff --git a/tests/columns/test_jsonb.py b/tests/columns/test_jsonb.py index 4a2ed1395..7c2be3a5a 100644 --- a/tests/columns/test_jsonb.py +++ b/tests/columns/test_jsonb.py @@ -159,6 +159,7 @@ def test_arrow(self): .first() .run_sync() ) + assert row is not None self.assertEqual(row["facilities"], "true") row = ( @@ -169,6 +170,7 @@ def test_arrow(self): .first() .run_sync() ) + assert row is not None self.assertEqual(row["facilities"], True) def test_arrow_as_alias(self): @@ -188,6 +190,7 @@ def test_arrow_as_alias(self): .first() .run_sync() ) + assert row is not None self.assertEqual(row["mixing_desk"], "true") def test_arrow_where(self): diff --git a/tests/columns/test_numeric.py b/tests/columns/test_numeric.py index 129db5946..872d739a9 100644 --- a/tests/columns/test_numeric.py +++ b/tests/columns/test_numeric.py @@ -22,6 +22,7 @@ def test_creation(self): row.save().run_sync() _row = MyTable.objects().first().run_sync() + assert _row is not None self.assertEqual(type(_row.column_a), Decimal) self.assertEqual(type(_row.column_b), Decimal) diff --git a/tests/columns/test_primary_key.py b/tests/columns/test_primary_key.py index 98d1f5d4c..1850944cc 100644 --- a/tests/columns/test_primary_key.py +++ b/tests/columns/test_primary_key.py @@ -133,6 +133,7 @@ def test_primary_key_queries(self): ) manager_dict = Manager.select().first().run_sync() + assert manager_dict is not None self.assertEqual( [i for i in manager_dict.keys()], @@ -151,6 +152,7 @@ def test_primary_key_queries(self): band.save().run_sync() band_dict = Band.select().first().run_sync() + assert band_dict is not None self.assertEqual( [i for i in band_dict.keys()], ["pk", "name", "manager"] @@ -163,6 +165,7 @@ def test_primary_key_queries(self): # type (i.e. `uuid.UUID`). manager = Manager.objects().first().run_sync() + assert manager is not None band_2 = Band(manager=manager.pk, name="Pythonistas 2") band_2.save().run_sync() @@ -178,9 +181,10 @@ def test_primary_key_queries(self): ####################################################################### # Make sure `get_related` works - self.assertEqual( - band_2.get_related(Band.manager).run_sync().pk, manager.pk - ) + manager_from_db = band_2.get_related(Band.manager).run_sync() + assert manager_from_db is not None + + self.assertEqual(manager_from_db.pk, manager.pk) ####################################################################### # Make sure `remove` works diff --git a/tests/columns/test_real.py b/tests/columns/test_real.py index 30dc4338d..3257111de 100644 --- a/tests/columns/test_real.py +++ b/tests/columns/test_real.py @@ -20,5 +20,6 @@ def test_creation(self): row.save().run_sync() _row = MyTable.objects().first().run_sync() + assert _row is not None self.assertEqual(type(_row.column_a), float) self.assertAlmostEqual(_row.column_a, 1.23) diff --git a/tests/columns/test_time.py b/tests/columns/test_time.py index b0be9768e..a6d931448 100644 --- a/tests/columns/test_time.py +++ b/tests/columns/test_time.py @@ -30,6 +30,7 @@ def test_timestamp(self): row.save().run_sync() result = MyTable.objects().first().run_sync() + assert result is not None self.assertEqual(result.created_on, created_on) @@ -49,6 +50,7 @@ def test_timestamp(self): _datetime = partial(datetime.datetime, year=2020, month=1, day=1) result = MyTableDefault.objects().first().run_sync() + assert result is not None self.assertLess( _datetime( hour=result.created_on.hour, diff --git a/tests/columns/test_timestamp.py b/tests/columns/test_timestamp.py index 2c79728e9..ad1fa01f0 100644 --- a/tests/columns/test_timestamp.py +++ b/tests/columns/test_timestamp.py @@ -35,6 +35,7 @@ def test_timestamp(self): row.save().run_sync() result = MyTable.objects().first().run_sync() + assert result is not None self.assertEqual(result.created_on, created_on) def test_timezone_aware(self): @@ -61,6 +62,7 @@ def test_timestamp(self): row.save().run_sync() result = MyTableDefault.objects().first().run_sync() + assert result is not None self.assertLess( result.created_on - created_on, datetime.timedelta(seconds=1) ) diff --git a/tests/columns/test_timestamptz.py b/tests/columns/test_timestamptz.py index 09755e340..8e239900b 100644 --- a/tests/columns/test_timestamptz.py +++ b/tests/columns/test_timestamptz.py @@ -71,6 +71,7 @@ def test_timestamptz_timezone_aware(self): .first() .run_sync() ) + assert result is not None self.assertEqual(result.created_on, created_on) # The database converts it to UTC @@ -93,6 +94,7 @@ def test_timestamptz_default(self): row.save().run_sync() result = MyTableDefault.objects().first().run_sync() + assert result is not None delta = result.created_on - created_on self.assertLess(delta, datetime.timedelta(seconds=1)) self.assertEqual(result.created_on.tzinfo, datetime.timezone.utc) diff --git a/tests/conftest.py b/tests/conftest.py index dd71cba21..457c4dae7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,7 +22,7 @@ async def drop_tables(): "mega_table", "small_table", ] - assert ENGINE + assert ENGINE is not None if ENGINE.engine_type == "sqlite": # SQLite doesn't allow us to drop more than one table at a time. diff --git a/tests/engine/test_extra_nodes.py b/tests/engine/test_extra_nodes.py index 2e078c9ee..0d59868c2 100644 --- a/tests/engine/test_extra_nodes.py +++ b/tests/engine/test_extra_nodes.py @@ -1,3 +1,4 @@ +import typing as t from unittest import TestCase from unittest.mock import MagicMock @@ -16,6 +17,9 @@ def test_extra_nodes(self): """ # Get the test database credentials: test_engine = engine_finder() + assert test_engine is not None + + test_engine = t.cast(PostgresEngine, test_engine) EXTRA_NODE = MagicMock(spec=PostgresEngine(config=test_engine.config)) EXTRA_NODE.run_querystring = AsyncMock(return_value=[]) diff --git a/tests/query/mixins/test_columns_delegate.py b/tests/query/mixins/test_columns_delegate.py index dd9110431..e16a13dda 100644 --- a/tests/query/mixins/test_columns_delegate.py +++ b/tests/query/mixins/test_columns_delegate.py @@ -37,26 +37,26 @@ def test_as_of(self): self.insert_rows() time.sleep(1) # Ensure time travel queries have some history to use! - result = ( + query = ( Band.select() .where(Band.name == "Pythonistas") .as_of("-500ms") .limit(1) ) - self.assertTrue("AS OF SYSTEM TIME '-500ms'" in str(result)) - result = result.run_sync() + self.assertTrue("AS OF SYSTEM TIME '-500ms'" in str(query)) + result = query.run_sync() self.assertTrue(result[0]["name"] == "Pythonistas") - result = Band.select().as_of() - self.assertTrue("AS OF SYSTEM TIME '-1s'" in str(result)) - result = result.run_sync() + query = Band.select().as_of() + self.assertTrue("AS OF SYSTEM TIME '-1s'" in str(query)) + result = query.run_sync() self.assertTrue(result[0]["name"] == "Pythonistas") # Alternative syntax. - result = Band.objects().get(Band.name == "Pythonistas").as_of("-1s") - self.assertTrue("AS OF SYSTEM TIME '-1s'" in str(result)) - result = result.run_sync() + query = Band.objects().get(Band.name == "Pythonistas").as_of("-1s") + self.assertTrue("AS OF SYSTEM TIME '-1s'" in str(query)) + result = query.run_sync() - self.assertTrue(result.name == "Pythonistas") + self.assertTrue(result.name == "Pythonistas") # type: ignore diff --git a/tests/query/test_slots.py b/tests/query/test_slots.py index 971019910..ad6322502 100644 --- a/tests/query/test_slots.py +++ b/tests/query/test_slots.py @@ -41,4 +41,4 @@ def test_attributes(self): AttributeError, msg=f"{class_name} didn't raised an error" ): print(f"Setting {class_name} attribute") - query_class(table=Manager).abc = 123 + query_class(table=Manager).abc = 123 # type: ignore diff --git a/tests/table/instance/test_get_related_readable.py b/tests/table/instance/test_get_related_readable.py index 2088c867f..982c4a5bc 100644 --- a/tests/table/instance/test_get_related_readable.py +++ b/tests/table/instance/test_get_related_readable.py @@ -34,7 +34,7 @@ def get_readable(cls): columns=[ cls.name, cls.thing_two.name, - cls.thing_two.thing_one.name, + cls.thing_two._.thing_one._.name, ], ) diff --git a/tests/table/instance/test_to_dict.py b/tests/table/instance/test_to_dict.py index b40bad790..b5d75f52b 100644 --- a/tests/table/instance/test_to_dict.py +++ b/tests/table/instance/test_to_dict.py @@ -10,6 +10,7 @@ def test_to_dict(self): self.insert_row() instance = Manager.objects().first().run_sync() + assert instance is not None dictionary = instance.to_dict() if engine_is("cockroach"): self.assertDictEqual( @@ -26,6 +27,7 @@ def test_nested(self): self.insert_row() instance = Band.objects(Band.manager).first().run_sync() + assert instance is not None dictionary = instance.to_dict() if engine_is("cockroach"): self.assertDictEqual( @@ -58,6 +60,7 @@ def test_filter_rows(self): self.insert_row() instance = Manager.objects().first().run_sync() + assert instance is not None dictionary = instance.to_dict(Manager.name) self.assertDictEqual(dictionary, {"name": "Guido"}) @@ -69,6 +72,7 @@ def test_nested_filter(self): self.insert_row() instance = Band.objects(Band.manager).first().run_sync() + assert instance is not None dictionary = instance.to_dict(Band.name, Band.manager.id) if engine_is("cockroach"): self.assertDictEqual( @@ -94,6 +98,7 @@ def test_aliases(self): self.insert_row() instance = Manager.objects().first().run_sync() + assert instance is not None dictionary = instance.to_dict( Manager.id, Manager.name.as_alias("title") ) diff --git a/tests/table/test_alter.py b/tests/table/test_alter.py index e8df1b004..836a552a8 100644 --- a/tests/table/test_alter.py +++ b/tests/table/test_alter.py @@ -90,7 +90,7 @@ class TestDropColumn(DBTestCase): SQLite has very limited support for ALTER statements. """ - def _test_drop(self, column: str): + def _test_drop(self, column: t.Union[str, Column]): self.insert_row() Band.alter().drop_column(column).run_sync() @@ -229,10 +229,9 @@ def test_integer_to_bigint(self): "BIGINT", ) - popularity = ( - Band.select(Band.popularity).first().run_sync()["popularity"] - ) - self.assertEqual(popularity, 1000) + row = Band.select(Band.popularity).first().run_sync() + assert row is not None + self.assertEqual(row["popularity"], 1000) def test_integer_to_varchar(self): """ @@ -252,10 +251,9 @@ def test_integer_to_varchar(self): "CHARACTER VARYING", ) - popularity = ( - Band.select(Band.popularity).first().run_sync()["popularity"] - ) - self.assertEqual(popularity, "1000") + row = Band.select(Band.popularity).first().run_sync() + assert row is not None + self.assertEqual(row["popularity"], "1000") def test_using_expression(self): """ @@ -271,8 +269,9 @@ def test_using_expression(self): ) alter_query.run_sync() - popularity = Band.select(Band.name).first().run_sync()["name"] - self.assertEqual(popularity, 1) + row = Band.select(Band.name).first().run_sync() + assert row is not None + self.assertEqual(row["name"], 1) @engines_only("postgres", "cockroach") @@ -321,12 +320,12 @@ def test_set_default(self): ).run_sync() manager = Manager.objects().first().run_sync() + assert manager is not None self.assertEqual(manager.name, "Pending") @engines_only("postgres", "cockroach") class TestSetSchema(TestCase): - schema_manager = SchemaManager() schema_name = "schema_1" diff --git a/tests/table/test_batch.py b/tests/table/test_batch.py index 8fca2944c..2de9e0c7d 100644 --- a/tests/table/test_batch.py +++ b/tests/table/test_batch.py @@ -104,6 +104,7 @@ def test_batch_extra_node(self): # Get the test database credentials: test_engine = engine_finder() + assert isinstance(test_engine, PostgresEngine) EXTRA_NODE = AsyncMock(spec=PostgresEngine(config=test_engine.config)) diff --git a/tests/table/test_inheritance.py b/tests/table/test_inheritance.py index 8030bb4b7..a7ab2c90e 100644 --- a/tests/table/test_inheritance.py +++ b/tests/table/test_inheritance.py @@ -61,6 +61,7 @@ def test_inheritance(self): ).save().run_sync() response = Manager.select().first().run_sync() + assert response is not None self.assertEqual(response["started_on"], started_on) self.assertEqual(response["name"], name) self.assertEqual(response["favourite"], favourite) @@ -98,6 +99,7 @@ def test_inheritance(self): _Table(name=name, started_on=started_on).save().run_sync() response = _Table.select().first().run_sync() + assert response is not None self.assertEqual(response["started_on"], started_on) self.assertEqual(response["name"], name) diff --git a/tests/table/test_insert.py b/tests/table/test_insert.py index 1c5fab732..b2c58e378 100644 --- a/tests/table/test_insert.py +++ b/tests/table/test_insert.py @@ -3,7 +3,7 @@ import pytest -from piccolo.columns import Integer, Varchar +from piccolo.columns import Integer, Serial, Varchar from piccolo.query.methods.insert import OnConflictAction from piccolo.table import Table from piccolo.utils.lazy_loader import LazyLoader @@ -98,6 +98,7 @@ def test_insert_returning_alias(self): ) class TestOnConflict(TestCase): class Band(Table): + id: Serial name = Varchar(unique=True) popularity = Integer() diff --git a/tests/table/test_join.py b/tests/table/test_join.py index b5ebc867d..1d3aba2f3 100644 --- a/tests/table/test_join.py +++ b/tests/table/test_join.py @@ -93,7 +93,7 @@ def test_join(self): ) # Now make sure that even deeper joins work: - select_query = Concert.select(Concert.band_1.manager.name) + select_query = Concert.select(Concert.band_1._.manager._.name) response = select_query.run_sync() self.assertEqual(response, [{"band_1.manager.name": "Guido"}]) @@ -126,10 +126,11 @@ def test_select_all_columns(self): explicitly specifying them. """ result = ( - Band.select(Band.name, Band.manager.all_columns()) + Band.select(Band.name, *Band.manager.all_columns()) .first() .run_sync() ) + assert result is not None if engine_is("cockroach"): self.assertDictEqual( @@ -156,13 +157,14 @@ def test_select_all_columns_deep(self): """ result = ( Concert.select( - Concert.venue.all_columns(), - Concert.band_1.manager.all_columns(), - Concert.band_2.manager.all_columns(), + *Concert.venue.all_columns(), + *Concert.band_1._.manager.all_columns(), + *Concert.band_2._.manager.all_columns(), ) .first() .run_sync() ) + assert result is not None if engine_is("cockroach"): self.assertDictEqual( @@ -203,7 +205,8 @@ def test_proxy_columns(self): # We call it multiple times to make sure it doesn't change with time. for _ in range(2): self.assertEqual( - len(Concert.band_1.manager._foreign_key_meta.proxy_columns), 2 + len(Concert.band_1._.manager._foreign_key_meta.proxy_columns), + 2, ) self.assertEqual( len(Concert.band_1._foreign_key_meta.proxy_columns), 4 @@ -216,12 +219,13 @@ def test_select_all_columns_root(self): """ result = ( Band.select( - Band.all_columns(), - Band.manager.all_columns(), + *Band.all_columns(), + *Band.manager.all_columns(), ) .first() .run_sync() ) + assert result is not None if engine_is("cockroach"): self.assertDictEqual( @@ -254,11 +258,12 @@ def test_select_all_columns_root_nested(self): with using it for referenced tables. """ result = ( - Band.select(Band.all_columns(), Band.manager.all_columns()) + Band.select(*Band.all_columns(), *Band.manager.all_columns()) .output(nested=True) .first() .run_sync() ) + assert result is not None if engine_is("cockroach"): self.assertDictEqual( @@ -290,23 +295,25 @@ def test_select_all_columns_exclude(self): """ result = ( Band.select( - Band.all_columns(exclude=[Band.id]), - Band.manager.all_columns(exclude=[Band.manager.id]), + *Band.all_columns(exclude=[Band.id]), + *Band.manager.all_columns(exclude=[Band.manager.id]), ) .output(nested=True) .first() .run_sync() ) + assert result is not None result_str_args = ( Band.select( - Band.all_columns(exclude=["id"]), - Band.manager.all_columns(exclude=["id"]), + *Band.all_columns(exclude=["id"]), + *Band.manager.all_columns(exclude=["id"]), ) .output(nested=True) .first() .run_sync() ) + assert result_str_args is not None for data in (result, result_str_args): self.assertDictEqual( @@ -325,6 +332,7 @@ def test_objects_nested(self): Make sure the prefetch argument works correctly for objects. """ band = Band.objects(Band.manager).first().run_sync() + assert band is not None self.assertIsInstance(band.manager, Manager) def test_objects__all_related__root(self): @@ -333,6 +341,7 @@ def test_objects__all_related__root(self): root table of the query. """ concert = Concert.objects(Concert.all_related()).first().run_sync() + assert concert is not None self.assertIsInstance(concert.band_1, Band) self.assertIsInstance(concert.band_2, Band) self.assertIsInstance(concert.venue, Venue) @@ -344,15 +353,16 @@ def test_objects_nested_deep(self): ticket = ( Ticket.objects( Ticket.concert, - Ticket.concert.band_1, - Ticket.concert.band_2, - Ticket.concert.venue, - Ticket.concert.band_1.manager, - Ticket.concert.band_2.manager, + Ticket.concert._.band_1, + Ticket.concert._.band_2, + Ticket.concert._.venue, + Ticket.concert._.band_1._.manager, + Ticket.concert._.band_2._.manager, ) .first() .run_sync() ) + assert ticket is not None self.assertIsInstance(ticket.concert, Concert) self.assertIsInstance(ticket.concert.band_1, Band) @@ -370,12 +380,13 @@ def test_objects__all_related__deep(self): Ticket.objects( Ticket.all_related(), Ticket.concert.all_related(), - Ticket.concert.band_1.all_related(), - Ticket.concert.band_2.all_related(), + Ticket.concert._.band_1.all_related(), + Ticket.concert._.band_2.all_related(), ) .first() .run_sync() ) + assert ticket is not None self.assertIsInstance(ticket.concert, Concert) self.assertIsInstance(ticket.concert.band_1, Band) @@ -393,12 +404,13 @@ def test_objects_prefetch_clause(self): .prefetch( Ticket.all_related(), Ticket.concert.all_related(), - Ticket.concert.band_1.all_related(), - Ticket.concert.band_2.all_related(), + Ticket.concert._.band_1.all_related(), + Ticket.concert._.band_2.all_related(), ) .first() .run_sync() ) + assert ticket is not None self.assertIsInstance(ticket.concert, Concert) self.assertIsInstance(ticket.concert.band_1, Band) @@ -415,11 +427,12 @@ def test_objects_prefetch_intermediate(self): ticket = ( Ticket.objects() .prefetch( - Ticket.concert.band_1.manager, + Ticket.concert._.band_1._.manager, ) .first() .run_sync() ) + assert ticket is not None self.assertIsInstance(ticket.price, decimal.Decimal) self.assertIsInstance(ticket.concert, Concert) @@ -444,12 +457,13 @@ def test_objects_prefetch_multiple_intermediate(self): ticket = ( Ticket.objects() .prefetch( - Ticket.concert.band_1.manager, - Ticket.concert.band_2.manager, + Ticket.concert._.band_1._.manager, + Ticket.concert._.band_2._.manager, ) .first() .run_sync() ) + assert ticket is not None self.assertIsInstance(ticket.price, decimal.Decimal) self.assertIsInstance(ticket.concert, Concert) diff --git a/tests/table/test_join_on.py b/tests/table/test_join_on.py index 7983b218a..5be16158c 100644 --- a/tests/table/test_join_on.py +++ b/tests/table/test_join_on.py @@ -1,26 +1,28 @@ from unittest import TestCase -from piccolo.columns import Varchar +from piccolo.columns import Serial, Varchar from piccolo.table import Table class Manager(Table): + id: Serial name = Varchar(unique=True) email = Varchar(unique=True) class Band(Table): + id: Serial name = Varchar(unique=True) manager_name = Varchar() class Concert(Table): + id: Serial title = Varchar() band_name = Varchar() class TestJoinOn(TestCase): - tables = [Manager, Band, Concert] def setUp(self): diff --git a/tests/table/test_objects.py b/tests/table/test_objects.py index 853de6a13..e2db53ba9 100644 --- a/tests/table/test_objects.py +++ b/tests/table/test_objects.py @@ -66,6 +66,7 @@ def test_get(self): self.insert_row() band = Band.objects().get(Band.name == "Pythonistas").run_sync() + assert band is not None self.assertEqual(band.name, "Pythonistas") @@ -79,7 +80,8 @@ def test_get_prefetch(self): .prefetch(Band.manager) .run_sync() ) - self.assertIsInstance(band.manager, Manager) + assert band is not None + self.assertIsInstance(band.manager, Manager) # type: ignore # Just passing it straight into objects band = ( @@ -87,6 +89,7 @@ def test_get_prefetch(self): .get((Band.name == "Pythonistas")) .run_sync() ) + assert band is not None self.assertIsInstance(band.manager, Manager) @@ -97,12 +100,14 @@ def test_simple_where_clause(self): """ # When the row doesn't exist in the db: Band.objects().get_or_create( - Band.name == "Pink Floyd", defaults={"popularity": 100} + Band.name == "Pink Floyd", + defaults={"popularity": 100}, # type: ignore ).run_sync() instance = ( Band.objects().where(Band.name == "Pink Floyd").first().run_sync() ) + assert instance is not None self.assertIsInstance(instance, Band) self.assertEqual(instance.name, "Pink Floyd") @@ -116,6 +121,7 @@ def test_simple_where_clause(self): instance = ( Band.objects().where(Band.name == "Pink Floyd").first().run_sync() ) + assert instance is not None self.assertIsInstance(instance, Band) self.assertEqual(instance.name, "Pink Floyd") @@ -219,8 +225,8 @@ def test_prefetch_existing_object(self): .prefetch(Band.manager) .run_sync() ) - self.assertIsInstance(band.manager, Manager) - self.assertEqual(band.manager.name, "Guido") + self.assertIsInstance(band.manager, Manager) # type: ignore + self.assertEqual(band.manager.name, "Guido") # type: ignore # Just passing it straight into objects band = ( @@ -248,8 +254,8 @@ def test_prefetch_new_object(self): .prefetch(Band.manager) .run_sync() ) - self.assertIsInstance(band.manager, Manager) - self.assertEqual(band.name, "New Band") + self.assertIsInstance(band.manager, Manager) # type: ignore + self.assertEqual(band.name, "New Band") # type: ignore # Just passing it straight into objects band = ( diff --git a/tests/table/test_output.py b/tests/table/test_output.py index 97256dfcb..8298396fd 100644 --- a/tests/table/test_output.py +++ b/tests/table/test_output.py @@ -101,6 +101,8 @@ def test_output_nested_with_first(self): .output(nested=True) .run_sync() ) + assert response is not None self.assertDictEqual( - response, {"name": "Pythonistas", "manager": {"name": "Guido"}} + response, # type: ignore + {"name": "Pythonistas", "manager": {"name": "Guido"}}, ) diff --git a/tests/table/test_repr.py b/tests/table/test_repr.py index 59ec57ae0..37a98d37b 100644 --- a/tests/table/test_repr.py +++ b/tests/table/test_repr.py @@ -11,4 +11,5 @@ def test_repr_postgres(self): self.insert_row() manager = Manager.objects().first().run_sync() + assert manager is not None self.assertEqual(manager.__repr__(), f"") diff --git a/tests/table/test_select.py b/tests/table/test_select.py index 892972aec..a2bb86981 100644 --- a/tests/table/test_select.py +++ b/tests/table/test_select.py @@ -278,7 +278,7 @@ def test_where_bool(self): ``where(Band.has_drummer is None)``, which evaluates to a boolean. """ with self.assertRaises(ValueError): - Band.select().where(False) + Band.select().where(False) # type: ignore def test_where_is_not_null(self): self.insert_rows() @@ -680,6 +680,7 @@ def test_avg(self): self.insert_rows() response = Band.select(Avg(Band.popularity)).first().run_sync() + assert response is not None self.assertEqual(float(response["avg"]), 1003.3333333333334) @@ -691,6 +692,7 @@ def test_avg_alias(self): .first() .run_sync() ) + assert response is not None self.assertEqual(float(response["popularity_avg"]), 1003.3333333333334) @@ -702,6 +704,7 @@ def test_avg_as_alias_method(self): .first() .run_sync() ) + assert response is not None self.assertEqual(float(response["popularity_avg"]), 1003.3333333333334) @@ -714,6 +717,7 @@ def test_avg_with_where_clause(self): .first() .run_sync() ) + assert response is not None self.assertEqual(response["avg"], 1500) @@ -730,6 +734,7 @@ def test_avg_alias_with_where_clause(self): .first() .run_sync() ) + assert response is not None self.assertEqual(response["popularity_avg"], 1500) @@ -746,6 +751,7 @@ def test_avg_as_alias_method_with_where_clause(self): .first() .run_sync() ) + assert response is not None self.assertEqual(response["popularity_avg"], 1500) @@ -753,6 +759,7 @@ def test_max(self): self.insert_rows() response = Band.select(Max(Band.popularity)).first().run_sync() + assert response is not None self.assertEqual(response["max"], 2000) @@ -764,6 +771,7 @@ def test_max_alias(self): .first() .run_sync() ) + assert response is not None self.assertEqual(response["popularity_max"], 2000) @@ -775,6 +783,7 @@ def test_max_as_alias_method(self): .first() .run_sync() ) + assert response is not None self.assertEqual(response["popularity_max"], 2000) @@ -782,6 +791,7 @@ def test_min(self): self.insert_rows() response = Band.select(Min(Band.popularity)).first().run_sync() + assert response is not None self.assertEqual(response["min"], 10) @@ -793,6 +803,7 @@ def test_min_alias(self): .first() .run_sync() ) + assert response is not None self.assertEqual(response["popularity_min"], 10) @@ -804,6 +815,7 @@ def test_min_as_alias_method(self): .first() .run_sync() ) + assert response is not None self.assertEqual(response["popularity_min"], 10) @@ -811,6 +823,7 @@ def test_sum(self): self.insert_rows() response = Band.select(Sum(Band.popularity)).first().run_sync() + assert response is not None self.assertEqual(response["sum"], 3010) @@ -822,6 +835,7 @@ def test_sum_alias(self): .first() .run_sync() ) + assert response is not None self.assertEqual(response["popularity_sum"], 3010) @@ -833,6 +847,7 @@ def test_sum_as_alias_method(self): .first() .run_sync() ) + assert response is not None self.assertEqual(response["popularity_sum"], 3010) @@ -845,6 +860,7 @@ def test_sum_with_where_clause(self): .first() .run_sync() ) + assert response is not None self.assertEqual(response["sum"], 3000) @@ -861,6 +877,7 @@ def test_sum_alias_with_where_clause(self): .first() .run_sync() ) + assert response is not None self.assertEqual(response["popularity_sum"], 3000) @@ -877,6 +894,7 @@ def test_sum_as_alias_method_with_where_clause(self): .first() .run_sync() ) + assert response is not None self.assertEqual(response["popularity_sum"], 3000) @@ -888,6 +906,7 @@ def test_chain_different_functions(self): .first() .run_sync() ) + assert response is not None self.assertEqual(float(response["avg"]), 1003.3333333333334) self.assertEqual(response["sum"], 3010) @@ -903,6 +922,7 @@ def test_chain_different_functions_alias(self): .first() .run_sync() ) + assert response is not None self.assertEqual(float(response["popularity_avg"]), 1003.3333333333334) self.assertEqual(response["popularity_sum"], 3010) @@ -929,7 +949,8 @@ def test_columns(self): .first() .run_sync() ) - self.assertEqual(response, {"name": "Pythonistas"}) + assert response is not None + self.assertDictEqual(response, {"name": "Pythonistas"}) # Multiple calls to 'columns' should be additive. response = ( @@ -940,6 +961,7 @@ def test_columns(self): .first() .run_sync() ) + assert response is not None if engine_is("cockroach"): self.assertEqual( @@ -953,7 +975,9 @@ def test_call_chain(self): Make sure the call chain lengths are the correct size. """ self.assertEqual(len(Concert.band_1.name._meta.call_chain), 1) - self.assertEqual(len(Concert.band_1.manager.name._meta.call_chain), 2) + self.assertEqual( + len(Concert.band_1._.manager._.name._meta.call_chain), 2 + ) def test_as_alias(self): """ @@ -1028,6 +1052,7 @@ def test_secret(self): user.save().run_sync() user_dict = BaseUser.select(exclude_secrets=True).first().run_sync() + assert user_dict is not None self.assertNotIn("password", user_dict.keys()) @@ -1047,6 +1072,7 @@ def test_secret_parameter(self): venue.save().run_sync() venue_dict = Venue.select(exclude_secrets=True).first().run_sync() + assert venue_dict is not None if engine_is("cockroach"): self.assertTrue( venue_dict, {"id": venue_dict["id"], "name": "The Garage"} @@ -1379,7 +1405,7 @@ def test_distinct_on_error(self): raise a ValueError. """ with self.assertRaises(ValueError) as manager: - Album.select().distinct(on=Album.band) + Album.select().distinct(on=Album.band) # type: ignore self.assertEqual( manager.exception.__str__(), diff --git a/tests/table/test_update.py b/tests/table/test_update.py index 366b54fce..554774f60 100644 --- a/tests/table/test_update.py +++ b/tests/table/test_update.py @@ -553,7 +553,9 @@ def test_edge_cases(self): with self.assertRaises(ValueError): # An error should be raised because we can't save at this level # of resolution - 1 millisecond is the minimum. - MyTable.timestamp + datetime.timedelta(microseconds=1) + MyTable.timestamp + datetime.timedelta( # type: ignore + microseconds=1 + ) ############################################################################### @@ -604,12 +606,18 @@ def test_update(self): # Insert a row for us to update AutoUpdateTable.insert(AutoUpdateTable(name="test")).run_sync() - self.assertDictEqual( + data = ( AutoUpdateTable.select( AutoUpdateTable.name, AutoUpdateTable.modified_on ) .first() - .run_sync(), + .run_sync() + ) + + assert data is not None + + self.assertDictEqual( + data, {"name": "test", "modified_on": None}, ) @@ -626,6 +634,7 @@ def test_update(self): .first() .run_sync() ) + assert updated_row is not None self.assertIsInstance(updated_row["modified_on"], datetime.datetime) self.assertEqual(updated_row["name"], "test 2") diff --git a/tests/test_schema.py b/tests/test_schema.py index 29a4652db..d8ec3d481 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -21,10 +21,11 @@ def test_list_tables(self): """ Make sure we can list all the tables in a schema. """ + schema_name = Band._meta.schema + + assert schema_name is not None table_list = ( - SchemaManager() - .list_tables(schema_name=Band._meta.schema) - .run_sync() + SchemaManager().list_tables(schema_name=schema_name).run_sync() ) self.assertListEqual(table_list, [Band._meta.tablename]) @@ -49,7 +50,6 @@ def test_create_and_drop(self): @engines_skip("sqlite") class TestMoveTable(TestCase): - new_schema = "schema_2" def setUp(self): @@ -89,7 +89,6 @@ def test_move_table(self): @engines_skip("sqlite") class TestRenameSchema(TestCase): - manager = SchemaManager() schema_name = "test_schema" new_schema_name = "test_schema_2" diff --git a/tests/testing/test_model_builder.py b/tests/testing/test_model_builder.py index f56fcf956..93a079e37 100644 --- a/tests/testing/test_model_builder.py +++ b/tests/testing/test_model_builder.py @@ -93,6 +93,7 @@ def test_choices(self): queried_shirt = ( Shirt.objects().where(Shirt.id == shirt.id).first().run_sync() ) + assert queried_shirt is not None self.assertIn( queried_shirt.size, @@ -157,6 +158,7 @@ def test_valid_column(self): .first() .run_sync() ) + assert queried_manager is not None self.assertEqual(queried_manager.name, "Guido") @@ -169,6 +171,7 @@ def test_valid_column_string(self): .first() .run_sync() ) + assert queried_manager is not None self.assertEqual(queried_manager.name, "Guido") From bd5ef61a7b7cd45fc20c00b4e59e35646a009d9e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 11 Mar 2024 21:53:00 +0000 Subject: [PATCH 542/727] bumped version --- CHANGES.rst | 19 +++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index dbfae8610..7228846af 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,25 @@ Changes ======= +1.4.0 +----- + +Improved how ``create_pydantic_model`` handles ``Array`` columns: + +* Multidimensional arrays (e.g. ``Array(Array(Integer))``) have more accurate + types. +* ``Array(Email())`` now validates that each item in the list is an email + address. +* ``Array(Varchar(length=10))`` now validates that each item is the correct + length (i.e. 10 in this example). + +Other changes +~~~~~~~~~~~~~ + +Some Pylance errors were fixed in the codebase. + +------------------------------------------------------------------------------- + 1.3.2 ----- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 922bcbf79..b2c6a8de9 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.3.2" +__VERSION__ = "1.4.0" From 39959b4eb624cccdaa38dd103372c0de49cf9c40 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 12 Mar 2024 20:21:54 +0000 Subject: [PATCH 543/727] 948 Adding a self referencing foreign key to an existing table which has a custom primary key (#949) * fix migration * catch exception * add test functions * reformat with black * fix typo in comment --- .../apps/migrations/auto/migration_manager.py | 59 ++++++++++++-- .../auto/integration/test_migrations.py | 77 +++++++++++++------ 2 files changed, 106 insertions(+), 30 deletions(-) diff --git a/piccolo/apps/migrations/auto/migration_manager.py b/piccolo/apps/migrations/auto/migration_manager.py index 5c18cf89f..772ec3edc 100644 --- a/piccolo/apps/migrations/auto/migration_manager.py +++ b/piccolo/apps/migrations/auto/migration_manager.py @@ -1,6 +1,7 @@ from __future__ import annotations import inspect +import logging import typing as t from dataclasses import dataclass, field @@ -14,7 +15,7 @@ ) from piccolo.apps.migrations.auto.serialisation import deserialise_params from piccolo.columns import Column, column_types -from piccolo.columns.column_types import Serial +from piccolo.columns.column_types import ForeignKey, Serial from piccolo.engine import engine_finder from piccolo.query import Query from piccolo.query.base import DDL @@ -22,6 +23,8 @@ from piccolo.table import Table, create_table_class, sort_table_classes from piccolo.utils.warnings import colored_warning +logger = logging.getLogger(__name__) + @dataclass class AddColumnClass: @@ -793,26 +796,72 @@ async def _run_add_columns(self, backwards: bool = False): AddColumnClass ] = self.add_columns.for_table_class_name(table_class_name) + ############################################################### # Define the table, with the columns, so the metaclass # sets up the columns correctly. + + table_class_members = { + add_column.column._meta.name: add_column.column + for add_column in add_columns + } + + # There's an extreme edge case, when we're adding a foreign + # key which references its own table, for example: + # + # fk = ForeignKey('self') + # + # And that table has a custom primary key, for example: + # + # id = UUID(primary_key=True) + # + # In this situation, we need to know the primary key of the + # table in order to correctly add this new foreign key. + for add_column in add_columns: + if ( + isinstance(add_column.column, ForeignKey) + and add_column.column._meta.params.get("references") + == "self" + ): + try: + existing_table = ( + await self.get_table_from_snapshot( + table_class_name=table_class_name, + app_name=self.app_name, + offset=-1, + ) + ) + except ValueError: + logger.error( + "Unable to find primary key for the table - " + "assuming Serial." + ) + else: + primary_key = existing_table._meta.primary_key + + table_class_members[ + primary_key._meta.name + ] = primary_key + + break + _Table = create_table_class( class_name=add_columns[0].table_class_name, class_kwargs={ "tablename": add_columns[0].tablename, "schema": add_columns[0].schema, }, - class_members={ - add_column.column._meta.name: add_column.column - for add_column in add_columns - }, + class_members=table_class_members, ) + ############################################################### + for add_column in add_columns: # We fetch the column from the Table, as the metaclass # copies and sets it up properly. column = _Table._meta.get_column_by_name( add_column.column._meta.name ) + await self._run_query( _Table.alter().add_column( name=column._meta.name, column=column diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index fcd4ba5ae..406ad75df 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -128,7 +128,7 @@ def _get_app_config(self) -> AppConfig: def _test_migrations( self, table_snapshots: t.List[t.List[t.Type[Table]]], - test_function: t.Optional[t.Callable[[RowMeta], None]] = None, + test_function: t.Optional[t.Callable[[RowMeta], bool]] = None, ): """ Writes a migration file to disk and runs it. @@ -1040,47 +1040,74 @@ def test_target_column(self): @engines_only("postgres", "cockroach") -class TestTargetColumnString(MigrationTestCase): +class TestForeignKeySelf(MigrationTestCase): def setUp(self): class TableA(Table): - name = Varchar(unique=True) - - class TableB(Table): - table_a = ForeignKey(TableA, target_column="name") + id = UUID(primary_key=True) + table_a = ForeignKey("self") - self.table_classes = [TableA, TableB] + self.table_classes: t.List[t.Type[Table]] = [TableA] def tearDown(self): drop_db_tables_sync(Migration, *self.table_classes) - def test_target_column(self): + def test_create_table(self): """ - Make sure migrations still work when a foreign key references a column - other than the primary key. + Make sure migrations still work when: + + * Creating a new table with a foreign key which references itself. + * The table has a custom primary key type (e.g. UUID). + """ self._test_migrations( table_snapshots=[self.table_classes], + test_function=lambda x: x.data_type == "uuid", ) for table_class in self.table_classes: self.assertTrue(table_class.table_exists().run_sync()) - # Make sure the constraint was created correctly. - response = self.run_sync( - """ - SELECT EXISTS( - SELECT 1 - FROM INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE CCU - JOIN INFORMATION_SCHEMA.TABLE_CONSTRAINTS TC ON - CCU.CONSTRAINT_NAME = TC.CONSTRAINT_NAME - WHERE CONSTRAINT_TYPE = 'FOREIGN KEY' - AND TC.TABLE_NAME = 'table_b' - AND CCU.TABLE_NAME = 'table_a' - AND CCU.COLUMN_NAME = 'name' - ) - """ + +@engines_only("postgres", "cockroach") +class TestAddForeignKeySelf(MigrationTestCase): + def setUp(self): + pass + + def tearDown(self): + drop_db_tables_sync(create_table_class("MyTable"), Migration) + + @patch("piccolo.conf.apps.Finder.get_app_config") + def test_add_column(self, get_app_config): + """ + Make sure migrations still work when: + + * A foreign key is added to an existing table. + * The foreign key references its own table. + * The table has a custom primary key (e.g. UUID). + + """ + get_app_config.return_value = self._get_app_config() + + self._test_migrations( + table_snapshots=[ + [ + create_table_class( + class_name="MyTable", + class_members={"id": UUID(primary_key=True)}, + ) + ], + [ + create_table_class( + class_name="MyTable", + class_members={ + "id": UUID(primary_key=True), + "fk": ForeignKey("self"), + }, + ) + ], + ], + test_function=lambda x: x.data_type == "uuid", ) - self.assertTrue(response[0]["exists"]) ############################################################################### From c55f8cf861f8c3f7522be90aaf52cf49a6e4382a Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 12 Mar 2024 22:10:27 +0000 Subject: [PATCH 544/727] bumped version --- CHANGES.rst | 24 ++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 7228846af..c21a9fd4a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,30 @@ Changes ======= +1.4.1 +----- + +Fixed an edge case with auto migrations. + +If starting from a table like this, with a custom primary key column: + +.. code-block:: python + + class MyTable(Table): + id = UUID(primary_key=True) + +When a foreign key is added to the table which references itself: + +.. code-block:: python + + class MyTable(Table): + id = UUID(primary_key=True) + fk = ForeignKey("self") + +The auto migrations could fail in some situations. + +------------------------------------------------------------------------------- + 1.4.0 ----- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index b2c6a8de9..8c69db6f3 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.4.0" +__VERSION__ = "1.4.1" From 0711d3a9b7ea88d76225c8eaf7a3051e643681e2 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 13 Mar 2024 01:02:32 +0000 Subject: [PATCH 545/727] 950 `ModelBuilder` and recursive foreign keys (#951) * fix `ModelBuilder` when using recursive foreign keys * fix linter warnings --- piccolo/testing/model_builder.py | 26 +++++++++++++------ .../auto/integration/test_migrations.py | 4 +-- tests/testing/test_model_builder.py | 15 +++++++++++ 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/piccolo/testing/model_builder.py b/piccolo/testing/model_builder.py index 2994b3ec3..87e1c87fb 100644 --- a/piccolo/testing/model_builder.py +++ b/piccolo/testing/model_builder.py @@ -122,14 +122,24 @@ async def _build( continue # Column value exists if isinstance(column, ForeignKey) and persist: - reference_model = await cls._build( - column._foreign_key_meta.resolved_references, - persist=True, - ) - random_value = getattr( - reference_model, - reference_model._meta.primary_key._meta.name, - ) + # Check for recursion + if column._foreign_key_meta.references is table_class: + if column._meta.null is True: + # We can avoid this problem entirely by setting it to + # None. + random_value = None + else: + # There's no way to avoid recursion in the situation. + raise ValueError("Recursive foreign key detected") + else: + reference_model = await cls._build( + column._foreign_key_meta.resolved_references, + persist=True, + ) + random_value = getattr( + reference_model, + reference_model._meta.primary_key._meta.name, + ) else: random_value = cls._randomize_attribute(column) diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index 406ad75df..849f8e79f 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -1041,10 +1041,10 @@ def test_target_column(self): @engines_only("postgres", "cockroach") class TestForeignKeySelf(MigrationTestCase): - def setUp(self): + def setUp(self) -> None: class TableA(Table): id = UUID(primary_key=True) - table_a = ForeignKey("self") + table_a: ForeignKey[TableA] = ForeignKey("self") self.table_classes: t.List[t.Type[Table]] = [TableA] diff --git a/tests/testing/test_model_builder.py b/tests/testing/test_model_builder.py index 93a079e37..242bac188 100644 --- a/tests/testing/test_model_builder.py +++ b/tests/testing/test_model_builder.py @@ -51,6 +51,10 @@ class BandWithLazyReference(Table): ) +class BandWithRecursiveReference(Table): + manager: ForeignKey["Manager"] = ForeignKey("self") + + TABLES = ( Manager, Band, @@ -63,6 +67,7 @@ class BandWithLazyReference(Table): TableWithArrayField, TableWithDecimal, BandWithLazyReference, + BandWithRecursiveReference, ) @@ -133,6 +138,16 @@ def test_lazy_foreign_key(self): Manager.exists().where(Manager.id == model.manager).run_sync() ) + def test_recursive_foreign_key(self): + """ + Make sure no infinite loops are created with recursive foreign keys. + """ + model = ModelBuilder.build_sync( + BandWithRecursiveReference, persist=True + ) + # It should be set to None, as this foreign key is nullable. + self.assertIsNone(model.manager) + def test_invalid_column(self): with self.assertRaises(ValueError): ModelBuilder.build_sync(Band, defaults={"X": 1}) From e6dba01b96fa0ee77fcc7464428527623bbbf58c Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 13 Mar 2024 01:04:20 +0000 Subject: [PATCH 546/727] bumped version --- CHANGES.rst | 7 +++++++ piccolo/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index c21a9fd4a..c05e8337a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,13 @@ Changes ======= +1.4.2 +----- + +Improved how ``ModelBuilder`` handles recursive foreign keys. + +------------------------------------------------------------------------------- + 1.4.1 ----- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 8c69db6f3..30b4f62e7 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.4.1" +__VERSION__ = "1.4.2" From 51f58a68bde448d72abaf5441ad988d40e83eb24 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Thu, 14 Mar 2024 23:58:08 +0800 Subject: [PATCH 547/727] Update objects.rst (#952) --- docs/src/piccolo/query_types/objects.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/piccolo/query_types/objects.rst b/docs/src/piccolo/query_types/objects.rst index 0d8420222..123f35756 100644 --- a/docs/src/piccolo/query_types/objects.rst +++ b/docs/src/piccolo/query_types/objects.rst @@ -191,7 +191,7 @@ prefer. ticket = await Ticket.objects().prefetch( Ticket.concert.all_related() - ) + ).first() ------------------------------------------------------------------------------- From 1b0f9b3d2d1ada00d0dad5baba7cf3b938046758 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 15 Mar 2024 19:06:52 +0000 Subject: [PATCH 548/727] 953 Add `array_columns` to `Table._meta` (#954) * add `TableMeta.array_columns` * fix black warning --- piccolo/table.py | 20 ++++++++++++++------ tests/table/test_metaclass.py | 12 ++++++++++++ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/piccolo/table.py b/piccolo/table.py index be2296597..d6735ac38 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -78,6 +78,7 @@ class TableMeta: columns: t.List[Column] = field(default_factory=list) default_columns: t.List[Column] = field(default_factory=list) non_default_columns: t.List[Column] = field(default_factory=list) + array_columns: t.List[Array] = field(default_factory=list) email_columns: t.List[Email] = field(default_factory=list) foreign_key_columns: t.List[ForeignKey] = field(default_factory=list) primary_key: Column = field(default_factory=Column) @@ -267,6 +268,7 @@ def __init_subclass__( columns: t.List[Column] = [] default_columns: t.List[Column] = [] non_default_columns: t.List[Column] = [] + array_columns: t.List[Array] = [] foreign_key_columns: t.List[ForeignKey] = [] secret_columns: t.List[Secret] = [] json_columns: t.List[t.Union[JSON, JSONB]] = [] @@ -304,6 +306,7 @@ def __init_subclass__( if isinstance(column, Array): column._setup_base_column(table_class=cls) + array_columns.append(column) if isinstance(column, Email): email_columns.append(column) @@ -337,6 +340,7 @@ def __init_subclass__( columns=columns, default_columns=default_columns, non_default_columns=non_default_columns, + array_columns=array_columns, email_columns=email_columns, primary_key=primary_key, foreign_key_columns=foreign_key_columns, @@ -809,9 +813,11 @@ def is_unquoted(arg): # If unquoted, dump it straight into the query. query = ",".join( [ - args_dict[column._meta.name].value - if is_unquoted(args_dict[column._meta.name]) - else "{}" + ( + args_dict[column._meta.name].value + if is_unquoted(args_dict[column._meta.name]) + else "{}" + ) for column in self._meta.columns ] ) @@ -995,9 +1001,11 @@ def _process_column_args( Convert any string arguments to column instances. """ return [ - cls._meta.get_column_by_name(column) - if (isinstance(column, str)) - else column + ( + cls._meta.get_column_by_name(column) + if (isinstance(column, str)) + else column + ) for column in columns ] diff --git a/tests/table/test_metaclass.py b/tests/table/test_metaclass.py index acc0176c8..4af11d1aa 100644 --- a/tests/table/test_metaclass.py +++ b/tests/table/test_metaclass.py @@ -5,6 +5,7 @@ from piccolo.columns.column_types import ( JSON, JSONB, + Array, Email, ForeignKey, Varchar, @@ -132,6 +133,17 @@ class MyTable(Table): self.assertEqual(MyTable._meta.email_columns, [MyTable.column_a]) + def test_arry_columns(self): + """ + Make sure ``TableMeta.array_columns`` are setup correctly. + """ + + class MyTable(Table): + column_a = Array(Varchar()) + column_b = Varchar() + + self.assertEqual(MyTable._meta.array_columns, [MyTable.column_a]) + def test_id_column(self): """ Makes sure an id column is added. From 33cce0453a8b30cbd4fa2549ad89ec827e778576 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 19 Mar 2024 23:39:47 +0000 Subject: [PATCH 549/727] 955 Add a method to the `Array` column for getting the number of dimensions of the array (#956) * add `_get_dimensions ` method * fix black * fix black formatting --- piccolo/columns/column_types.py | 20 ++++++++++++++++++++ piccolo/utils/pydantic.py | 5 +++++ tests/columns/test_array.py | 11 +++++++++++ 3 files changed, 36 insertions(+) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 7012355d7..f380fed31 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -2597,6 +2597,26 @@ def _setup_base_column(self, table_class: t.Type[Table]): if isinstance(self.base_column, Array): self.base_column._setup_base_column(table_class=table_class) + def _get_dimensions(self, start: int = 0) -> int: + """ + A helper function to get the number of dimensions for the array. For + example:: + + >>> Array(Varchar())._get_dimensions() + 1 + + >>> Array(Array(Varchar()))._get_dimensions() + 2 + + :param start: + Ignore this - it's just used for calling this method recursively. + + """ + if isinstance(self.base_column, Array): + return self.base_column._get_dimensions(start=start + 1) + else: + return start + 1 + def __getitem__(self, value: int) -> Array: """ Allows queries which retrieve an item from the array. The index starts diff --git a/piccolo/utils/pydantic.py b/piccolo/utils/pydantic.py index a11b88752..a57aad297 100644 --- a/piccolo/utils/pydantic.py +++ b/piccolo/utils/pydantic.py @@ -333,6 +333,11 @@ def create_pydantic_model( elif isinstance(column, Timestamptz): extra["widget"] = "timestamptz" + # It is useful for Piccolo API and Piccolo Admin to easily know + # how many dimensions the array has. + if isinstance(column, Array): + extra["dimensions"] = column._get_dimensions() + field = pydantic.Field( json_schema_extra={"extra": extra}, **params, diff --git a/tests/columns/test_array.py b/tests/columns/test_array.py index bc3c863db..87f728a1b 100644 --- a/tests/columns/test_array.py +++ b/tests/columns/test_array.py @@ -200,3 +200,14 @@ def test_storage(self): row = NestedArrayTable.objects().first().run_sync() assert row is not None self.assertEqual(row.value, [[1, 2, 3], [4, 5, 6]]) + + +class TestGetDimensions(TestCase): + def test_get_dimensions(self): + """ + Make sure that `_get_dimensions` returns the correct value. + """ + + self.assertEqual(Array(Integer())._get_dimensions(), 1) + self.assertEqual(Array(Array(Integer()))._get_dimensions(), 2) + self.assertEqual(Array(Array(Array(Integer())))._get_dimensions(), 3) From c3d75e2c87544c2946ca379a2550b7fd81fb95de Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 21 Mar 2024 13:58:34 +0000 Subject: [PATCH 550/727] add new section header for multiple `where` clauses (#963) --- docs/src/piccolo/query_clauses/where.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/src/piccolo/query_clauses/where.rst b/docs/src/piccolo/query_clauses/where.rst index 7f2bd9ef8..472c9d796 100644 --- a/docs/src/piccolo/query_clauses/where.rst +++ b/docs/src/piccolo/query_clauses/where.rst @@ -171,6 +171,9 @@ careful to include brackets in the correct place. ((b.popularity >= 100) & (b.manager.name == 'Guido')) | (b.popularity > 1000) +Multiple ``where`` clauses +~~~~~~~~~~~~~~~~~~~~~~~~~~ + Using multiple ``where`` clauses is equivalent to an AND. .. code-block:: python From c812ef459b426e167b63f7086193b9aefda2c0a7 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 21 Mar 2024 14:07:14 +0000 Subject: [PATCH 551/727] remove banner (#965) --- docs/src/conf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/src/conf.py b/docs/src/conf.py index 05bafcd39..bae741f09 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -69,7 +69,6 @@ globaltoc_maxdepth = 3 html_theme_options = { "source_url": "https://github.com/piccolo-orm/piccolo/", - "banner_text": 'Piccolo v1 is now available! Learn more here.', # noqa: E501 } # -- Options for HTMLHelp output --------------------------------------------- From f0887ef3001750f4cde04c1fd6dd36b86bfd17a4 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 21 Mar 2024 15:49:10 +0000 Subject: [PATCH 552/727] bump black version (#967) --- .../apps/migrations/auto/migration_manager.py | 18 +- piccolo/apps/migrations/auto/serialisation.py | 6 +- piccolo/apps/migrations/commands/backwards.py | 6 +- piccolo/apps/migrations/commands/base.py | 6 +- piccolo/apps/migrations/commands/forwards.py | 6 +- piccolo/apps/playground/commands/run.py | 1 + piccolo/apps/user/tables.py | 1 + piccolo/columns/base.py | 8 +- piccolo/columns/column_types.py | 176 ++++++------------ piccolo/columns/reference.py | 1 + piccolo/conf/apps.py | 3 +- piccolo/query/methods/objects.py | 16 +- piccolo/query/methods/select.py | 12 +- piccolo/query/mixins.py | 6 +- piccolo/query/proxy.py | 5 +- piccolo/table.py | 12 +- requirements/dev-requirements.txt | 2 +- tests/columns/test_reference.py | 1 + tests/conf/example.py | 1 + 19 files changed, 115 insertions(+), 172 deletions(-) diff --git a/piccolo/apps/migrations/auto/migration_manager.py b/piccolo/apps/migrations/auto/migration_manager.py index 772ec3edc..fca36e8e7 100644 --- a/piccolo/apps/migrations/auto/migration_manager.py +++ b/piccolo/apps/migrations/auto/migration_manager.py @@ -737,9 +737,9 @@ async def _run_rename_columns(self, backwards: bool = False): async def _run_add_tables(self, backwards: bool = False): table_classes: t.List[t.Type[Table]] = [] for add_table in self.add_tables: - add_columns: t.List[ - AddColumnClass - ] = self.add_columns.for_table_class_name(add_table.class_name) + add_columns: t.List[AddColumnClass] = ( + self.add_columns.for_table_class_name(add_table.class_name) + ) _Table: t.Type[Table] = create_table_class( class_name=add_table.class_name, class_kwargs={ @@ -792,9 +792,9 @@ async def _run_add_columns(self, backwards: bool = False): if table_class_name in [i.class_name for i in self.add_tables]: continue # No need to add columns to new tables - add_columns: t.List[ - AddColumnClass - ] = self.add_columns.for_table_class_name(table_class_name) + add_columns: t.List[AddColumnClass] = ( + self.add_columns.for_table_class_name(table_class_name) + ) ############################################################### # Define the table, with the columns, so the metaclass @@ -838,9 +838,9 @@ async def _run_add_columns(self, backwards: bool = False): else: primary_key = existing_table._meta.primary_key - table_class_members[ - primary_key._meta.name - ] = primary_key + table_class_members[primary_key._meta.name] = ( + primary_key + ) break diff --git a/piccolo/apps/migrations/auto/serialisation.py b/piccolo/apps/migrations/auto/serialisation.py index d1fd5ee47..b3644b853 100644 --- a/piccolo/apps/migrations/auto/serialisation.py +++ b/piccolo/apps/migrations/auto/serialisation.py @@ -25,8 +25,7 @@ class CanConflictWithGlobalNames(abc.ABC): @abc.abstractmethod - def warn_if_is_conflicting_with_global_name(self): - ... + def warn_if_is_conflicting_with_global_name(self): ... class UniqueGlobalNamesMeta(type): @@ -237,8 +236,7 @@ def warn_if_is_conflicting_with_global_name(self): class Definition(CanConflictWithGlobalNames, abc.ABC): @abc.abstractmethod - def __repr__(self): - ... + def __repr__(self): ... ########################################################################### # To allow sorting: diff --git a/piccolo/apps/migrations/commands/backwards.py b/piccolo/apps/migrations/commands/backwards.py index a0a454d90..c84e54556 100644 --- a/piccolo/apps/migrations/commands/backwards.py +++ b/piccolo/apps/migrations/commands/backwards.py @@ -31,9 +31,9 @@ def __init__( super().__init__() async def run_migrations_backwards(self, app_config: AppConfig): - migration_modules: t.Dict[ - str, MigrationModule - ] = self.get_migration_modules(app_config.migrations_folder_path) + migration_modules: t.Dict[str, MigrationModule] = ( + self.get_migration_modules(app_config.migrations_folder_path) + ) ran_migration_ids = await Migration.get_migrations_which_ran( app_name=self.app_name diff --git a/piccolo/apps/migrations/commands/base.py b/piccolo/apps/migrations/commands/base.py index 3b4cee100..a3966f7c3 100644 --- a/piccolo/apps/migrations/commands/base.py +++ b/piccolo/apps/migrations/commands/base.py @@ -88,9 +88,9 @@ async def get_migration_managers( migrations_folder = app_config.migrations_folder_path - migration_modules: t.Dict[ - str, MigrationModule - ] = self.get_migration_modules(migrations_folder) + migration_modules: t.Dict[str, MigrationModule] = ( + self.get_migration_modules(migrations_folder) + ) migration_ids = sorted(migration_modules.keys()) diff --git a/piccolo/apps/migrations/commands/forwards.py b/piccolo/apps/migrations/commands/forwards.py index f060b493a..6d967dd5e 100644 --- a/piccolo/apps/migrations/commands/forwards.py +++ b/piccolo/apps/migrations/commands/forwards.py @@ -32,9 +32,9 @@ async def run_migrations(self, app_config: AppConfig) -> MigrationResult: app_name=app_config.app_name ) - migration_modules: t.Dict[ - str, MigrationModule - ] = self.get_migration_modules(app_config.migrations_folder_path) + migration_modules: t.Dict[str, MigrationModule] = ( + self.get_migration_modules(app_config.migrations_folder_path) + ) ids = self.get_migration_ids(migration_modules) n = len(ids) diff --git a/piccolo/apps/playground/commands/run.py b/piccolo/apps/playground/commands/run.py index 32f840d51..b4bc23f70 100644 --- a/piccolo/apps/playground/commands/run.py +++ b/piccolo/apps/playground/commands/run.py @@ -2,6 +2,7 @@ Populates a database with an example schema and data, and launches a shell for interacting with the data using Piccolo. """ + import datetime import sys import typing as t diff --git a/piccolo/apps/user/tables.py b/piccolo/apps/user/tables.py index 878f06708..a9a389101 100644 --- a/piccolo/apps/user/tables.py +++ b/piccolo/apps/user/tables.py @@ -1,6 +1,7 @@ """ A User model, used for authentication. """ + from __future__ import annotations import datetime diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index d477dc992..886a0ee48 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -887,9 +887,11 @@ def get_sql_value(self, value: t.Any) -> t.Any: return ( "'{" + ", ".join( - f'"{i}"' - if isinstance(i, str) - else str(self.get_sql_value(i)) + ( + f'"{i}"' + if isinstance(i, str) + else str(self.get_sql_value(i)) + ) for i in value ) ) + "}'" diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index f380fed31..886ad16d7 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -350,12 +350,10 @@ def __radd__(self, value: t.Union[str, Varchar, Text]) -> QueryString: # Descriptors @t.overload - def __get__(self, obj: Table, objtype=None) -> str: - ... + def __get__(self, obj: Table, objtype=None) -> str: ... @t.overload - def __get__(self, obj: None, objtype=None) -> Varchar: - ... + def __get__(self, obj: None, objtype=None) -> Varchar: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self @@ -389,12 +387,10 @@ def __init__(self, *args, **kwargs): # Descriptors @t.overload - def __get__(self, obj: Table, objtype=None) -> str: - ... + def __get__(self, obj: Table, objtype=None) -> str: ... @t.overload - def __get__(self, obj: None, objtype=None) -> Secret: - ... + def __get__(self, obj: None, objtype=None) -> Secret: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self @@ -456,12 +452,10 @@ def __radd__(self, value: t.Union[str, Varchar, Text]) -> QueryString: # Descriptors @t.overload - def __get__(self, obj: Table, objtype=None) -> str: - ... + def __get__(self, obj: Table, objtype=None) -> str: ... @t.overload - def __get__(self, obj: None, objtype=None) -> Text: - ... + def __get__(self, obj: None, objtype=None) -> Text: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self @@ -521,12 +515,10 @@ def __init__(self, default: UUIDArg = UUID4(), **kwargs) -> None: # Descriptors @t.overload - def __get__(self, obj: Table, objtype=None) -> uuid.UUID: - ... + def __get__(self, obj: Table, objtype=None) -> uuid.UUID: ... @t.overload - def __get__(self, obj: None, objtype=None) -> UUID: - ... + def __get__(self, obj: None, objtype=None) -> UUID: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self @@ -641,12 +633,10 @@ def __rfloordiv__( # Descriptors @t.overload - def __get__(self, obj: Table, objtype=None) -> int: - ... + def __get__(self, obj: Table, objtype=None) -> int: ... @t.overload - def __get__(self, obj: None, objtype=None) -> Integer: - ... + def __get__(self, obj: None, objtype=None) -> Integer: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self @@ -699,12 +689,10 @@ def column_type(self): # Descriptors @t.overload - def __get__(self, obj: Table, objtype=None) -> int: - ... + def __get__(self, obj: Table, objtype=None) -> int: ... @t.overload - def __get__(self, obj: None, objtype=None) -> BigInt: - ... + def __get__(self, obj: None, objtype=None) -> BigInt: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self @@ -749,12 +737,10 @@ def column_type(self): # Descriptors @t.overload - def __get__(self, obj: Table, objtype=None) -> int: - ... + def __get__(self, obj: Table, objtype=None) -> int: ... @t.overload - def __get__(self, obj: None, objtype=None) -> SmallInt: - ... + def __get__(self, obj: None, objtype=None) -> SmallInt: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self @@ -801,12 +787,10 @@ def default(self): # Descriptors @t.overload - def __get__(self, obj: Table, objtype=None) -> int: - ... + def __get__(self, obj: Table, objtype=None) -> int: ... @t.overload - def __get__(self, obj: None, objtype=None) -> Serial: - ... + def __get__(self, obj: None, objtype=None) -> Serial: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self @@ -835,12 +819,10 @@ def column_type(self): # Descriptors @t.overload - def __get__(self, obj: Table, objtype=None) -> int: - ... + def __get__(self, obj: Table, objtype=None) -> int: ... @t.overload - def __get__(self, obj: None, objtype=None) -> BigSerial: - ... + def __get__(self, obj: None, objtype=None) -> BigSerial: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self @@ -870,12 +852,10 @@ def __init__(self, **kwargs) -> None: # Descriptors @t.overload - def __get__(self, obj: Table, objtype=None) -> int: - ... + def __get__(self, obj: Table, objtype=None) -> int: ... @t.overload - def __get__(self, obj: None, objtype=None) -> PrimaryKey: - ... + def __get__(self, obj: None, objtype=None) -> PrimaryKey: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self @@ -960,12 +940,10 @@ def __sub__(self, value: timedelta) -> QueryString: # Descriptors @t.overload - def __get__(self, obj: Table, objtype=None) -> datetime: - ... + def __get__(self, obj: Table, objtype=None) -> datetime: ... @t.overload - def __get__(self, obj: None, objtype=None) -> Timestamp: - ... + def __get__(self, obj: None, objtype=None) -> Timestamp: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self @@ -1057,12 +1035,10 @@ def __sub__(self, value: timedelta) -> QueryString: # Descriptors @t.overload - def __get__(self, obj: Table, objtype=None) -> datetime: - ... + def __get__(self, obj: Table, objtype=None) -> datetime: ... @t.overload - def __get__(self, obj: None, objtype=None) -> Timestamptz: - ... + def __get__(self, obj: None, objtype=None) -> Timestamptz: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self @@ -1137,12 +1113,10 @@ def __sub__(self, value: timedelta) -> QueryString: # Descriptors @t.overload - def __get__(self, obj: Table, objtype=None) -> date: - ... + def __get__(self, obj: Table, objtype=None) -> date: ... @t.overload - def __get__(self, obj: None, objtype=None) -> Date: - ... + def __get__(self, obj: None, objtype=None) -> Date: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self @@ -1214,12 +1188,10 @@ def __sub__(self, value: timedelta) -> QueryString: # Descriptors @t.overload - def __get__(self, obj: Table, objtype=None) -> time: - ... + def __get__(self, obj: Table, objtype=None) -> time: ... @t.overload - def __get__(self, obj: None, objtype=None) -> Time: - ... + def __get__(self, obj: None, objtype=None) -> Time: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self @@ -1305,12 +1277,10 @@ def __sub__(self, value: timedelta) -> QueryString: # Descriptors @t.overload - def __get__(self, obj: Table, objtype=None) -> timedelta: - ... + def __get__(self, obj: Table, objtype=None) -> timedelta: ... @t.overload - def __get__(self, obj: None, objtype=None) -> Interval: - ... + def __get__(self, obj: None, objtype=None) -> Interval: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self @@ -1399,12 +1369,10 @@ def ne(self, value) -> Where: # Descriptors @t.overload - def __get__(self, obj: Table, objtype=None) -> bool: - ... + def __get__(self, obj: Table, objtype=None) -> bool: ... @t.overload - def __get__(self, obj: None, objtype=None) -> Boolean: - ... + def __get__(self, obj: None, objtype=None) -> Boolean: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self @@ -1502,12 +1470,10 @@ def __init__( # Descriptors @t.overload - def __get__(self, obj: Table, objtype=None) -> decimal.Decimal: - ... + def __get__(self, obj: Table, objtype=None) -> decimal.Decimal: ... @t.overload - def __get__(self, obj: None, objtype=None) -> Numeric: - ... + def __get__(self, obj: None, objtype=None) -> Numeric: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self @@ -1525,12 +1491,10 @@ class Decimal(Numeric): # Descriptors @t.overload - def __get__(self, obj: Table, objtype=None) -> decimal.Decimal: - ... + def __get__(self, obj: Table, objtype=None) -> decimal.Decimal: ... @t.overload - def __get__(self, obj: None, objtype=None) -> Decimal: - ... + def __get__(self, obj: None, objtype=None) -> Decimal: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self @@ -1576,12 +1540,10 @@ def __init__( # Descriptors @t.overload - def __get__(self, obj: Table, objtype=None) -> float: - ... + def __get__(self, obj: Table, objtype=None) -> float: ... @t.overload - def __get__(self, obj: None, objtype=None) -> Real: - ... + def __get__(self, obj: None, objtype=None) -> Real: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self @@ -1599,12 +1561,10 @@ class Float(Real): # Descriptors @t.overload - def __get__(self, obj: Table, objtype=None) -> float: - ... + def __get__(self, obj: Table, objtype=None) -> float: ... @t.overload - def __get__(self, obj: None, objtype=None) -> Float: - ... + def __get__(self, obj: None, objtype=None) -> Float: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self @@ -1626,12 +1586,10 @@ def column_type(self): # Descriptors @t.overload - def __get__(self, obj: Table, objtype=None) -> float: - ... + def __get__(self, obj: Table, objtype=None) -> float: ... @t.overload - def __get__(self, obj: None, objtype=None) -> DoublePrecision: - ... + def __get__(self, obj: None, objtype=None) -> DoublePrecision: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self @@ -1871,8 +1829,7 @@ def __init__( on_update: OnUpdate = OnUpdate.cascade, target_column: t.Union[str, Column, None] = None, **kwargs, - ) -> None: - ... + ) -> None: ... @t.overload def __init__( @@ -1884,8 +1841,7 @@ def __init__( on_update: OnUpdate = OnUpdate.cascade, target_column: t.Union[str, Column, None] = None, **kwargs, - ) -> None: - ... + ) -> None: ... @t.overload def __init__( @@ -1897,8 +1853,7 @@ def __init__( on_update: OnUpdate = OnUpdate.cascade, target_column: t.Union[str, Column, None] = None, **kwargs, - ) -> None: - ... + ) -> None: ... def __init__( self, @@ -2248,16 +2203,15 @@ def __getattribute__(self, name: str) -> t.Union[Column, t.Any]: # Descriptors @t.overload - def __get__(self, obj: Table, objtype=None) -> t.Any: - ... + def __get__(self, obj: Table, objtype=None) -> t.Any: ... @t.overload - def __get__(self, obj: None, objtype=None) -> ForeignKey[ReferencedTable]: - ... + def __get__( + self, obj: None, objtype=None + ) -> ForeignKey[ReferencedTable]: ... @t.overload - def __get__(self, obj: t.Any, objtype=None) -> t.Any: - ... + def __get__(self, obj: t.Any, objtype=None) -> t.Any: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self @@ -2317,12 +2271,10 @@ def column_type(self): # Descriptors @t.overload - def __get__(self, obj: Table, objtype=None) -> str: - ... + def __get__(self, obj: Table, objtype=None) -> str: ... @t.overload - def __get__(self, obj: None, objtype=None) -> JSON: - ... + def __get__(self, obj: None, objtype=None) -> JSON: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self @@ -2387,12 +2339,10 @@ def ne(self, value) -> Where: # Descriptors @t.overload - def __get__(self, obj: Table, objtype=None) -> str: - ... + def __get__(self, obj: Table, objtype=None) -> str: ... @t.overload - def __get__(self, obj: None, objtype=None) -> JSONB: - ... + def __get__(self, obj: None, objtype=None) -> JSONB: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self @@ -2460,12 +2410,10 @@ def __init__( # Descriptors @t.overload - def __get__(self, obj: Table, objtype=None) -> bytes: - ... + def __get__(self, obj: Table, objtype=None) -> bytes: ... @t.overload - def __get__(self, obj: None, objtype=None) -> Bytea: - ... + def __get__(self, obj: None, objtype=None) -> Bytea: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self @@ -2483,12 +2431,10 @@ class Blob(Bytea): # Descriptors @t.overload - def __get__(self, obj: Table, objtype=None) -> bytes: - ... + def __get__(self, obj: Table, objtype=None) -> bytes: ... @t.overload - def __get__(self, obj: None, objtype=None) -> Blob: - ... + def __get__(self, obj: None, objtype=None) -> Blob: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self @@ -2739,12 +2685,10 @@ def __add__(self, value: t.List[t.Any]) -> QueryString: # Descriptors @t.overload - def __get__(self, obj: Table, objtype=None) -> t.List[t.Any]: - ... + def __get__(self, obj: Table, objtype=None) -> t.List[t.Any]: ... @t.overload - def __get__(self, obj: None, objtype=None) -> Array: - ... + def __get__(self, obj: None, objtype=None) -> Array: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self diff --git a/piccolo/columns/reference.py b/piccolo/columns/reference.py index 841545eeb..f6edcdd56 100644 --- a/piccolo/columns/reference.py +++ b/piccolo/columns/reference.py @@ -1,6 +1,7 @@ """ Dataclasses for storing lazy references between ForeignKey columns and tables. """ + from __future__ import annotations import importlib diff --git a/piccolo/conf/apps.py b/piccolo/conf/apps.py index c311e1569..47631c478 100644 --- a/piccolo/conf/apps.py +++ b/piccolo/conf/apps.py @@ -25,8 +25,7 @@ class MigrationModule(ModuleType): @staticmethod @abstractmethod - async def forwards() -> MigrationManager: - ... + async def forwards() -> MigrationManager: ... class PiccoloAppModule(ModuleType): diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index 5b1c96002..7b8c3ad43 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -124,10 +124,10 @@ async def run( results = objects[0] if objects else None - modified_response: t.Optional[ - TableInstance - ] = await self.query.callback_delegate.invoke( - results=results, kind=CallbackType.success + modified_response: t.Optional[TableInstance] = ( + await self.query.callback_delegate.invoke( + results=results, kind=CallbackType.success + ) ) return modified_response @@ -355,10 +355,10 @@ async def run( # With callbacks, the user can return any data that they want. # Assume that most of the time they will still return a list of # Table instances. - modified: t.List[ - TableInstance - ] = await self.callback_delegate.invoke( - results, kind=CallbackType.success + modified: t.List[TableInstance] = ( + await self.callback_delegate.invoke( + results, kind=CallbackType.success + ) ) return modified else: diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index a00745e48..a2a77b155 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -604,20 +604,16 @@ def order_by( return self @t.overload - def output(self: Self, *, as_list: bool) -> SelectList: - ... + def output(self: Self, *, as_list: bool) -> SelectList: ... @t.overload - def output(self: Self, *, as_json: bool) -> SelectJSON: - ... + def output(self: Self, *, as_json: bool) -> SelectJSON: ... @t.overload - def output(self: Self, *, load_json: bool) -> Self: - ... + def output(self: Self, *, load_json: bool) -> Self: ... @t.overload - def output(self: Self, *, nested: bool) -> Self: - ... + def output(self: Self, *, nested: bool) -> Self: ... def output( self: Self, diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index 56be35a8f..8d7c6a4a9 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -639,9 +639,9 @@ class OnConflictAction(str, Enum): class OnConflictItem: target: t.Optional[t.Union[str, Column, t.Tuple[Column, ...]]] = None action: t.Optional[OnConflictAction] = None - values: t.Optional[ - t.Sequence[t.Union[Column, t.Tuple[Column, t.Any]]] - ] = None + values: t.Optional[t.Sequence[t.Union[Column, t.Tuple[Column, t.Any]]]] = ( + None + ) where: t.Optional[Combinable] = None @property diff --git a/piccolo/query/proxy.py b/piccolo/query/proxy.py index 7ded47b84..30ce06083 100644 --- a/piccolo/query/proxy.py +++ b/piccolo/query/proxy.py @@ -8,8 +8,9 @@ class Runnable(Protocol): - async def run(self, node: t.Optional[str] = None, in_pool: bool = True): - ... + async def run( + self, node: t.Optional[str] = None, in_pool: bool = True + ): ... QueryType = t.TypeVar("QueryType", bound=Runnable) diff --git a/piccolo/table.py b/piccolo/table.py index d6735ac38..b4fcbf942 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -566,12 +566,10 @@ def refresh( @t.overload def get_related( self, foreign_key: ForeignKey[ReferencedTable] - ) -> First[ReferencedTable]: - ... + ) -> First[ReferencedTable]: ... @t.overload - def get_related(self, foreign_key: str) -> First[Table]: - ... + def get_related(self, foreign_key: str) -> First[Table]: ... def get_related( self, foreign_key: t.Union[str, ForeignKey[ReferencedTable]] @@ -745,9 +743,9 @@ def to_dict(self, *columns: Column) -> t.Dict[str, t.Any]: if isinstance(value, Table): value = value.to_dict(*columns) - output[ - alias_names.get(column._meta.name) or column._meta.name - ] = value + output[alias_names.get(column._meta.name) or column._meta.name] = ( + value + ) return output def __setitem__(self, key: str, value: t.Any): diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt index 853726ee9..6fbda712d 100644 --- a/requirements/dev-requirements.txt +++ b/requirements/dev-requirements.txt @@ -1,4 +1,4 @@ -black==22.3.0 +black==24.3.0 ipdb==0.13.9 ipython>=7.31.1 flake8==6.1.0 diff --git a/tests/columns/test_reference.py b/tests/columns/test_reference.py index ea9887e00..21daa2f58 100644 --- a/tests/columns/test_reference.py +++ b/tests/columns/test_reference.py @@ -2,6 +2,7 @@ Most of the tests for piccolo/columns/reference.py are covered in piccolo/columns/test_foreignkey.py """ + from unittest import TestCase from piccolo.columns.reference import LazyTableReference diff --git a/tests/conf/example.py b/tests/conf/example.py index c7dce05a5..ef4541627 100644 --- a/tests/conf/example.py +++ b/tests/conf/example.py @@ -2,6 +2,7 @@ This file is used by test_apps.py to make sure we can exclude imported ``Table`` subclasses when using ``table_finder``. """ + from piccolo.apps.user.tables import BaseUser from piccolo.columns.column_types import ForeignKey, Varchar from piccolo.table import Table From a479b547408cf8e445b4727ca099a23501e311d7 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Thu, 21 Mar 2024 20:32:56 +0100 Subject: [PATCH 553/727] add a unique argument to the column extra (#968) --- piccolo/utils/pydantic.py | 1 + tests/utils/test_pydantic.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/piccolo/utils/pydantic.py b/piccolo/utils/pydantic.py index a57aad297..b556ace85 100644 --- a/piccolo/utils/pydantic.py +++ b/piccolo/utils/pydantic.py @@ -280,6 +280,7 @@ def create_pydantic_model( "choices": column._meta.get_choices_dict(), "secret": column._meta.secret, "nullable": column._meta.null, + "unique": column._meta.unique, } if isinstance(column, ForeignKey): diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index 2c773b8de..5447361e9 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -329,6 +329,34 @@ class Movie(Table, help_text=help_text): ) +class TestUniqueColumn(TestCase): + def test_unique_column_true(self): + class Director(Table): + name = Varchar(unique=True) + + pydantic_model = create_pydantic_model(table=Director) + + self.assertEqual( + pydantic_model.model_json_schema()["properties"]["name"]["extra"][ + "unique" + ], + True, + ) + + def test_unique_column_false(self): + class Director(Table): + name = Varchar() + + pydantic_model = create_pydantic_model(table=Director) + + self.assertEqual( + pydantic_model.model_json_schema()["properties"]["name"]["extra"][ + "unique" + ], + False, + ) + + class TestJSONColumn(TestCase): def test_default(self): class Movie(Table): From 91d62388424f77f7e6c299b9d250b40a79193167 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 21 Mar 2024 23:42:40 +0000 Subject: [PATCH 554/727] add a utility method for getting the inner type for an `Array` column (#970) --- piccolo/columns/column_types.py | 17 +++++++++++++++++ tests/columns/test_array.py | 12 +++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 886ad16d7..139622543 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -2563,6 +2563,23 @@ def _get_dimensions(self, start: int = 0) -> int: else: return start + 1 + def _get_inner_type(self) -> t.Type: + """ + A helper function to get the innermost type for the array. For + example:: + + >>> Array(Varchar())._get_inner_type() + str + + >>> Array(Array(Varchar()))._get_inner_type() + str + + """ + if isinstance(self.base_column, Array): + return self.base_column._get_inner_type() + else: + return self.base_column.value_type + def __getitem__(self, value: int) -> Array: """ Allows queries which retrieve an item from the array. The index starts diff --git a/tests/columns/test_array.py b/tests/columns/test_array.py index 87f728a1b..2243fdc03 100644 --- a/tests/columns/test_array.py +++ b/tests/columns/test_array.py @@ -207,7 +207,17 @@ def test_get_dimensions(self): """ Make sure that `_get_dimensions` returns the correct value. """ - self.assertEqual(Array(Integer())._get_dimensions(), 1) self.assertEqual(Array(Array(Integer()))._get_dimensions(), 2) self.assertEqual(Array(Array(Array(Integer())))._get_dimensions(), 3) + + +class TestGetInnerType(TestCase): + + def test_get_inner_type(self): + """ + Make sure that `_get_inner_type` returns the correct base type. + """ + self.assertEqual(Array(Integer())._get_inner_type(), int) + self.assertEqual(Array(Array(Integer()))._get_inner_type(), int) + self.assertEqual(Array(Array(Array(Integer())))._get_inner_type(), int) From e150af7243354dcbfef4f0d3978c708811aa2569 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Fri, 22 Mar 2024 16:57:42 +0100 Subject: [PATCH 555/727] update readme for Esmerald (#972) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 82bfe5702..28ba7b03d 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ Let Piccolo scaffold you an ASGI web app, using Piccolo as the ORM: piccolo asgi new ``` -[Starlette](https://www.starlette.io/), [FastAPI](https://fastapi.tiangolo.com/), [BlackSheep](https://www.neoteroi.dev/blacksheep/) and [Litestar](https://litestar.dev/) are currently supported. +[Starlette](https://www.starlette.io/), [FastAPI](https://fastapi.tiangolo.com/), [BlackSheep](https://www.neoteroi.dev/blacksheep/), [Litestar](https://litestar.dev/) and [Esmerald](https://esmerald.dev/) are currently supported. ## Are you a Django user? From ab9cbfdc8e16f99ca3916cf45b2f10d6d573c03c Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 22 Mar 2024 16:32:01 +0000 Subject: [PATCH 556/727] rename `_get_inner_type` to `_get_inner_value_type` --- piccolo/columns/column_types.py | 10 +++++----- tests/columns/test_array.py | 14 ++++++++------ 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 139622543..2afcfb741 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -2563,20 +2563,20 @@ def _get_dimensions(self, start: int = 0) -> int: else: return start + 1 - def _get_inner_type(self) -> t.Type: + def _get_inner_value_type(self) -> t.Type: """ - A helper function to get the innermost type for the array. For + A helper function to get the innermost value type for the array. For example:: - >>> Array(Varchar())._get_inner_type() + >>> Array(Varchar())._get_inner_value_type() str - >>> Array(Array(Varchar()))._get_inner_type() + >>> Array(Array(Varchar()))._get_inner_value_type() str """ if isinstance(self.base_column, Array): - return self.base_column._get_inner_type() + return self.base_column._get_inner_value_type() else: return self.base_column.value_type diff --git a/tests/columns/test_array.py b/tests/columns/test_array.py index 2243fdc03..4508e7bed 100644 --- a/tests/columns/test_array.py +++ b/tests/columns/test_array.py @@ -212,12 +212,14 @@ def test_get_dimensions(self): self.assertEqual(Array(Array(Array(Integer())))._get_dimensions(), 3) -class TestGetInnerType(TestCase): +class TestGetInnerValueType(TestCase): - def test_get_inner_type(self): + def test_get_inner_value_type(self): """ - Make sure that `_get_inner_type` returns the correct base type. + Make sure that `_get_inner_value_type` returns the correct base type. """ - self.assertEqual(Array(Integer())._get_inner_type(), int) - self.assertEqual(Array(Array(Integer()))._get_inner_type(), int) - self.assertEqual(Array(Array(Array(Integer())))._get_inner_type(), int) + self.assertEqual(Array(Integer())._get_inner_value_type(), int) + self.assertEqual(Array(Array(Integer()))._get_inner_value_type(), int) + self.assertEqual( + Array(Array(Array(Integer())))._get_inner_value_type(), int + ) From 363d683e5279d2b0284318226c290cd794ed6b6d Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 22 Mar 2024 16:57:29 +0000 Subject: [PATCH 557/727] bumped version --- CHANGES.rst | 8 ++++++++ piccolo/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index c05e8337a..2586bb8bb 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,14 @@ Changes ======= +1.5.0 +----- + +Lots of internal improvements, mostly to support new functionality in Piccolo +Admin. + +------------------------------------------------------------------------------- + 1.4.2 ----- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 30b4f62e7..7e6a9ccf8 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.4.2" +__VERSION__ = "1.5.0" From 919983ac211ebf0d5953b126f6126b57deddb4cb Mon Sep 17 00:00:00 2001 From: Tiago Silva Date: Sat, 6 Apr 2024 10:33:31 +0100 Subject: [PATCH 558/727] Add support for Lilya (#977) * Add support for Lilya * Fix event handlers * Fix linting and formating * Fix linting and formating with updated requirements * Update README.md --- README.md | 2 +- docs/src/index.rst | 2 +- docs/src/piccolo/asgi/index.rst | 5 ++- piccolo/apps/asgi/commands/new.py | 1 + .../templates/app/_lilya_app.py.jinja | 45 +++++++++++++++++++ .../asgi/commands/templates/app/app.py.jinja | 2 + .../app/home/_lilya_endpoints.py.jinja | 21 +++++++++ .../templates/app/home/endpoints.py.jinja | 2 + .../app/home/templates/home.html.jinja_raw | 5 +++ tests/columns/test_array.py | 1 - 10 files changed, 81 insertions(+), 5 deletions(-) create mode 100644 piccolo/apps/asgi/commands/templates/app/_lilya_app.py.jinja create mode 100644 piccolo/apps/asgi/commands/templates/app/home/_lilya_endpoints.py.jinja diff --git a/README.md b/README.md index 28ba7b03d..a1804d46a 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ Let Piccolo scaffold you an ASGI web app, using Piccolo as the ORM: piccolo asgi new ``` -[Starlette](https://www.starlette.io/), [FastAPI](https://fastapi.tiangolo.com/), [BlackSheep](https://www.neoteroi.dev/blacksheep/), [Litestar](https://litestar.dev/) and [Esmerald](https://esmerald.dev/) are currently supported. +[Starlette](https://www.starlette.io/), [FastAPI](https://fastapi.tiangolo.com/), [BlackSheep](https://www.neoteroi.dev/blacksheep/), [Litestar](https://litestar.dev/), [Esmerald](https://esmerald.dev/) and [Lilya](https://lilya.dev) are currently supported. ## Are you a Django user? diff --git a/docs/src/index.rst b/docs/src/index.rst index 577cb7620..ef956ce49 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -59,7 +59,7 @@ Give me an ASGI web app! piccolo asgi new -FastAPI, Starlette, BlackSheep, Litestar and Esmerald are currently supported, +FastAPI, Starlette, BlackSheep, Litestar, Esmerald and Lilya are currently supported, with more coming soon. ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/asgi/index.rst b/docs/src/piccolo/asgi/index.rst index 57f1553c4..a3d57899e 100644 --- a/docs/src/piccolo/asgi/index.rst +++ b/docs/src/piccolo/asgi/index.rst @@ -22,7 +22,8 @@ Routing frameworks Currently, `Starlette `_, `FastAPI `_, `BlackSheep `_, -`Litestar `_ and `Esmerald `_ are supported. +`Litestar `_, `Esmerald `_ and +`Lilya `_ are supported. Other great ASGI routing frameworks exist, and may be supported in the future (`Quart `_ , @@ -32,7 +33,7 @@ Other great ASGI routing frameworks exist, and may be supported in the future Which to use? ============= -All are great choices. FastAPI and Esmerald are built on top of Starlette, so they're +All are great choices. FastAPI is built on top of Starlette and Esmerald is built on top of Lilya, so they're very similar. FastAPI, BlackSheep and Esmerald are great if you want to document a REST API, as they have built-in OpenAPI support. diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index 70ecefdc8..83d343c40 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -16,6 +16,7 @@ "blacksheep": ["blacksheep"], "litestar": ["litestar"], "esmerald": ["esmerald"], + "lilya": ["lilya"], } ROUTERS = list(ROUTER_DEPENDENCIES.keys()) diff --git a/piccolo/apps/asgi/commands/templates/app/_lilya_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_lilya_app.py.jinja new file mode 100644 index 000000000..f65dd7f06 --- /dev/null +++ b/piccolo/apps/asgi/commands/templates/app/_lilya_app.py.jinja @@ -0,0 +1,45 @@ +from piccolo_admin.endpoints import create_admin +from piccolo_api.crud.endpoints import PiccoloCRUD +from piccolo.engine import engine_finder +from lilya.routing import Path, Include +from lilya.apps import Lilya +from lilya.staticfiles import StaticFiles + +from home.endpoints import HomeController +from home.piccolo_app import APP_CONFIG +from home.tables import Task + + +app = Lilya( + routes=[ + Path("/", HomeController), + Include( + "/admin/", + create_admin( + tables=APP_CONFIG.table_classes, + # Required when running under HTTPS: + # allowed_hosts=['my_site.com'] + ) + ), + Include("/static/", StaticFiles(directory="static")), + Include("/tasks/", PiccoloCRUD(table=Task)) + ], +) + + +@app.on_event("on_startup") +async def open_database_connection_pool(): + try: + engine = engine_finder() + await engine.start_connection_pool() + except Exception: + print("Unable to connect to the database") + + +@app.on_event("on_shutdown") +async def close_database_connection_pool(): + try: + engine = engine_finder() + await engine.close_connection_pool() + except Exception: + print("Unable to connect to the database") diff --git a/piccolo/apps/asgi/commands/templates/app/app.py.jinja b/piccolo/apps/asgi/commands/templates/app/app.py.jinja index 234286a3b..165510466 100644 --- a/piccolo/apps/asgi/commands/templates/app/app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/app.py.jinja @@ -8,4 +8,6 @@ {% include '_litestar_app.py.jinja' %} {% elif router == 'esmerald' %} {% include '_esmerald_app.py.jinja' %} +{% elif router == 'lilya' %} + {% include '_lilya_app.py.jinja' %} {% endif %} diff --git a/piccolo/apps/asgi/commands/templates/app/home/_lilya_endpoints.py.jinja b/piccolo/apps/asgi/commands/templates/app/home/_lilya_endpoints.py.jinja new file mode 100644 index 000000000..1b3328fed --- /dev/null +++ b/piccolo/apps/asgi/commands/templates/app/home/_lilya_endpoints.py.jinja @@ -0,0 +1,21 @@ +import os + +import jinja2 +from lilya.controllers import Controller +from lilya.responses import HTMLResponse + + +ENVIRONMENT = jinja2.Environment( + loader=jinja2.FileSystemLoader( + searchpath=os.path.join(os.path.dirname(__file__), "templates") + ) +) + + +class HomeController(Controller): + async def get(self, request): + template = ENVIRONMENT.get_template("home.html.jinja") + + content = template.render(title="Piccolo + ASGI",) + + return HTMLResponse(content) diff --git a/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja b/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja index cbce94e2f..31503063c 100644 --- a/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja @@ -6,4 +6,6 @@ {% include '_litestar_endpoints.py.jinja' %} {% elif router == 'esmerald' %} {% include '_esmerald_endpoints.py.jinja' %} +{% elif router == 'lilya' %} + {% include '_lilya_endpoints.py.jinja' %} {% endif %} diff --git a/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw b/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw index 644bc4265..5d48d767b 100644 --- a/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw +++ b/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw @@ -61,6 +61,11 @@
  • Admin
  • Swagger API
    • +

    Lilya

    + {% endblock content %} diff --git a/tests/columns/test_array.py b/tests/columns/test_array.py index 4508e7bed..6e325fbd4 100644 --- a/tests/columns/test_array.py +++ b/tests/columns/test_array.py @@ -213,7 +213,6 @@ def test_get_dimensions(self): class TestGetInnerValueType(TestCase): - def test_get_inner_value_type(self): """ Make sure that `_get_inner_value_type` returns the correct base type. From 373289e8b8a73c6df4c906722dce1142fe7ec1dd Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 21 May 2024 12:00:07 +0100 Subject: [PATCH 559/727] fix bug when user enters an empty string (#992) --- piccolo/apps/migrations/commands/backwards.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/piccolo/apps/migrations/commands/backwards.py b/piccolo/apps/migrations/commands/backwards.py index c84e54556..6627fe8af 100644 --- a/piccolo/apps/migrations/commands/backwards.py +++ b/piccolo/apps/migrations/commands/backwards.py @@ -78,9 +78,11 @@ async def run_migrations_backwards(self, app_config: AppConfig): _continue = ( "y" if self.auto_agree - else input(f"Reverse {n} migration{'s' if n != 1 else ''}? [y/N] ") + else input( + f"Reverse {n} migration{'s' if n != 1 else ''}? [y/N] " + ).lower() ) - if _continue in "yY": + if _continue == "y": for migration_id in reversed_migration_ids: migration_module = migration_modules[migration_id] response = await migration_module.forwards() @@ -131,10 +133,10 @@ async def run_backwards( "apps:\n" f"{', '.join(names)}\n" "Are you sure you want to continue? [y/N] " - ) + ).lower() ) - if _continue not in "yY": + if _continue != "y": return MigrationResult(success=False, message="user cancelled") for _app_name in sorted_app_names: print_heading(_app_name) From c29838a5a0cc942ef357dd87845fec488e8744fa Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 21 May 2024 12:13:32 +0100 Subject: [PATCH 560/727] bumped version --- CHANGES.rst | 10 ++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2586bb8bb..83d246b8f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,16 @@ Changes ======= +1.5.1 +----- + +Fixed a bug with the CLI when reversing migrations (thanks to @metakot for +reporting this). + +Updated the ASGI templates (thanks to @tarsil for adding Lilya). + +------------------------------------------------------------------------------- + 1.5.0 ----- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 7e6a9ccf8..f14dd8fd5 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.5.0" +__VERSION__ = "1.5.1" From fb6fc2106feee86a9cc763ec72dcff9b9665f8ff Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 28 May 2024 21:38:36 +0100 Subject: [PATCH 561/727] 997 Fix type warnings in `playground/commands/run.py` (#998) * use new syntax for joins * add annotations for `id` * fix ipython import --- piccolo/apps/playground/commands/run.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/piccolo/apps/playground/commands/run.py b/piccolo/apps/playground/commands/run.py index b4bc23f70..29dd98a66 100644 --- a/piccolo/apps/playground/commands/run.py +++ b/piccolo/apps/playground/commands/run.py @@ -5,7 +5,6 @@ import datetime import sys -import typing as t import uuid from decimal import Decimal from enum import Enum @@ -18,6 +17,7 @@ Integer, Interval, Numeric, + Serial, Timestamp, Varchar, ) @@ -29,6 +29,7 @@ class Manager(Table): + id: Serial name = Varchar(length=50) @classmethod @@ -40,6 +41,7 @@ def get_readable(cls) -> Readable: class Band(Table): + id: Serial name = Varchar(length=50) manager = ForeignKey(references=Manager, null=True) popularity = Integer() @@ -53,6 +55,7 @@ def get_readable(cls) -> Readable: class Venue(Table): + id: Serial name = Varchar(length=100) capacity = Integer(default=0) @@ -65,6 +68,7 @@ def get_readable(cls) -> Readable: class Concert(Table): + id: Serial band_1 = ForeignKey(Band) band_2 = ForeignKey(Band) venue = ForeignKey(Venue) @@ -89,6 +93,7 @@ class TicketType(Enum): standing = "standing" premium = "premium" + id: Serial concert = ForeignKey(Concert) price = Numeric(digits=(5, 2)) ticket_type = Varchar(choices=TicketType, default=TicketType.standing) @@ -98,13 +103,14 @@ def get_readable(cls) -> Readable: return Readable( template="%s - %s", columns=[ - t.cast(t.Type[Venue], cls.concert.venue).name, + cls.concert._.venue._.name, cls.ticket_type, ], ) class DiscountCode(Table): + id: Serial code = UUID() active = Boolean(default=True, null=True) @@ -117,6 +123,7 @@ def get_readable(cls) -> Readable: class RecordingStudio(Table): + id: Serial name = Varchar(length=100) facilities = JSON(null=True) @@ -278,7 +285,7 @@ def run( populate() - from IPython.core.interactiveshell import _asyncio_runner + from IPython.core.async_helpers import _asyncio_runner if ipython_profile: print(colored_string("Using your IPython profile\n")) From c49edcb70602b2fabbacb4816072e228492719f6 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 28 May 2024 22:17:46 +0100 Subject: [PATCH 562/727] 999 Add `Album` table to playground (#1000) * add album table * load data * add `id` --- piccolo/apps/playground/commands/run.py | 75 ++++++++++++++++++++----- 1 file changed, 60 insertions(+), 15 deletions(-) diff --git a/piccolo/apps/playground/commands/run.py b/piccolo/apps/playground/commands/run.py index 29dd98a66..e1bc00130 100644 --- a/piccolo/apps/playground/commands/run.py +++ b/piccolo/apps/playground/commands/run.py @@ -13,6 +13,7 @@ JSON, UUID, Boolean, + Date, ForeignKey, Integer, Interval, @@ -135,7 +136,31 @@ def get_readable(cls) -> Readable: ) -TABLES = (Manager, Band, Venue, Concert, Ticket, DiscountCode, RecordingStudio) +class Album(Table): + id: Serial + name = Varchar() + band = ForeignKey(Band) + release_date = Date() + recorded_at = ForeignKey(RecordingStudio) + + @classmethod + def get_readable(cls) -> Readable: + return Readable( + template="%s - %s", + columns=[cls.name, cls.band._.name], + ) + + +TABLES = ( + Manager, + Band, + Venue, + Concert, + Ticket, + DiscountCode, + RecordingStudio, + Album, +) def populate(): @@ -191,24 +216,44 @@ def populate(): *[DiscountCode({DiscountCode.code: uuid.uuid4()}) for _ in range(5)] ).run_sync() - RecordingStudio.insert( - RecordingStudio( + recording_studio_1 = RecordingStudio( + { + RecordingStudio.name: "Abbey Road", + RecordingStudio.facilities: { + "restaurant": True, + "mixing_desk": True, + }, + } + ) + recording_studio_1.save().run_sync() + + recording_studio_2 = RecordingStudio( + { + RecordingStudio.name: "Electric Lady", + RecordingStudio.facilities: { + "restaurant": False, + "mixing_desk": True, + }, + }, + ) + recording_studio_2.save().run_sync() + + Album.insert( + Album( { - RecordingStudio.name: "Abbey Road", - RecordingStudio.facilities: { - "restaurant": True, - "mixing_desk": True, - }, + Album.name: "Awesome album 1", + Album.recorded_at: recording_studio_1, + Album.band: pythonistas, + Album.release_date: datetime.date(year=2021, month=1, day=1), } ), - RecordingStudio( + Album( { - RecordingStudio.name: "Electric Lady", - RecordingStudio.facilities: { - "restaurant": False, - "mixing_desk": True, - }, - }, + Album.name: "Awesome album 2", + Album.recorded_at: recording_studio_2, + Album.band: rustaceans, + Album.release_date: datetime.date(year=2022, month=2, day=2), + } ), ).run_sync() From 9aa6e46ae3979ee77f4b79ccf37febc517c0acaa Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 28 May 2024 23:49:47 +0100 Subject: [PATCH 563/727] 1001 Fix `load_json` on joined tables (#1002) * fix `load_json` on joined tables * add tests * reduce repetition in tests * update tests * add migration * format migration file * update more tests --- piccolo/query/base.py | 6 +- .../commands/test_forwards_backwards.py | 3 + tests/apps/shell/commands/test_run.py | 1 + tests/conf/test_apps.py | 6 + tests/conftest.py | 1 + .../music_2024_05_28t23_15_41_018844.py | 84 ++++++++++++ tests/example_apps/music/tables.py | 10 ++ tests/table/test_output.py | 124 +++++++++++++----- 8 files changed, 194 insertions(+), 41 deletions(-) create mode 100644 tests/example_apps/music/piccolo_migrations/music_2024_05_28t23_15_41_018844.py diff --git a/piccolo/query/base.py b/piccolo/query/base.py index b10d42ee0..c169fde0e 100644 --- a/piccolo/query/base.py +++ b/piccolo/query/base.py @@ -87,13 +87,9 @@ async def _process_results(self, results) -> QueryResponseType: for column in json_columns: if column._alias is not None: json_column_names.append(column._alias) - elif column.json_operator is not None: - json_column_names.append(column._meta.name) elif len(column._meta.call_chain) > 0: json_column_names.append( - column.get_select_string( - engine_type=column._meta.engine_type - ) + column._meta.get_default_alias().replace("$", ".") ) else: json_column_names.append(column._meta.name) diff --git a/tests/apps/migrations/commands/test_forwards_backwards.py b/tests/apps/migrations/commands/test_forwards_backwards.py index d70d7d506..c3c2c6592 100644 --- a/tests/apps/migrations/commands/test_forwards_backwards.py +++ b/tests/apps/migrations/commands/test_forwards_backwards.py @@ -13,6 +13,7 @@ from tests.example_apps.music.tables import ( Band, Concert, + Instrument, Manager, Poster, RecordingStudio, @@ -33,6 +34,7 @@ Poster, Shirt, RecordingStudio, + Instrument, ] @@ -211,6 +213,7 @@ def test_forwards_fake(self): "2021-07-25T22:38:48:009306", "2021-09-06T13:58:23:024723", "2021-11-13T14:01:46:114725", + "2024-05-28T23:15:41:018844", ], ) diff --git a/tests/apps/shell/commands/test_run.py b/tests/apps/shell/commands/test_run.py index 8594f7101..dd5122b41 100644 --- a/tests/apps/shell/commands/test_run.py +++ b/tests/apps/shell/commands/test_run.py @@ -20,6 +20,7 @@ def test_run(self, print_: MagicMock, start_ipython_shell: MagicMock): call("Importing music tables:"), call("- Band"), call("- Concert"), + call("- Instrument"), call("- Manager"), call("- Poster"), call("- RecordingStudio"), diff --git a/tests/conf/test_apps.py b/tests/conf/test_apps.py index 907064e1f..44b2d4a4a 100644 --- a/tests/conf/test_apps.py +++ b/tests/conf/test_apps.py @@ -9,6 +9,7 @@ from tests.example_apps.music.tables import ( Band, Concert, + Instrument, Manager, Poster, RecordingStudio, @@ -113,6 +114,7 @@ def test_table_finder(self): [ "Band", "Concert", + "Instrument", "Manager", "Poster", "RecordingStudio", @@ -139,6 +141,7 @@ def test_table_finder_coercion(self): [ "Band", "Concert", + "Instrument", "Manager", "Poster", "RecordingStudio", @@ -182,6 +185,7 @@ def test_exclude_tags(self): [ "Band", "Concert", + "Instrument", "Manager", "RecordingStudio", "Shirt", @@ -228,6 +232,7 @@ def test_get_table_classes(self): [ Band, Concert, + Instrument, Manager, MegaTable, Poster, @@ -247,6 +252,7 @@ def test_get_table_classes(self): [ Band, Concert, + Instrument, Manager, Poster, RecordingStudio, diff --git a/tests/conftest.py b/tests/conftest.py index 457c4dae7..8411ebc38 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,6 +17,7 @@ async def drop_tables(): "musician", "my_table", "recording_studio", + "instrument", "shirt", "instrument", "mega_table", diff --git a/tests/example_apps/music/piccolo_migrations/music_2024_05_28t23_15_41_018844.py b/tests/example_apps/music/piccolo_migrations/music_2024_05_28t23_15_41_018844.py new file mode 100644 index 000000000..a1428eead --- /dev/null +++ b/tests/example_apps/music/piccolo_migrations/music_2024_05_28t23_15_41_018844.py @@ -0,0 +1,84 @@ +from piccolo.apps.migrations.auto.migration_manager import MigrationManager +from piccolo.columns.base import OnDelete, OnUpdate +from piccolo.columns.column_types import ForeignKey, Serial, Varchar +from piccolo.columns.indexes import IndexMethod +from piccolo.table import Table + + +class RecordingStudio(Table, tablename="recording_studio", schema=None): + id = Serial( + null=False, + primary_key=True, + unique=False, + index=False, + index_method=IndexMethod.btree, + choices=None, + db_column_name="id", + secret=False, + ) + + +ID = "2024-05-28T23:15:41:018844" +VERSION = "1.5.1" +DESCRIPTION = "" + + +async def forwards(): + manager = MigrationManager( + migration_id=ID, app_name="music", description=DESCRIPTION + ) + + manager.add_table( + class_name="Instrument", + tablename="instrument", + schema=None, + columns=None, + ) + + manager.add_column( + table_class_name="Instrument", + tablename="instrument", + column_name="name", + db_column_name="name", + column_class_name="Varchar", + column_class=Varchar, + params={ + "length": 255, + "default": "", + "null": False, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + "db_column_name": None, + "secret": False, + }, + schema=None, + ) + + manager.add_column( + table_class_name="Instrument", + tablename="instrument", + column_name="recording_studio", + db_column_name="recording_studio", + column_class_name="ForeignKey", + column_class=ForeignKey, + params={ + "references": RecordingStudio, + "on_delete": OnDelete.cascade, + "on_update": OnUpdate.cascade, + "target_column": None, + "null": True, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + "db_column_name": None, + "secret": False, + }, + schema=None, + ) + + return manager diff --git a/tests/example_apps/music/tables.py b/tests/example_apps/music/tables.py index dff416c2f..9e0cdbb39 100644 --- a/tests/example_apps/music/tables.py +++ b/tests/example_apps/music/tables.py @@ -115,3 +115,13 @@ class RecordingStudio(Table): id: Serial facilities = JSON() facilities_b = JSONB() + + +class Instrument(Table): + """ + Used for testing foreign keys to a table with a JSON column. + """ + + id: Serial + name = Varchar() + recording_studio = ForeignKey(RecordingStudio) diff --git a/tests/table/test_output.py b/tests/table/test_output.py index 8298396fd..ecfc997bc 100644 --- a/tests/table/test_output.py +++ b/tests/table/test_output.py @@ -1,8 +1,9 @@ import json from unittest import TestCase -from tests.base import DBTestCase, engine_is -from tests.example_apps.music.tables import Band, RecordingStudio +from piccolo.table import create_db_tables_sync, drop_db_tables_sync +from tests.base import DBTestCase +from tests.example_apps.music.tables import Band, Instrument, RecordingStudio class TestOutputList(DBTestCase): @@ -32,51 +33,102 @@ def test_output_as_json(self): class TestOutputLoadJSON(TestCase): + tables = [RecordingStudio, Instrument] + json = {"a": 123} + def setUp(self): - RecordingStudio.create_table().run_sync() + create_db_tables_sync(*self.tables) + + recording_studio = RecordingStudio( + { + RecordingStudio.facilities: self.json, + RecordingStudio.facilities_b: self.json, + } + ) + recording_studio.save().run_sync() + + instrument = Instrument( + { + Instrument.recording_studio: recording_studio, + Instrument.name: "Piccolo", + } + ) + instrument.save().run_sync() def tearDown(self): - RecordingStudio.alter().drop_table().run_sync() + drop_db_tables_sync(*self.tables) def test_select(self): - json = {"a": 123} - - RecordingStudio(facilities=json, facilities_b=json).save().run_sync() - - results = RecordingStudio.select().output(load_json=True).run_sync() - - if engine_is("cockroach"): - self.assertEqual( - results, - [ - { - "id": results[0]["id"], - "facilities": {"a": 123}, - "facilities_b": {"a": 123}, - } - ], + results = ( + RecordingStudio.select( + RecordingStudio.facilities, RecordingStudio.facilities_b ) - else: - self.assertEqual( - results, - [ - { - "id": 1, - "facilities": {"a": 123}, - "facilities_b": {"a": 123}, - } - ], + .output(load_json=True) + .run_sync() + ) + + self.assertEqual( + results, + [ + { + "facilities": self.json, + "facilities_b": self.json, + } + ], + ) + + def test_join(self): + """ + Make sure it works correctly when the JSON column is on a joined table. + + https://github.com/piccolo-orm/piccolo/issues/1001 + + """ + results = ( + Instrument.select( + Instrument.name, + Instrument.recording_studio._.facilities, ) + .output(load_json=True) + .run_sync() + ) - def test_objects(self): - json = {"a": 123} + self.assertEqual( + results, + [ + { + "name": "Piccolo", + "recording_studio.facilities": self.json, + } + ], + ) - RecordingStudio(facilities=json, facilities_b=json).save().run_sync() + def test_join_with_alias(self): + results = ( + Instrument.select( + Instrument.name, + Instrument.recording_studio._.facilities.as_alias( + "facilities" + ), + ) + .output(load_json=True) + .run_sync() + ) - results = RecordingStudio.objects().output(load_json=True).run_sync() + self.assertEqual( + results, + [ + { + "name": "Piccolo", + "facilities": self.json, + } + ], + ) - self.assertEqual(results[0].facilities, json) - self.assertEqual(results[0].facilities_b, json) + def test_objects(self): + results = RecordingStudio.objects().output(load_json=True).run_sync() + self.assertEqual(results[0].facilities, self.json) + self.assertEqual(results[0].facilities_b, self.json) class TestOutputNested(DBTestCase): From da92fbe0f40584c5a9bee4d3c9e6c199b247d99a Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 28 May 2024 23:55:46 +0100 Subject: [PATCH 564/727] bumped version --- CHANGES.rst | 11 +++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 83d246b8f..e1fb2c900 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,17 @@ Changes ======= +1.5.2 +----- + +Added an ``Album`` table to the playground, along with some other +improvements. + +Fixed a bug with the ``output(load_json=True)`` clause, when used on joined +tables. + +------------------------------------------------------------------------------- + 1.5.1 ----- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index f14dd8fd5..2c74cfe9e 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.5.1" +__VERSION__ = "1.5.2" From a85b404bd7ae1bcec75be3168a78324c45aff34c Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 30 May 2024 00:57:27 +0100 Subject: [PATCH 565/727] 994 Add support for more functions (upper, lower etc) (#996) * support upper and lower * remove prototype code * rename `operator` to `outer_function` * remove check for `call_chain` * allow multiple outer_functions to be passed in * update docstring * use querystrings instead * fix some linter issues * fix linter errors * add album table * fix `load_json` on joined tables * move logic for `get_select_string` from `Function` to `QueryString` * use columns in querystring for joins * add `not_in` to `querystring` * add `get_where_string` * set `_alias` in querystring __init__ * refactor `table_alias` * move functions into a new folder * re-export `Upper` and `Lower` * add ltrim and rtrim functions * add more functions * improve error message * add default value for `getattr` when fetching querystring columns * add initial tests * add a test for alias * deprecate `Unquoted` - `QueryString` can be used directly * simplify alias handling for `Function` * don't get alias from child `QueryString` * add `Reverse` function * add `TestNested` * fix sqlite tests * improve tracking of columns within querystrings * increase test timeouts * add missing imports * improve functions nested within `QueryString` * refactor aggregate functions to use new format * make sure where clauses work with functions * fix linter errors * update docs --- .github/workflows/tests.yaml | 10 +- docs/src/piccolo/query_clauses/group_by.rst | 4 +- docs/src/piccolo/query_types/count.rst | 2 +- docs/src/piccolo/query_types/select.rst | 37 ++- piccolo/columns/base.py | 71 +++--- piccolo/columns/column_types.py | 19 +- piccolo/columns/m2m.py | 22 +- piccolo/columns/readable.py | 16 +- piccolo/query/__init__.py | 5 +- piccolo/query/functions/__init__.py | 16 ++ piccolo/query/functions/aggregate.py | 179 ++++++++++++++ piccolo/query/functions/base.py | 21 ++ piccolo/query/functions/string.py | 73 ++++++ piccolo/query/methods/__init__.py | 19 +- piccolo/query/methods/count.py | 6 +- piccolo/query/methods/delete.py | 2 +- piccolo/query/methods/exists.py | 2 +- piccolo/query/methods/objects.py | 2 +- piccolo/query/methods/select.py | 249 ++------------------ piccolo/query/methods/update.py | 2 +- piccolo/query/mixins.py | 11 +- piccolo/querystring.py | 114 ++++++++- piccolo/table.py | 32 +-- tests/query/test_functions.py | 102 ++++++++ tests/table/test_select.py | 11 +- 25 files changed, 660 insertions(+), 367 deletions(-) create mode 100644 piccolo/query/functions/__init__.py create mode 100644 piccolo/query/functions/aggregate.py create mode 100644 piccolo/query/functions/base.py create mode 100644 piccolo/query/functions/string.py create mode 100644 tests/query/test_functions.py diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index d71d78e7e..136f86680 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -13,7 +13,7 @@ on: jobs: linters: runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 60 strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] @@ -35,7 +35,7 @@ jobs: integration: runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 60 strategy: matrix: # These tests are slow, so we only run on the latest Python @@ -82,7 +82,7 @@ jobs: postgres: runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 60 strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] @@ -138,7 +138,7 @@ jobs: cockroach: runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 60 strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] @@ -172,7 +172,7 @@ jobs: sqlite: runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 60 strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] diff --git a/docs/src/piccolo/query_clauses/group_by.rst b/docs/src/piccolo/query_clauses/group_by.rst index d3eb5af87..516a4ac00 100644 --- a/docs/src/piccolo/query_clauses/group_by.rst +++ b/docs/src/piccolo/query_clauses/group_by.rst @@ -19,13 +19,13 @@ In the following query, we get a count of the number of bands per manager: .. code-block:: python - >>> from piccolo.query.methods.select import Count + >>> from piccolo.query.functions.aggregate import Count >>> await Band.select( ... Band.manager.name.as_alias('manager_name'), ... Count(alias='band_count') ... ).group_by( - ... Band.manager + ... Band.manager.name ... ) [ diff --git a/docs/src/piccolo/query_types/count.rst b/docs/src/piccolo/query_types/count.rst index 033667c18..794d125bc 100644 --- a/docs/src/piccolo/query_types/count.rst +++ b/docs/src/piccolo/query_types/count.rst @@ -15,7 +15,7 @@ It's equivalent to this ``select`` query: .. code-block:: python - from piccolo.query.methods.select import Count + from piccolo.query.functions.aggregate import Count >>> response = await Band.select(Count()) >>> response[0]['count'] diff --git a/docs/src/piccolo/query_types/select.rst b/docs/src/piccolo/query_types/select.rst index 1591e3580..092291e4c 100644 --- a/docs/src/piccolo/query_types/select.rst +++ b/docs/src/piccolo/query_types/select.rst @@ -165,6 +165,31 @@ convenient. ------------------------------------------------------------------------------- +String functions +---------------- + +Piccolo has lots of string functions built-in. See +``piccolo/query/functions/string.py``. Here's an example using ``Upper``, to +convert values to uppercase: + +.. code-block:: python + + from piccolo.query.functions.string import Upper + + >> await Band.select(Upper(Band.name, alias='name')) + [{'name': 'PYTHONISTAS'}, ...] + +You can also use these within where clauses: + +.. code-block:: python + + from piccolo.query.functions.string import Upper + + >> await Band.select(Band.name).where(Upper(Band.manager.name) == 'GUIDO') + [{'name': 'Pythonistas'}] + +------------------------------------------------------------------------------- + .. _AggregateFunctions: Aggregate functions @@ -182,7 +207,7 @@ Returns the number of matching rows. .. code-block:: python - from piccolo.query.methods.select import Count + from piccolo.query.functions.aggregate import Count >> await Band.select(Count()).where(Band.popularity > 100) [{'count': 3}] @@ -196,7 +221,7 @@ Returns the average for a given column: .. code-block:: python - >>> from piccolo.query import Avg + >>> from piccolo.query.functions.aggregate import Avg >>> response = await Band.select(Avg(Band.popularity)).first() >>> response["avg"] 750.0 @@ -208,7 +233,7 @@ Returns the sum for a given column: .. code-block:: python - >>> from piccolo.query import Sum + >>> from piccolo.query.functions.aggregate import Sum >>> response = await Band.select(Sum(Band.popularity)).first() >>> response["sum"] 1500 @@ -220,7 +245,7 @@ Returns the maximum for a given column: .. code-block:: python - >>> from piccolo.query import Max + >>> from piccolo.query.functions.aggregate import Max >>> response = await Band.select(Max(Band.popularity)).first() >>> response["max"] 1000 @@ -232,7 +257,7 @@ Returns the minimum for a given column: .. code-block:: python - >>> from piccolo.query import Min + >>> from piccolo.query.functions.aggregate import Min >>> response = await Band.select(Min(Band.popularity)).first() >>> response["min"] 500 @@ -244,7 +269,7 @@ You also can have multiple different aggregate functions in one query: .. code-block:: python - >>> from piccolo.query import Avg, Sum + >>> from piccolo.query.functions.aggregate import Avg, Sum >>> response = await Band.select( ... Avg(Band.popularity), ... Sum(Band.popularity) diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 886a0ee48..ce452260c 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -6,7 +6,6 @@ import inspect import typing as t import uuid -from abc import ABCMeta, abstractmethod from dataclasses import dataclass, field, fields from enum import Enum @@ -32,6 +31,7 @@ NotLike, ) from piccolo.columns.reference import LazyTableReference +from piccolo.querystring import QueryString, Selectable from piccolo.utils.warnings import colored_warning if t.TYPE_CHECKING: # pragma: no cover @@ -205,7 +205,6 @@ def table(self) -> t.Type[Table]: # Used by Foreign Keys: call_chain: t.List["ForeignKey"] = field(default_factory=list) - table_alias: t.Optional[str] = None ########################################################################### @@ -260,7 +259,7 @@ def _get_path(self, include_quotes: bool = False): column_name = self.db_column_name if self.call_chain: - table_alias = self.call_chain[-1]._meta.table_alias + table_alias = self.call_chain[-1].table_alias if include_quotes: return f'"{table_alias}"."{column_name}"' else: @@ -272,7 +271,9 @@ def _get_path(self, include_quotes: bool = False): return f"{self.table._meta.tablename}.{column_name}" def get_full_name( - self, with_alias: bool = True, include_quotes: bool = True + self, + with_alias: bool = True, + include_quotes: bool = True, ) -> str: """ Returns the full column name, taking into account joins. @@ -302,11 +303,10 @@ def get_full_name( >>> column._meta.get_full_name(include_quotes=False) 'my_table_name.my_column_name' - """ full_name = self._get_path(include_quotes=include_quotes) - if with_alias and self.call_chain: + if with_alias: alias = self.get_default_alias() if include_quotes: full_name += f' AS "{alias}"' @@ -346,32 +346,6 @@ def __deepcopy__(self, memo) -> ColumnMeta: return self.copy() -class Selectable(metaclass=ABCMeta): - """ - Anything which inherits from this can be used in a select query. - """ - - _alias: t.Optional[str] - - @abstractmethod - def get_select_string( - self, engine_type: str, with_alias: bool = True - ) -> str: - """ - In a query, what to output after the select statement - could be a - column name, a sub query, a function etc. For a column it will be the - column name. - """ - raise NotImplementedError() - - def as_alias(self, alias: str) -> Selectable: - """ - Allows column names to be changed in the result of a select. - """ - self._alias = alias - return self - - class Column(Selectable): """ All other columns inherit from ``Column``. Don't use it directly. @@ -822,25 +796,32 @@ def get_default_value(self) -> t.Any: def get_select_string( self, engine_type: str, with_alias: bool = True - ) -> str: + ) -> QueryString: """ How to refer to this column in a SQL query, taking account of any joins and aliases. """ + if with_alias: if self._alias: original_name = self._meta.get_full_name( with_alias=False, ) - return f'{original_name} AS "{self._alias}"' + return QueryString(f'{original_name} AS "{self._alias}"') else: - return self._meta.get_full_name( - with_alias=True, + return QueryString( + self._meta.get_full_name( + with_alias=True, + ) ) - return self._meta.get_full_name(with_alias=False) + return QueryString( + self._meta.get_full_name( + with_alias=False, + ) + ) - def get_where_string(self, engine_type: str) -> str: + def get_where_string(self, engine_type: str) -> QueryString: return self.get_select_string( engine_type=engine_type, with_alias=False ) @@ -902,6 +883,13 @@ def get_sql_value(self, value: t.Any) -> t.Any: def column_type(self): return self.__class__.__name__.upper() + @property + def table_alias(self) -> str: + return "$".join( + f"{_key._meta.table._meta.tablename}${_key._meta.name}" + for _key in [*self._meta.call_chain, self] + ) + @property def ddl(self) -> str: """ @@ -945,8 +933,8 @@ def ddl(self) -> str: return query - def copy(self) -> Column: - column: Column = copy.copy(self) + def copy(self: Self) -> Self: + column = copy.copy(self) column._meta = self._meta.copy() return column @@ -971,3 +959,6 @@ def __repr__(self): f"{table_class_name}.{self._meta.name} - " f"{self.__class__.__name__}" ) + + +Self = t.TypeVar("Self", bound=Column) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 2afcfb741..add0c6f5c 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -60,7 +60,7 @@ class Band(Table): from piccolo.columns.operators.comparison import ArrayAll, ArrayAny from piccolo.columns.operators.string import Concat from piccolo.columns.reference import LazyTableReference -from piccolo.querystring import QueryString, Unquoted +from piccolo.querystring import QueryString from piccolo.utils.encoding import dump_json from piccolo.utils.warnings import colored_warning @@ -752,8 +752,8 @@ def __set__(self, obj, value: t.Union[int, None]): ############################################################################### -DEFAULT = Unquoted("DEFAULT") -NULL = Unquoted("null") +DEFAULT = QueryString("DEFAULT") +NULL = QueryString("null") class Serial(Column): @@ -778,7 +778,7 @@ def default(self): if engine_type == "postgres": return DEFAULT elif engine_type == "cockroach": - return Unquoted("unique_rowid()") + return QueryString("unique_rowid()") elif engine_type == "sqlite": return NULL raise Exception("Unrecognized engine type") @@ -2194,6 +2194,7 @@ def __getattribute__(self, name: str) -> t.Union[Column, t.Any]: column_meta: ColumnMeta = object.__getattribute__(self, "_meta") new_column._meta.call_chain = column_meta.call_chain.copy() + new_column._meta.call_chain.append(self) return new_column else: @@ -2311,7 +2312,7 @@ def arrow(self, key: str) -> JSONB: def get_select_string( self, engine_type: str, with_alias: bool = True - ) -> str: + ) -> QueryString: select_string = self._meta.get_full_name(with_alias=False) if self.json_operator is not None: @@ -2321,7 +2322,7 @@ def get_select_string( alias = self._alias or self._meta.get_default_alias() select_string += f' AS "{alias}"' - return select_string + return QueryString(select_string) def eq(self, value) -> Where: """ @@ -2616,7 +2617,9 @@ def __getitem__(self, value: int) -> Array: else: raise ValueError("Only integers can be used for indexing.") - def get_select_string(self, engine_type: str, with_alias=True) -> str: + def get_select_string( + self, engine_type: str, with_alias=True + ) -> QueryString: select_string = self._meta.get_full_name(with_alias=False) if isinstance(self.index, int): @@ -2626,7 +2629,7 @@ def get_select_string(self, engine_type: str, with_alias=True) -> str: alias = self._alias or self._meta.get_default_alias() select_string += f' AS "{alias}"' - return select_string + return QueryString(select_string) def any(self, value: t.Any) -> Where: """ diff --git a/piccolo/columns/m2m.py b/piccolo/columns/m2m.py index 0eefd22e7..90469fc1f 100644 --- a/piccolo/columns/m2m.py +++ b/piccolo/columns/m2m.py @@ -4,7 +4,6 @@ import typing as t from dataclasses import dataclass -from piccolo.columns.base import Selectable from piccolo.columns.column_types import ( JSON, JSONB, @@ -12,6 +11,7 @@ ForeignKey, LazyTableReference, ) +from piccolo.querystring import QueryString, Selectable from piccolo.utils.list import flatten from piccolo.utils.sync import run_sync @@ -56,7 +56,9 @@ def __init__( for column in columns ) - def get_select_string(self, engine_type: str, with_alias=True) -> str: + def get_select_string( + self, engine_type: str, with_alias=True + ) -> QueryString: m2m_table_name_with_schema = ( self.m2m._meta.resolved_joining_table._meta.get_formatted_tablename() # noqa: E501 ) # noqa: E501 @@ -90,28 +92,33 @@ def get_select_string(self, engine_type: str, with_alias=True) -> str: if engine_type in ("postgres", "cockroach"): if self.as_list: column_name = self.columns[0]._meta.db_column_name - return f""" + return QueryString( + f""" ARRAY( SELECT "inner_{table_2_name}"."{column_name}" FROM {inner_select} ) AS "{m2m_relationship_name}" """ + ) elif not self.serialisation_safe: column_name = table_2_pk_name - return f""" + return QueryString( + f""" ARRAY( SELECT "inner_{table_2_name}"."{column_name}" FROM {inner_select} ) AS "{m2m_relationship_name}" """ + ) else: column_names = ", ".join( f'"inner_{table_2_name}"."{column._meta.db_column_name}"' for column in self.columns ) - return f""" + return QueryString( + f""" ( SELECT JSON_AGG({m2m_relationship_name}_results) FROM ( @@ -119,13 +126,15 @@ def get_select_string(self, engine_type: str, with_alias=True) -> str: ) AS "{m2m_relationship_name}_results" ) AS "{m2m_relationship_name}" """ + ) elif engine_type == "sqlite": if len(self.columns) > 1 or not self.serialisation_safe: column_name = table_2_pk_name else: column_name = self.columns[0]._meta.db_column_name - return f""" + return QueryString( + f""" ( SELECT group_concat( "inner_{table_2_name}"."{column_name}" @@ -134,6 +143,7 @@ def get_select_string(self, engine_type: str, with_alias=True) -> str: ) AS "{m2m_relationship_name} [M2M]" """ + ) else: raise ValueError(f"{engine_type} is an unrecognised engine type") diff --git a/piccolo/columns/readable.py b/piccolo/columns/readable.py index 2748648d8..ebd32bf51 100644 --- a/piccolo/columns/readable.py +++ b/piccolo/columns/readable.py @@ -3,7 +3,7 @@ import typing as t from dataclasses import dataclass -from piccolo.columns.base import Selectable +from piccolo.querystring import QueryString, Selectable if t.TYPE_CHECKING: # pragma: no cover from piccolo.columns.base import Column @@ -27,25 +27,27 @@ def _columns_string(self) -> str: i._meta.get_full_name(with_alias=False) for i in self.columns ) - def _get_string(self, operator: str) -> str: - return ( + def _get_string(self, operator: str) -> QueryString: + return QueryString( f"{operator}('{self.template}', {self._columns_string}) AS " f"{self.output_name}" ) @property - def sqlite_string(self) -> str: + def sqlite_string(self) -> QueryString: return self._get_string(operator="PRINTF") @property - def postgres_string(self) -> str: + def postgres_string(self) -> QueryString: return self._get_string(operator="FORMAT") @property - def cockroach_string(self) -> str: + def cockroach_string(self) -> QueryString: return self._get_string(operator="FORMAT") - def get_select_string(self, engine_type: str, with_alias=True) -> str: + def get_select_string( + self, engine_type: str, with_alias=True + ) -> QueryString: try: return getattr(self, f"{engine_type}_string") except AttributeError as e: diff --git a/piccolo/query/__init__.py b/piccolo/query/__init__.py index 000a47e76..2fcc2df7e 100644 --- a/piccolo/query/__init__.py +++ b/piccolo/query/__init__.py @@ -1,9 +1,9 @@ from piccolo.columns.combination import WhereRaw from .base import Query +from .functions.aggregate import Avg, Max, Min, Sum from .methods import ( Alter, - Avg, Count, Create, CreateIndex, @@ -11,12 +11,9 @@ DropIndex, Exists, Insert, - Max, - Min, Objects, Raw, Select, - Sum, TableExists, Update, ) diff --git a/piccolo/query/functions/__init__.py b/piccolo/query/functions/__init__.py new file mode 100644 index 000000000..d0195cc40 --- /dev/null +++ b/piccolo/query/functions/__init__.py @@ -0,0 +1,16 @@ +from .aggregate import Avg, Count, Max, Min, Sum +from .string import Length, Lower, Ltrim, Reverse, Rtrim, Upper + +__all__ = ( + "Avg", + "Count", + "Length", + "Lower", + "Ltrim", + "Max", + "Min", + "Reverse", + "Rtrim", + "Sum", + "Upper", +) diff --git a/piccolo/query/functions/aggregate.py b/piccolo/query/functions/aggregate.py new file mode 100644 index 000000000..61dd36a46 --- /dev/null +++ b/piccolo/query/functions/aggregate.py @@ -0,0 +1,179 @@ +import typing as t + +from piccolo.columns.base import Column +from piccolo.querystring import QueryString + +from .base import Function + + +class Avg(Function): + """ + ``AVG()`` SQL function. Column type must be numeric to run the query. + + .. code-block:: python + + await Band.select(Avg(Band.popularity)).run() + + # We can use an alias. These two are equivalent: + + await Band.select( + Avg(Band.popularity, alias="popularity_avg") + ).run() + + await Band.select( + Avg(Band.popularity).as_alias("popularity_avg") + ).run() + + """ + + function_name = "AVG" + + +class Count(QueryString): + """ + Used in ``Select`` queries, usually in conjunction with the ``group_by`` + clause:: + + >>> await Band.select( + ... Band.manager.name.as_alias('manager_name'), + ... Count(alias='band_count') + ... ).group_by(Band.manager) + [{'manager_name': 'Guido', 'count': 1}, ...] + + It can also be used without the ``group_by`` clause (though you may prefer + to the :meth:`Table.count ` method instead, as + it's more convenient):: + + >>> await Band.select(Count()) + [{'count': 3}] + + """ + + def __init__( + self, + column: t.Optional[Column] = None, + distinct: t.Optional[t.Sequence[Column]] = None, + alias: str = "count", + ): + """ + :param column: + If specified, the count is for non-null values in that column. + :param distinct: + If specified, the count is for distinct values in those columns. + :param alias: + The name of the value in the response:: + + # These two are equivalent: + + await Band.select( + Band.name, Count(alias="total") + ).group_by(Band.name) + + await Band.select( + Band.name, + Count().as_alias("total") + ).group_by(Band.name) + + """ + if distinct and column: + raise ValueError("Only specify `column` or `distinct`") + + if distinct: + engine_type = distinct[0]._meta.engine_type + if engine_type == "sqlite": + # SQLite doesn't allow us to specify multiple columns, so + # instead we concatenate the values. + column_names = " || ".join("{}" for _ in distinct) + else: + column_names = ", ".join("{}" for _ in distinct) + + return super().__init__( + f"COUNT(DISTINCT({column_names}))", *distinct, alias=alias + ) + else: + if column: + return super().__init__("COUNT({})", column, alias=alias) + else: + return super().__init__("COUNT(*)", alias=alias) + + +class Min(Function): + """ + ``MIN()`` SQL function. + + .. code-block:: python + + await Band.select(Min(Band.popularity)).run() + + # We can use an alias. These two are equivalent: + + await Band.select( + Min(Band.popularity, alias="popularity_min") + ).run() + + await Band.select( + Min(Band.popularity).as_alias("popularity_min") + ).run() + + """ + + function_name = "MIN" + + +class Max(Function): + """ + ``MAX()`` SQL function. + + .. code-block:: python + + await Band.select( + Max(Band.popularity) + ).run() + + # We can use an alias. These two are equivalent: + + await Band.select( + Max(Band.popularity, alias="popularity_max") + ).run() + + await Band.select( + Max(Band.popularity).as_alias("popularity_max") + ).run() + + """ + + function_name = "MAX" + + +class Sum(Function): + """ + ``SUM()`` SQL function. Column type must be numeric to run the query. + + .. code-block:: python + + await Band.select( + Sum(Band.popularity) + ).run() + + # We can use an alias. These two are equivalent: + + await Band.select( + Sum(Band.popularity, alias="popularity_sum") + ).run() + + await Band.select( + Sum(Band.popularity).as_alias("popularity_sum") + ).run() + + """ + + function_name = "SUM" + + +__all__ = ( + "Avg", + "Count", + "Min", + "Max", + "Sum", +) diff --git a/piccolo/query/functions/base.py b/piccolo/query/functions/base.py new file mode 100644 index 000000000..c4181aca6 --- /dev/null +++ b/piccolo/query/functions/base.py @@ -0,0 +1,21 @@ +import typing as t + +from piccolo.columns.base import Column +from piccolo.querystring import QueryString + + +class Function(QueryString): + function_name: str + + def __init__( + self, + identifier: t.Union[Column, QueryString, str], + alias: t.Optional[str] = None, + ): + alias = alias or self.__class__.__name__.lower() + + super().__init__( + f"{self.function_name}({{}})", + identifier, + alias=alias, + ) diff --git a/piccolo/query/functions/string.py b/piccolo/query/functions/string.py new file mode 100644 index 000000000..556817a12 --- /dev/null +++ b/piccolo/query/functions/string.py @@ -0,0 +1,73 @@ +""" +These functions mirror their counterparts in the Postgresql docs: + +https://www.postgresql.org/docs/current/functions-string.html + +""" + +from .base import Function + + +class Length(Function): + """ + Returns the number of characters in the string. + """ + + function_name = "LENGTH" + + +class Lower(Function): + """ + Converts the string to all lower case, according to the rules of the + database's locale. + """ + + function_name = "LOWER" + + +class Ltrim(Function): + """ + Removes the longest string containing only characters in characters (a + space by default) from the start of string. + """ + + function_name = "LTRIM" + + +class Reverse(Function): + """ + Return reversed string. + + Not supported in SQLite. + + """ + + function_name = "REVERSE" + + +class Rtrim(Function): + """ + Removes the longest string containing only characters in characters (a + space by default) from the end of string. + """ + + function_name = "RTRIM" + + +class Upper(Function): + """ + Converts the string to all upper case, according to the rules of the + database's locale. + """ + + function_name = "UPPER" + + +__all__ = ( + "Length", + "Lower", + "Ltrim", + "Reverse", + "Rtrim", + "Upper", +) diff --git a/piccolo/query/methods/__init__.py b/piccolo/query/methods/__init__.py index 6c1854381..f4b9a59f1 100644 --- a/piccolo/query/methods/__init__.py +++ b/piccolo/query/methods/__init__.py @@ -9,6 +9,23 @@ from .objects import Objects from .raw import Raw from .refresh import Refresh -from .select import Avg, Max, Min, Select, Sum +from .select import Select from .table_exists import TableExists from .update import Update + +__all__ = ( + "Alter", + "Count", + "Create", + "CreateIndex", + "Delete", + "DropIndex", + "Exists", + "Insert", + "Objects", + "Raw", + "Refresh", + "Select", + "TableExists", + "Update", +) diff --git a/piccolo/query/methods/count.py b/piccolo/query/methods/count.py index fdd0972cf..99d46c39b 100644 --- a/piccolo/query/methods/count.py +++ b/piccolo/query/methods/count.py @@ -4,7 +4,7 @@ from piccolo.custom_types import Combinable from piccolo.query.base import Query -from piccolo.query.methods.select import Count as SelectCount +from piccolo.query.functions.aggregate import Count as CountFunction from piccolo.query.mixins import WhereDelegate from piccolo.querystring import QueryString @@ -32,7 +32,7 @@ def __init__( ########################################################################### # Clauses - def where(self: Self, *where: Combinable) -> Self: + def where(self: Self, *where: t.Union[Combinable, QueryString]) -> Self: self.where_delegate.where(*where) return self @@ -50,7 +50,7 @@ def default_querystrings(self) -> t.Sequence[QueryString]: table: t.Type[Table] = self.table query = table.select( - SelectCount(column=self.column, distinct=self._distinct) + CountFunction(column=self.column, distinct=self._distinct) ) query.where_delegate._where = self.where_delegate._where diff --git a/piccolo/query/methods/delete.py b/piccolo/query/methods/delete.py index bc0746063..628b89b8e 100644 --- a/piccolo/query/methods/delete.py +++ b/piccolo/query/methods/delete.py @@ -30,7 +30,7 @@ def __init__(self, table: t.Type[Table], force: bool = False, **kwargs): self.returning_delegate = ReturningDelegate() self.where_delegate = WhereDelegate() - def where(self: Self, *where: Combinable) -> Self: + def where(self: Self, *where: t.Union[Combinable, QueryString]) -> Self: self.where_delegate.where(*where) return self diff --git a/piccolo/query/methods/exists.py b/piccolo/query/methods/exists.py index 26d25e03e..7fac83a75 100644 --- a/piccolo/query/methods/exists.py +++ b/piccolo/query/methods/exists.py @@ -16,7 +16,7 @@ def __init__(self, table: t.Type[TableInstance], **kwargs): super().__init__(table, **kwargs) self.where_delegate = WhereDelegate() - def where(self: Self, *where: Combinable) -> Self: + def where(self: Self, *where: t.Union[Combinable, QueryString]) -> Self: self.where_delegate.where(*where) return self diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index 7b8c3ad43..f11f78e8e 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -262,7 +262,7 @@ def order_by( self.order_by_delegate.order_by(*_columns, ascending=ascending) return self - def where(self: Self, *where: Combinable) -> Self: + def where(self: Self, *where: t.Union[Combinable, QueryString]) -> Self: self.where_delegate.where(*where) return self diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index a2a77b155..fdb929f8a 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -1,6 +1,5 @@ from __future__ import annotations -import decimal import itertools import typing as t from collections import OrderedDict @@ -36,9 +35,8 @@ from piccolo.custom_types import Combinable from piccolo.table import Table # noqa - -def is_numeric_column(column: Column) -> bool: - return column.value_type in (int, decimal.Decimal, float) +# Here to avoid breaking changes - will be removed in the future. +from piccolo.query.functions.aggregate import Count # noqa: F401 class SelectRaw(Selectable): @@ -59,224 +57,8 @@ def __init__(self, sql: str, *args: t.Any) -> None: def get_select_string( self, engine_type: str, with_alias: bool = True - ) -> str: - return self.querystring.__str__() - - -class Avg(Selectable): - """ - ``AVG()`` SQL function. Column type must be numeric to run the query. - - .. code-block:: python - - await Band.select(Avg(Band.popularity)).run() - - # We can use an alias. These two are equivalent: - - await Band.select( - Avg(Band.popularity, alias="popularity_avg") - ).run() - - await Band.select( - Avg(Band.popularity).as_alias("popularity_avg") - ).run() - - """ - - def __init__(self, column: Column, alias: str = "avg"): - if is_numeric_column(column): - self.column = column - else: - raise ValueError("Column type must be numeric to run the query.") - self._alias = alias - - def get_select_string( - self, engine_type: str, with_alias: bool = True - ) -> str: - column_name = self.column._meta.get_full_name(with_alias=False) - return f'AVG({column_name}) AS "{self._alias}"' - - -class Count(Selectable): - """ - Used in ``Select`` queries, usually in conjunction with the ``group_by`` - clause:: - - >>> await Band.select( - ... Band.manager.name.as_alias('manager_name'), - ... Count(alias='band_count') - ... ).group_by(Band.manager) - [{'manager_name': 'Guido', 'count': 1}, ...] - - It can also be used without the ``group_by`` clause (though you may prefer - to the :meth:`Table.count ` method instead, as - it's more convenient):: - - >>> await Band.select(Count()) - [{'count': 3}] - - """ - - def __init__( - self, - column: t.Optional[Column] = None, - distinct: t.Optional[t.Sequence[Column]] = None, - alias: str = "count", - ): - """ - :param column: - If specified, the count is for non-null values in that column. - :param distinct: - If specified, the count is for distinct values in those columns. - :param alias: - The name of the value in the response:: - - # These two are equivalent: - - await Band.select( - Band.name, Count(alias="total") - ).group_by(Band.name) - - await Band.select( - Band.name, - Count().as_alias("total") - ).group_by(Band.name) - - """ - if distinct and column: - raise ValueError("Only specify `column` or `distinct`") - - self.column = column - self.distinct = distinct - self._alias = alias - - def get_select_string( - self, engine_type: str, with_alias: bool = True - ) -> str: - expression: str - - if self.distinct: - if engine_type == "sqlite": - # SQLite doesn't allow us to specify multiple columns, so - # instead we concatenate the values. - column_names = " || ".join( - i._meta.get_full_name(with_alias=False) - for i in self.distinct - ) - else: - column_names = ", ".join( - i._meta.get_full_name(with_alias=False) - for i in self.distinct - ) - - expression = f"DISTINCT ({column_names})" - else: - if self.column: - expression = self.column._meta.get_full_name(with_alias=False) - else: - expression = "*" - - return f'COUNT({expression}) AS "{self._alias}"' - - -class Max(Selectable): - """ - ``MAX()`` SQL function. - - .. code-block:: python - - await Band.select( - Max(Band.popularity) - ).run() - - # We can use an alias. These two are equivalent: - - await Band.select( - Max(Band.popularity, alias="popularity_max") - ).run() - - await Band.select( - Max(Band.popularity).as_alias("popularity_max") - ).run() - - """ - - def __init__(self, column: Column, alias: str = "max"): - self.column = column - self._alias = alias - - def get_select_string( - self, engine_type: str, with_alias: bool = True - ) -> str: - column_name = self.column._meta.get_full_name(with_alias=False) - return f'MAX({column_name}) AS "{self._alias}"' - - -class Min(Selectable): - """ - ``MIN()`` SQL function. - - .. code-block:: python - - await Band.select(Min(Band.popularity)).run() - - # We can use an alias. These two are equivalent: - - await Band.select( - Min(Band.popularity, alias="popularity_min") - ).run() - - await Band.select( - Min(Band.popularity).as_alias("popularity_min") - ).run() - - """ - - def __init__(self, column: Column, alias: str = "min"): - self.column = column - self._alias = alias - - def get_select_string( - self, engine_type: str, with_alias: bool = True - ) -> str: - column_name = self.column._meta.get_full_name(with_alias=False) - return f'MIN({column_name}) AS "{self._alias}"' - - -class Sum(Selectable): - """ - ``SUM()`` SQL function. Column type must be numeric to run the query. - - .. code-block:: python - - await Band.select( - Sum(Band.popularity) - ).run() - - # We can use an alias. These two are equivalent: - - await Band.select( - Sum(Band.popularity, alias="popularity_sum") - ).run() - - await Band.select( - Sum(Band.popularity).as_alias("popularity_sum") - ).run() - - """ - - def __init__(self, column: Column, alias: str = "sum"): - if is_numeric_column(column): - self.column = column - else: - raise ValueError("Column type must be numeric to run the query.") - self._alias = alias - - def get_select_string( - self, engine_type: str, with_alias: bool = True - ) -> str: - column_name = self.column._meta.get_full_name(with_alias=False) - return f'SUM({column_name}) AS "{self._alias}"' + ) -> QueryString: + return self.querystring OptionalDict = t.Optional[t.Dict[str, t.Any]] @@ -645,7 +427,7 @@ def callback( self.callback_delegate.callback(callbacks, on=on) return self - def where(self: Self, *where: Combinable) -> Self: + def where(self: Self, *where: t.Union[Combinable, QueryString]) -> Self: self.where_delegate.where(*where) return self @@ -678,23 +460,25 @@ def _get_joins(self, columns: t.Sequence[Selectable]) -> t.List[str]: for readable in readables: columns += readable.columns + querystrings: t.List[QueryString] = [ + i for i in columns if isinstance(i, QueryString) + ] + for querystring in querystrings: + if querystring_columns := getattr(querystring, "columns", []): + columns += querystring_columns + for column in columns: if not isinstance(column, Column): continue _joins: t.List[str] = [] for index, key in enumerate(column._meta.call_chain, 0): - table_alias = "$".join( - f"{_key._meta.table._meta.tablename}${_key._meta.name}" - for _key in column._meta.call_chain[: index + 1] - ) - - key._meta.table_alias = table_alias + table_alias = key.table_alias if index > 0: left_tablename = column._meta.call_chain[ index - 1 - ]._meta.table_alias + ].table_alias else: left_tablename = ( key._meta.table._meta.get_formatted_tablename() @@ -761,11 +545,10 @@ def default_querystrings(self) -> t.Sequence[QueryString]: engine_type = self.table._meta.db.engine_type - select_strings: t.List[str] = [ + select_strings: t.List[QueryString] = [ c.get_select_string(engine_type=engine_type) for c in self.columns_delegate.selected_columns ] - columns_str = ", ".join(select_strings) ####################################################################### @@ -779,7 +562,9 @@ def default_querystrings(self) -> t.Sequence[QueryString]: query += "{}" args.append(distinct.querystring) + columns_str = ", ".join("{}" for i in select_strings) query += f" {columns_str} FROM {self.table._meta.get_formatted_tablename()}" # noqa: E501 + args.extend(select_strings) for join in joins: query += f" {join}" diff --git a/piccolo/query/methods/update.py b/piccolo/query/methods/update.py index ff6a10589..f75854c43 100644 --- a/piccolo/query/methods/update.py +++ b/piccolo/query/methods/update.py @@ -50,7 +50,7 @@ def values( self.values_delegate.values(values) return self - def where(self, *where: Combinable) -> Update: + def where(self, *where: t.Union[Combinable, QueryString]) -> Update: self.where_delegate.where(*where) return self diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index 8d7c6a4a9..214d1b8d7 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -9,13 +9,14 @@ from piccolo.columns import And, Column, Or, Where from piccolo.columns.column_types import ForeignKey +from piccolo.columns.combination import WhereRaw from piccolo.custom_types import Combinable from piccolo.querystring import QueryString from piccolo.utils.list import flatten from piccolo.utils.sql_values import convert_to_sql_value if t.TYPE_CHECKING: # pragma: no cover - from piccolo.columns.base import Selectable + from piccolo.querystring import Selectable from piccolo.table import Table # noqa @@ -254,8 +255,10 @@ def _extract_columns(self, combinable: Combinable): elif isinstance(combinable, (And, Or)): self._extract_columns(combinable.first) self._extract_columns(combinable.second) + elif isinstance(combinable, WhereRaw): + self._where_columns.extend(combinable.querystring.columns) - def where(self, *where: Combinable): + def where(self, *where: t.Union[Combinable, QueryString]): for arg in where: if isinstance(arg, bool): raise ValueError( @@ -265,6 +268,10 @@ def where(self, *where: Combinable): "`.where(MyTable.some_column.is_null())`." ) + if isinstance(arg, QueryString): + # If a raw QueryString is passed in. + arg = WhereRaw(arg.template, *arg.args) + self._where = And(self._where, arg) if self._where else arg diff --git a/piccolo/querystring.py b/piccolo/querystring.py index 3c23d86dc..7f3f3e42a 100644 --- a/piccolo/querystring.py +++ b/piccolo/querystring.py @@ -1,6 +1,7 @@ from __future__ import annotations import typing as t +from abc import ABCMeta, abstractmethod from dataclasses import dataclass from datetime import datetime from importlib.util import find_spec @@ -8,6 +9,7 @@ if t.TYPE_CHECKING: # pragma: no cover from piccolo.table import Table + from piccolo.columns import Column from uuid import UUID @@ -17,22 +19,32 @@ apgUUID = UUID -@dataclass -class Unquoted: +class Selectable(metaclass=ABCMeta): """ - Used when we want the value to be unquoted because it's a Postgres - keyword - for example DEFAULT. + Anything which inherits from this can be used in a select query. """ - __slots__ = ("value",) + __slots__ = ("_alias",) - value: str + _alias: t.Optional[str] - def __repr__(self): - return f"{self.value}" + @abstractmethod + def get_select_string( + self, engine_type: str, with_alias: bool = True + ) -> QueryString: + """ + In a query, what to output after the select statement - could be a + column name, a sub query, a function etc. For a column it will be the + column name. + """ + raise NotImplementedError() - def __str__(self): - return f"{self.value}" + def as_alias(self, alias: str) -> Selectable: + """ + Allows column names to be changed in the result of a select. + """ + self._alias = alias + return self @dataclass @@ -42,7 +54,7 @@ class Fragment: no_arg: bool = False -class QueryString: +class QueryString(Selectable): """ When we're composing complex queries, we're combining QueryStrings, rather than concatenating strings directly. The reason for this is QueryStrings @@ -56,6 +68,7 @@ class QueryString: "query_type", "table", "_frozen_compiled_strings", + "columns", ) def __init__( @@ -64,6 +77,7 @@ def __init__( *args: t.Any, query_type: str = "generic", table: t.Optional[t.Type[Table]] = None, + alias: t.Optional[str] = None, ) -> None: """ :param template: @@ -83,12 +97,42 @@ def __init__( """ self.template = template - self.args = args self.query_type = query_type self.table = table self._frozen_compiled_strings: t.Optional[ t.Tuple[str, t.List[t.Any]] ] = None + self._alias = alias + self.args, self.columns = self.process_args(args) + + def process_args( + self, args: t.Sequence[t.Any] + ) -> t.Tuple[t.Sequence[t.Any], t.Sequence[Column]]: + """ + If a Column is passed in, we convert it to the name of the column + (including joins). + """ + from piccolo.columns import Column + + processed_args = [] + columns = [] + + for arg in args: + if isinstance(arg, Column): + columns.append(arg) + arg = QueryString( + f"{arg._meta.get_full_name(with_alias=False)}" + ) + elif isinstance(arg, QueryString): + columns.extend(arg.columns) + + processed_args.append(arg) + + return (processed_args, columns) + + def as_alias(self, alias: str) -> QueryString: + self._alias = alias + return self def __str__(self): """ @@ -143,7 +187,7 @@ def bundle( fragment.no_arg = True bundled.append(fragment) else: - if isinstance(value, self.__class__): + if isinstance(value, QueryString): fragment.no_arg = True bundled.append(fragment) @@ -195,3 +239,47 @@ def freeze(self, engine_type: str = "postgres"): self._frozen_compiled_strings = self.compile_string( engine_type=engine_type ) + + ########################################################################### + + def get_select_string( + self, engine_type: str, with_alias: bool = True + ) -> QueryString: + if with_alias and self._alias: + return QueryString("{} AS " + self._alias, self) + else: + return self + + def get_where_string(self, engine_type: str) -> QueryString: + return self.get_select_string( + engine_type=engine_type, with_alias=False + ) + + ########################################################################### + # Basic logic + + def __eq__(self, value) -> QueryString: # type: ignore[override] + return QueryString("{} = {}", self, value) + + def __ne__(self, value) -> QueryString: # type: ignore[override] + return QueryString("{} != {}", self, value) + + def __add__(self, value) -> QueryString: + return QueryString("{} + {}", self, value) + + def __sub__(self, value) -> QueryString: + return QueryString("{} - {}", self, value) + + def is_in(self, value) -> QueryString: + return QueryString("{} IN {}", self, value) + + def not_in(self, value) -> QueryString: + return QueryString("{} NOT IN {}", self, value) + + +class Unquoted(QueryString): + """ + This is deprecated - just use QueryString directly. + """ + + pass diff --git a/piccolo/table.py b/piccolo/table.py index b4fcbf942..7882db95e 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -48,7 +48,7 @@ from piccolo.query.methods.indexes import Indexes from piccolo.query.methods.objects import First from piccolo.query.methods.refresh import Refresh -from piccolo.querystring import QueryString, Unquoted +from piccolo.querystring import QueryString from piccolo.utils import _camel_to_snake from piccolo.utils.graphlib import TopologicalSorter from piccolo.utils.sql_values import convert_to_sql_value @@ -56,7 +56,7 @@ from piccolo.utils.warnings import colored_warning if t.TYPE_CHECKING: # pragma: no cover - from piccolo.columns import Selectable + from piccolo.querystring import Selectable PROTECTED_TABLENAMES = ("user",) TABLENAME_WARNING = ( @@ -796,30 +796,14 @@ def querystring(self) -> QueryString: """ Used when inserting rows. """ - args_dict = {} - for col in self._meta.columns: - column_name = col._meta.name - value = convert_to_sql_value(value=self[column_name], column=col) - args_dict[column_name] = value - - def is_unquoted(arg): - return isinstance(arg, Unquoted) - - # Strip out any args which are unquoted. - filtered_args = [i for i in args_dict.values() if not is_unquoted(i)] + args = [ + convert_to_sql_value(value=self[column._meta.name], column=column) + for column in self._meta.columns + ] # If unquoted, dump it straight into the query. - query = ",".join( - [ - ( - args_dict[column._meta.name].value - if is_unquoted(args_dict[column._meta.name]) - else "{}" - ) - for column in self._meta.columns - ] - ) - return QueryString(f"({query})", *filtered_args) + query = ",".join(["{}" for _ in args]) + return QueryString(f"({query})", *args) def __str__(self) -> str: return self.querystring.__str__() diff --git a/tests/query/test_functions.py b/tests/query/test_functions.py new file mode 100644 index 000000000..abe9a5f01 --- /dev/null +++ b/tests/query/test_functions.py @@ -0,0 +1,102 @@ +from unittest import TestCase + +from piccolo.query.functions.string import Reverse, Upper +from piccolo.querystring import QueryString +from piccolo.table import create_db_tables_sync, drop_db_tables_sync +from tests.base import engines_skip +from tests.example_apps.music.tables import Band, Manager + + +class FunctionTest(TestCase): + tables = (Band, Manager) + + def setUp(self) -> None: + create_db_tables_sync(*self.tables) + + manager = Manager({Manager.name: "Guido"}) + manager.save().run_sync() + + band = Band({Band.name: "Pythonistas", Band.manager: manager}) + band.save().run_sync() + + def tearDown(self) -> None: + drop_db_tables_sync(*self.tables) + + +class TestUpperFunction(FunctionTest): + + def test_column(self): + """ + Make sure we can uppercase a column's value. + """ + response = Band.select(Upper(Band.name)).run_sync() + self.assertListEqual(response, [{"upper": "PYTHONISTAS"}]) + + def test_alias(self): + response = Band.select(Upper(Band.name, alias="name")).run_sync() + self.assertListEqual(response, [{"name": "PYTHONISTAS"}]) + + def test_joined_column(self): + """ + Make sure we can uppercase a column's value from a joined table. + """ + response = Band.select(Upper(Band.manager._.name)).run_sync() + self.assertListEqual(response, [{"upper": "GUIDO"}]) + + +@engines_skip("sqlite") +class TestNested(FunctionTest): + """ + Skip the the test for SQLite, as it doesn't support ``Reverse``. + """ + + def test_nested(self): + """ + Make sure we can nest functions. + """ + response = Band.select(Upper(Reverse(Band.name))).run_sync() + self.assertListEqual(response, [{"upper": "SATSINOHTYP"}]) + + def test_nested_with_joined_column(self): + """ + Make sure nested functions can be used on a column from a joined table. + """ + response = Band.select(Upper(Reverse(Band.manager._.name))).run_sync() + self.assertListEqual(response, [{"upper": "ODIUG"}]) + + def test_nested_within_querystring(self): + """ + If we wrap a function in a custom QueryString - make sure the columns + are still accessible, so joins are successful. + """ + response = Band.select( + QueryString("CONCAT({}, '!')", Upper(Band.manager._.name)), + ).run_sync() + + self.assertListEqual(response, [{"concat": "GUIDO!"}]) + + +class TestWhereClause(FunctionTest): + + def test_where(self): + """ + Make sure where clauses work with functions. + """ + response = ( + Band.select(Band.name) + .where(Upper(Band.name) == "PYTHONISTAS") + .run_sync() + ) + self.assertListEqual(response, [{"name": "Pythonistas"}]) + + def test_where_with_joined_column(self): + """ + Make sure where clauses work with functions, when a joined column is + used. + """ + response = ( + Band.select(Band.name) + .where(Upper(Band.manager._.name) == "GUIDO") + .run_sync() + ) + self.assertListEqual(response, [{"name": "Pythonistas"}]) diff --git a/tests/table/test_select.py b/tests/table/test_select.py index a2bb86981..ebf2c3ff8 100644 --- a/tests/table/test_select.py +++ b/tests/table/test_select.py @@ -7,7 +7,8 @@ from piccolo.columns import Date, Varchar from piccolo.columns.combination import WhereRaw from piccolo.query import OrderByRaw -from piccolo.query.methods.select import Avg, Count, Max, Min, SelectRaw, Sum +from piccolo.query.functions.aggregate import Avg, Count, Max, Min, Sum +from piccolo.query.methods.select import SelectRaw from piccolo.query.mixins import DistinctOnError from piccolo.table import Table, create_db_tables_sync, drop_db_tables_sync from tests.base import ( @@ -927,14 +928,6 @@ def test_chain_different_functions_alias(self): self.assertEqual(float(response["popularity_avg"]), 1003.3333333333334) self.assertEqual(response["popularity_sum"], 3010) - def test_avg_validation(self): - with self.assertRaises(ValueError): - Band.select(Avg(Band.name)).run_sync() - - def test_sum_validation(self): - with self.assertRaises(ValueError): - Band.select(Sum(Band.name)).run_sync() - def test_columns(self): """ Make sure the colums method can be used to specify which columns to From 14af0e3ad14454a2b29da386765df5479da30a1b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 30 May 2024 01:02:40 +0100 Subject: [PATCH 566/727] bumped version --- CHANGES.rst | 21 +++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index e1fb2c900..493159c7d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,27 @@ Changes ======= +1.6.0 +----- + +Added support for a bunch of Postgres functions, like ``Upper``, ``Lower``, +``Length``, and ``Ltrim``. They can be used in ``select`` queries: + +.. code-block:: python + + from piccolo.query.functions.string import Upper + >>> await Band.select(Upper(Band.name)) + [["name": "PYTHONISTAS"]] + +And also in ``where`` clauses: + +.. code-block:: python + + >>> await Band.select().where(Upper(Band.manager.name) == 'GUIDO') + [["name": "Pythonistas"]] + +------------------------------------------------------------------------------- + 1.5.2 ----- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 2c74cfe9e..bb270431a 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.5.2" +__VERSION__ = "1.6.0" From b1de70a500a76e7cc679312ebebece2fbe252b1e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 30 May 2024 01:05:11 +0100 Subject: [PATCH 567/727] fix typos in changelog --- CHANGES.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 493159c7d..fbdbec34e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -10,15 +10,15 @@ Added support for a bunch of Postgres functions, like ``Upper``, ``Lower``, .. code-block:: python from piccolo.query.functions.string import Upper - >>> await Band.select(Upper(Band.name)) - [["name": "PYTHONISTAS"]] + >>> await Band.select(Upper(Band.name, alias="name")) + [{"name": "PYTHONISTAS"}] And also in ``where`` clauses: .. code-block:: python >>> await Band.select().where(Upper(Band.manager.name) == 'GUIDO') - [["name": "Pythonistas"]] + [{"name": "Pythonistas"}] ------------------------------------------------------------------------------- From 9f1a4d8d1e61d2d6b1b100fe0cd64ea44261ed25 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 31 May 2024 12:32:11 +0100 Subject: [PATCH 568/727] 1003 Support arrays of timestamp / timestamptz / date / time in SQLite (#1004) * support date / time / timestamp arrays in SQLite * don't run tests for cockroachdb for now --- piccolo/columns/column_types.py | 31 ++++- piccolo/engine/sqlite.py | 228 ++++++++++++++++++++++++-------- tests/columns/test_array.py | 91 ++++++++++++- 3 files changed, 286 insertions(+), 64 deletions(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index add0c6f5c..e16ffe6ab 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -2532,7 +2532,14 @@ def column_type(self): if engine_type in ("postgres", "cockroach"): return f"{self.base_column.column_type}[]" elif engine_type == "sqlite": - return "ARRAY" + inner_column = self._get_inner_column() + return ( + f"ARRAY_{inner_column.column_type}" + if isinstance( + inner_column, (Date, Timestamp, Timestamptz, Time) + ) + else "ARRAY" + ) raise Exception("Unrecognized engine type") def _setup_base_column(self, table_class: t.Type[Table]): @@ -2564,6 +2571,23 @@ def _get_dimensions(self, start: int = 0) -> int: else: return start + 1 + def _get_inner_column(self) -> Column: + """ + A helper function to get the innermost ``Column`` for the array. For + example:: + + >>> Array(Varchar())._get_inner_column() + Varchar + + >>> Array(Array(Varchar()))._get_inner_column() + Varchar + + """ + if isinstance(self.base_column, Array): + return self.base_column._get_inner_column() + else: + return self.base_column + def _get_inner_value_type(self) -> t.Type: """ A helper function to get the innermost value type for the array. For @@ -2576,10 +2600,7 @@ def _get_inner_value_type(self) -> t.Type: str """ - if isinstance(self.base_column, Array): - return self.base_column._get_inner_value_type() - else: - return self.base_column.value_type + return self._get_inner_column().value_type def __getitem__(self, value: int) -> Array: """ diff --git a/piccolo/engine/sqlite.py b/piccolo/engine/sqlite.py index 862afaa71..f6fbd4e38 100644 --- a/piccolo/engine/sqlite.py +++ b/piccolo/engine/sqlite.py @@ -9,6 +9,7 @@ import uuid from dataclasses import dataclass from decimal import Decimal +from functools import partial, wraps from piccolo.engine.base import Batch, Engine, validate_savepoint_name from piccolo.engine.exceptions import TransactionError @@ -35,14 +36,14 @@ # In -def convert_numeric_in(value): +def convert_numeric_in(value: Decimal) -> float: """ Convert any Decimal values into floats. """ return float(value) -def convert_uuid_in(value) -> str: +def convert_uuid_in(value: uuid.UUID) -> str: """ Converts the UUID value being passed into sqlite. """ @@ -56,7 +57,7 @@ def convert_time_in(value: datetime.time) -> str: return value.isoformat() -def convert_date_in(value: datetime.date): +def convert_date_in(value: datetime.date) -> str: """ Converts the date value being passed into sqlite. """ @@ -74,122 +75,235 @@ def convert_datetime_in(value: datetime.datetime) -> str: return str(value) -def convert_timedelta_in(value: datetime.timedelta): +def convert_timedelta_in(value: datetime.timedelta) -> float: """ Converts the timedelta value being passed into sqlite. """ return value.total_seconds() -def convert_array_in(value: list): +def convert_array_in(value: list) -> str: """ - Converts a list value into a string. + Converts a list value into a string (it handles nested lists, and type like + dateime/ time / date which aren't usually JSON serialisable.). + """ - if value and type(value[0]) not in [str, int, float, list]: - raise ValueError("Can only serialise str, int, float, and list.") - return dump_json(value) + def serialise(data: list): + output = [] + + for item in data: + if isinstance(item, list): + output.append(serialise(item)) + elif isinstance( + item, (datetime.datetime, datetime.time, datetime.date) + ): + if adapter := ADAPTERS.get(type(item)): + output.append(adapter(item)) + else: + raise ValueError("The adapter wasn't found.") + elif item is None or isinstance(item, (str, int, float, list)): + # We can safely JSON serialise these. + output.append(item) + else: + raise ValueError("We can't currently serialise this value.") + + return output + + return dump_json(serialise(value)) + + +############################################################################### + +# Register adapters + +ADAPTERS: t.Dict[t.Type, t.Callable[[t.Any], t.Any]] = { + Decimal: convert_numeric_in, + uuid.UUID: convert_uuid_in, + datetime.time: convert_time_in, + datetime.date: convert_date_in, + datetime.datetime: convert_datetime_in, + datetime.timedelta: convert_timedelta_in, + list: convert_array_in, +} +for value_type, adapter in ADAPTERS.items(): + sqlite3.register_adapter(value_type, adapter) + +############################################################################### # Out -def convert_numeric_out(value: bytes) -> Decimal: +def decode_to_string(converter: t.Callable[[str], t.Any]): + """ + This means we can use our converters with string and bytes. They are + passed bytes when used directly via SQLite, and are passed strings when + used by the array converters. + """ + + @wraps(converter) + def wrapper(value: t.Union[str, bytes]) -> t.Any: + if isinstance(value, bytes): + return converter(value.decode("utf8")) + elif isinstance(value, str): + return converter(value) + else: + raise ValueError("Unsupported type") + + return wrapper + + +@decode_to_string +def convert_numeric_out(value: str) -> Decimal: """ Convert float values into Decimals. """ - return Decimal(value.decode("ascii")) + return Decimal(value) -def convert_int_out(value: bytes) -> int: +@decode_to_string +def convert_int_out(value: str) -> int: """ Make sure Integer values are actually of type int. """ return int(float(value)) -def convert_uuid_out(value: bytes) -> uuid.UUID: +@decode_to_string +def convert_uuid_out(value: str) -> uuid.UUID: """ If the value is a uuid, convert it to a UUID instance. """ - return uuid.UUID(value.decode("utf8")) + return uuid.UUID(value) -def convert_date_out(value: bytes) -> datetime.date: - return datetime.date.fromisoformat(value.decode("utf8")) +@decode_to_string +def convert_date_out(value: str) -> datetime.date: + return datetime.date.fromisoformat(value) -def convert_time_out(value: bytes) -> datetime.time: +@decode_to_string +def convert_time_out(value: str) -> datetime.time: """ If the value is a time, convert it to a UUID instance. """ - return datetime.time.fromisoformat(value.decode("utf8")) + return datetime.time.fromisoformat(value) -def convert_seconds_out(value: bytes) -> datetime.timedelta: +@decode_to_string +def convert_seconds_out(value: str) -> datetime.timedelta: """ If the value is from a seconds column, convert it to a timedelta instance. """ - return datetime.timedelta(seconds=float(value.decode("utf8"))) + return datetime.timedelta(seconds=float(value)) -def convert_boolean_out(value: bytes) -> bool: +@decode_to_string +def convert_boolean_out(value: str) -> bool: """ If the value is from a boolean column, convert it to a bool value. """ - _value = value.decode("utf8") - return _value == "1" + return value == "1" -def convert_timestamp_out(value: bytes) -> datetime.datetime: +@decode_to_string +def convert_timestamp_out(value: str) -> datetime.datetime: """ If the value is from a timestamp column, convert it to a datetime value. """ - return datetime.datetime.fromisoformat(value.decode("utf8")) + return datetime.datetime.fromisoformat(value) -def convert_timestamptz_out(value: bytes) -> datetime.datetime: +@decode_to_string +def convert_timestamptz_out(value: str) -> datetime.datetime: """ If the value is from a timestamptz column, convert it to a datetime value, with a timezone of UTC. """ - _value = datetime.datetime.fromisoformat(value.decode("utf8")) - _value = _value.replace(tzinfo=datetime.timezone.utc) - return _value + return datetime.datetime.fromisoformat(value).replace( + tzinfo=datetime.timezone.utc + ) -def convert_array_out(value: bytes) -> t.List: +@decode_to_string +def convert_array_out(value: str) -> t.List: """ If the value if from an array column, deserialise the string back into a list. """ - return load_json(value.decode("utf8")) - - -def convert_M2M_out(value: bytes) -> t.List: - _value = value.decode("utf8") - return _value.split(",") - - -sqlite3.register_converter("Numeric", convert_numeric_out) -sqlite3.register_converter("Integer", convert_int_out) -sqlite3.register_converter("UUID", convert_uuid_out) -sqlite3.register_converter("Date", convert_date_out) -sqlite3.register_converter("Time", convert_time_out) -sqlite3.register_converter("Seconds", convert_seconds_out) -sqlite3.register_converter("Boolean", convert_boolean_out) -sqlite3.register_converter("Timestamp", convert_timestamp_out) -sqlite3.register_converter("Timestamptz", convert_timestamptz_out) -sqlite3.register_converter("Array", convert_array_out) -sqlite3.register_converter("M2M", convert_M2M_out) - -sqlite3.register_adapter(Decimal, convert_numeric_in) -sqlite3.register_adapter(uuid.UUID, convert_uuid_in) -sqlite3.register_adapter(datetime.time, convert_time_in) -sqlite3.register_adapter(datetime.date, convert_date_in) -sqlite3.register_adapter(datetime.datetime, convert_datetime_in) -sqlite3.register_adapter(datetime.timedelta, convert_timedelta_in) -sqlite3.register_adapter(list, convert_array_in) + return load_json(value) + + +def convert_complex_array_out(value: bytes, converter: t.Callable): + """ + This is used to handle arrays of things like timestamps, which we can't + just load from JSON without doing additional work to convert the elements + back into Python objects. + """ + parsed = load_json(value.decode("utf8")) + + def convert_list(list_value: t.List): + output = [] + + for value in list_value: + if isinstance(value, list): + # For nested arrays + output.append(convert_list(value)) + elif isinstance(value, str): + output.append(converter(value)) + else: + output.append(value) + + return output + + if isinstance(parsed, list): + return convert_list(parsed) + else: + return parsed + + +@decode_to_string +def convert_M2M_out(value: str) -> t.List: + return value.split(",") + + +############################################################################### +# Register the basic converters + +CONVERTERS = { + "NUMERIC": convert_numeric_out, + "INTEGER": convert_int_out, + "UUID": convert_uuid_out, + "DATE": convert_date_out, + "TIME": convert_time_out, + "SECONDS": convert_seconds_out, + "BOOLEAN": convert_boolean_out, + "TIMESTAMP": convert_timestamp_out, + "TIMESTAMPTZ": convert_timestamptz_out, + "M2M": convert_M2M_out, +} + +for column_name, converter in CONVERTERS.items(): + sqlite3.register_converter(column_name, converter) + +############################################################################### +# Register the array converters + +# The ARRAY column type handles values which can be easily serialised to and +# from JSON. +sqlite3.register_converter("ARRAY", convert_array_out) + +# We have special column types for arrays of timestamps etc, as simply loading +# the JSON isn't sufficient. +for column_name in ("TIMESTAMP", "TIMESTAMPTZ", "DATE", "TIME"): + sqlite3.register_converter( + f"ARRAY_{column_name}", + partial( + convert_complex_array_out, + converter=CONVERTERS[column_name], + ), + ) ############################################################################### diff --git a/tests/columns/test_array.py b/tests/columns/test_array.py index 6e325fbd4..4677ef995 100644 --- a/tests/columns/test_array.py +++ b/tests/columns/test_array.py @@ -1,6 +1,15 @@ +import datetime from unittest import TestCase -from piccolo.columns.column_types import Array, BigInt, Integer +from piccolo.columns.column_types import ( + Array, + BigInt, + Date, + Integer, + Time, + Timestamp, + Timestamptz, +) from piccolo.table import Table from tests.base import engines_only, sqlite_only @@ -22,7 +31,7 @@ def test_array_default(self): class TestArray(TestCase): """ - Make sure an Array column can be created, and work correctly. + Make sure an Array column can be created, and works correctly. """ def setUp(self): @@ -166,6 +175,84 @@ def test_cat_sqlite(self): ) +############################################################################### +# Date and time arrays + + +class DateTimeArrayTable(Table): + date = Array(Date()) + time = Array(Time()) + timestamp = Array(Timestamp()) + timestamptz = Array(Timestamptz()) + date_nullable = Array(Date(), null=True) + time_nullable = Array(Time(), null=True) + timestamp_nullable = Array(Timestamp(), null=True) + timestamptz_nullable = Array(Timestamptz(), null=True) + + +class TestDateTimeArray(TestCase): + """ + Make sure that data can be stored and retrieved when using arrays of + date / time / timestamp. + + We have to serialise / deserialise it in a special way in SQLite, hence + the tests. + + """ + + def setUp(self): + DateTimeArrayTable.create_table().run_sync() + + def tearDown(self): + DateTimeArrayTable.alter().drop_table().run_sync() + + @engines_only("postgres", "sqlite") + def test_storage(self): + test_date = datetime.date(year=2024, month=1, day=1) + test_time = datetime.time(hour=12, minute=0) + test_timestamp = datetime.datetime( + year=2024, month=1, day=1, hour=12, minute=0 + ) + test_timestamptz = datetime.datetime( + year=2024, + month=1, + day=1, + hour=12, + minute=0, + tzinfo=datetime.timezone.utc, + ) + + DateTimeArrayTable( + { + DateTimeArrayTable.date: [test_date], + DateTimeArrayTable.time: [test_time], + DateTimeArrayTable.timestamp: [test_timestamp], + DateTimeArrayTable.timestamptz: [test_timestamptz], + DateTimeArrayTable.date_nullable: None, + DateTimeArrayTable.time_nullable: None, + DateTimeArrayTable.timestamp_nullable: None, + DateTimeArrayTable.timestamptz_nullable: None, + } + ).save().run_sync() + + row = DateTimeArrayTable.objects().first().run_sync() + assert row is not None + + self.assertListEqual(row.date, [test_date]) + self.assertListEqual(row.time, [test_time]) + self.assertListEqual(row.timestamp, [test_timestamp]) + self.assertListEqual(row.timestamptz, [test_timestamptz]) + + self.assertIsNone(row.date_nullable) + self.assertIsNone(row.time_nullable) + self.assertIsNone(row.timestamp_nullable) + self.assertIsNone(row.timestamptz_nullable) + + +############################################################################### +# Nested arrays + + class NestedArrayTable(Table): value = Array(base_column=Array(base_column=BigInt())) From a45369b3d1b9dcb345faba73ccc9f00fb78902bd Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 31 May 2024 12:36:38 +0100 Subject: [PATCH 569/727] bumped version --- CHANGES.rst | 17 +++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index fbdbec34e..c6c515a3d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,23 @@ Changes ======= +1.7.0 +----- + +Arrays of date / time / timestamps now work in SQLite. + +For example: + +.. code-block:: python + + class MyTable(Table): + times = Array(Time()) + dates = Array(Date()) + timestamps = Array(Timestamp()) + timestamps_tz = Array(Timestamptz()) + +------------------------------------------------------------------------------- + 1.6.0 ----- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index bb270431a..64888e7c6 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.6.0" +__VERSION__ = "1.7.0" From a4d5a4e891348d5664a76af0e8957279dcabea27 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 31 May 2024 12:42:17 +0100 Subject: [PATCH 570/727] improve changelog --- CHANGES.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index c6c515a3d..dd77fba43 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,7 +4,8 @@ Changes 1.7.0 ----- -Arrays of date / time / timestamps now work in SQLite. +Arrays of ``Date`` / ``Time`` / ``Timestamp`` / ``Timestamptz`` now work in +SQLite. For example: From acaa750805aad5c29500b926f11cd3a3d030f093 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 4 Jun 2024 00:33:34 +0100 Subject: [PATCH 571/727] upgrade CockroachDB (#1007) --- .github/workflows/tests.yaml | 2 +- .../auto/integration/test_migrations.py | 33 ++++++++++++++++--- .../migrations/auto/test_migration_manager.py | 30 ++++++++--------- 3 files changed, 44 insertions(+), 21 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 136f86680..26690051f 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -142,7 +142,7 @@ jobs: strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - cockroachdb-version: ["v22.2.0"] + cockroachdb-version: ["v24.1.0"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index 849f8e79f..2851aaee9 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -288,7 +288,12 @@ def test_text_column(self): [ x.data_type == "text", x.is_nullable == "NO", - x.column_default in ("''::text", "'':::STRING"), + x.column_default + in ( + "''", + "''::text", + "'':::STRING", + ), ] ), ) @@ -461,6 +466,7 @@ def test_timestamp_column(self): in ( "now()", "CURRENT_TIMESTAMP", + "current_timestamp()::TIMESTAMP", "current_timestamp():::TIMESTAMPTZ::TIMESTAMP", ), ] @@ -541,7 +547,11 @@ def test_interval_column(self): x.data_type == "interval", x.is_nullable == "NO", x.column_default - in ("'00:00:00'::interval", "'00:00:00':::INTERVAL"), + in ( + "'00:00:00'", + "'00:00:00'::interval", + "'00:00:00':::INTERVAL", + ), ] ), ) @@ -743,7 +753,12 @@ def test_jsonb_column(self): [ x.data_type == "jsonb", x.is_nullable == "NO", - x.column_default in ("'{}'::jsonb", "'{}':::JSONB"), + x.column_default + in ( + "'{}'", + "'{}'::jsonb", + "'{}':::JSONB", + ), ] ), ) @@ -766,7 +781,11 @@ def test_db_column_name(self): x.data_type == "character varying", x.is_nullable == "NO", x.column_default - in ("''::character varying", "'':::STRING"), + in ( + "''", + "''::character varying", + "'':::STRING", + ), ] ), ) @@ -788,7 +807,11 @@ def test_db_column_name_initial(self): x.data_type == "character varying", x.is_nullable == "NO", x.column_default - in ("''::character varying", "'':::STRING"), + in ( + "''", + "''::character varying", + "'':::STRING", + ), ] ), ) diff --git a/tests/apps/migrations/auto/test_migration_manager.py b/tests/apps/migrations/auto/test_migration_manager.py index a1988a029..872b9b936 100644 --- a/tests/apps/migrations/auto/test_migration_manager.py +++ b/tests/apps/migrations/auto/test_migration_manager.py @@ -768,15 +768,15 @@ def test_alter_column_set_default_alt(self): ) asyncio.run(manager.run()) - self.assertEqual( - self._get_column_default(), - [{"column_default": "'Unknown':::STRING"}], + self.assertIn( + self._get_column_default()[0]["column_default"], + ["'Unknown'", "'Unknown':::STRING"], ) asyncio.run(manager.run(backwards=True)) - self.assertEqual( - self._get_column_default(), - [{"column_default": "'':::STRING"}], + self.assertIn( + self._get_column_default()[0]["column_default"], + ["''", "'':::STRING"], ) @engines_only("postgres") @@ -856,9 +856,9 @@ def test_alter_column_drop_default_alt(self): old_params={"default": None}, ) asyncio.run(manager_1.run()) - self.assertEqual( - self._get_column_default(), - [{"column_default": "'Mr Manager':::STRING"}], + self.assertIn( + self._get_column_default()[0]["column_default"], + ["'Mr Manager'", "'Mr Manager':::STRING"], ) # Drop the default. @@ -879,9 +879,9 @@ def test_alter_column_drop_default_alt(self): # And add it back once more to be sure. manager_3 = manager_1 asyncio.run(manager_3.run()) - self.assertEqual( - self._get_column_default(), - [{"column_default": "'Mr Manager':::STRING"}], + self.assertIn( + self._get_column_default()[0]["column_default"], + ["'Mr Manager'", "'Mr Manager':::STRING"], ) # Run them all backwards @@ -892,9 +892,9 @@ def test_alter_column_drop_default_alt(self): ) asyncio.run(manager_2.run(backwards=True)) - self.assertEqual( - self._get_column_default(), - [{"column_default": "'Mr Manager':::STRING"}], + self.assertIn( + self._get_column_default()[0]["column_default"], + ["'Mr Manager'", "'Mr Manager':::STRING"], ) asyncio.run(manager_1.run(backwards=True)) From 9d1ab2d492c26a677d689823f85d12303f548bfa Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 7 Jun 2024 21:48:46 +0100 Subject: [PATCH 572/727] 1010 Add support for `CAST` to convert between types (#1011) * add `cast` function * fix typo * revert accidental `test_array.py` changes they weren't meant to be in this branch --- docs/src/index.rst | 1 + docs/src/piccolo/functions/aggregate.rst | 34 +++++ docs/src/piccolo/functions/basic_usage.rst | 53 +++++++ docs/src/piccolo/functions/index.rst | 12 ++ docs/src/piccolo/functions/string.rst | 40 +++++ .../src/piccolo/functions/type_conversion.rst | 25 ++++ piccolo/columns/base.py | 4 + piccolo/query/functions/__init__.py | 2 + piccolo/query/functions/aggregate.py | 24 +-- piccolo/query/functions/type_conversion.py | 82 ++++++++++ piccolo/querystring.py | 12 ++ tests/query/test_functions.py | 140 +++++++++++++++++- 12 files changed, 415 insertions(+), 14 deletions(-) create mode 100644 docs/src/piccolo/functions/aggregate.rst create mode 100644 docs/src/piccolo/functions/basic_usage.rst create mode 100644 docs/src/piccolo/functions/index.rst create mode 100644 docs/src/piccolo/functions/string.rst create mode 100644 docs/src/piccolo/functions/type_conversion.rst create mode 100644 piccolo/query/functions/type_conversion.py diff --git a/docs/src/index.rst b/docs/src/index.rst index ef956ce49..20b48aa5f 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -16,6 +16,7 @@ batteries included. piccolo/getting_started/index piccolo/query_types/index piccolo/query_clauses/index + piccolo/functions/index piccolo/schema/index piccolo/projects_and_apps/index piccolo/engines/index diff --git a/docs/src/piccolo/functions/aggregate.rst b/docs/src/piccolo/functions/aggregate.rst new file mode 100644 index 000000000..3b1a95a4e --- /dev/null +++ b/docs/src/piccolo/functions/aggregate.rst @@ -0,0 +1,34 @@ +Aggregate functions +=================== + +.. currentmodule:: piccolo.query.functions.aggregate + +Avg +--- + +.. autoclass:: Avg + :class-doc-from: class + +Count +----- + +.. autoclass:: Count + :class-doc-from: class + +Min +--- + +.. autoclass:: Min + :class-doc-from: class + +Max +--- + +.. autoclass:: Max + :class-doc-from: class + +Sum +--- + +.. autoclass:: Sum + :class-doc-from: class diff --git a/docs/src/piccolo/functions/basic_usage.rst b/docs/src/piccolo/functions/basic_usage.rst new file mode 100644 index 000000000..de45883b0 --- /dev/null +++ b/docs/src/piccolo/functions/basic_usage.rst @@ -0,0 +1,53 @@ +Basic Usage +=========== + +Select queries +-------------- + +Functions can be used in ``select`` queries - here's an example, where we +convert the values to uppercase: + +.. code-block:: python + + >>> from piccolo.query.functions import Upper + + >>> await Band.select( + ... Upper(Band.name, alias="name") + ... ) + + [{"name": "PYTHONISTAS"}] + +Where clauses +------------- + +Functions can also be used in ``where`` clauses. + +.. code-block:: python + + >>> from piccolo.query.functions import Length + + >>> await Band.select( + ... Band.name + ... ).where( + ... Length(Band.name) > 10 + ... ) + + [{"name": "Pythonistas"}] + +Update queries +-------------- + +And even in ``update`` queries: + +.. code-block:: python + + >>> from piccolo.query.functions import Upper + + >>> await Band.update( + ... {Band.name: Upper(Band.name)}, + ... force=True + ... ).returning(Band.name) + + [{"name": "PYTHONISTAS"}, {"name": "RUSTACEANS"}, {"name": "C-SHARPS"}] + +Pretty much everywhere. diff --git a/docs/src/piccolo/functions/index.rst b/docs/src/piccolo/functions/index.rst new file mode 100644 index 000000000..d9a412bbc --- /dev/null +++ b/docs/src/piccolo/functions/index.rst @@ -0,0 +1,12 @@ +Functions +========= + +Functions can be used to modify how queries are run, and what is returned. + +.. toctree:: + :maxdepth: 1 + + ./basic_usage + ./string + ./type_conversion + ./aggregate diff --git a/docs/src/piccolo/functions/string.rst b/docs/src/piccolo/functions/string.rst new file mode 100644 index 000000000..dbc09125a --- /dev/null +++ b/docs/src/piccolo/functions/string.rst @@ -0,0 +1,40 @@ +String functions +================ + +.. currentmodule:: piccolo.query.functions.string + +Length +------ + +.. autoclass:: Length + :class-doc-from: class + +Lower +----- + +.. autoclass:: Lower + :class-doc-from: class + +Ltrim +----- + +.. autoclass:: Ltrim + :class-doc-from: class + +Reverse +------- + +.. autoclass:: Reverse + :class-doc-from: class + +Rtrim +----- + +.. autoclass:: Rtrim + :class-doc-from: class + +Upper +----- + +.. autoclass:: Upper + :class-doc-from: class diff --git a/docs/src/piccolo/functions/type_conversion.rst b/docs/src/piccolo/functions/type_conversion.rst new file mode 100644 index 000000000..7e2743d2b --- /dev/null +++ b/docs/src/piccolo/functions/type_conversion.rst @@ -0,0 +1,25 @@ +Type conversion functions +========================= + +Cast +---- + +.. currentmodule:: piccolo.query.functions.type_conversion + +.. autoclass:: Cast + +Notes on databases +------------------ + +Postgres and CockroachDB have very rich type systems, and you can convert +between most types. SQLite is more limited. + +The following query will work in Postgres / Cockroach, but you might get +unexpected results in SQLite, because it doesn't have a native ``TIME`` column +type: + +.. code-block:: python + + >>> from piccolo.columns import Time + >>> from piccolo.query.functions import Cast + >>> await Concert.select(Cast(Concert.starts, Time())) diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index ce452260c..2520502d6 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -201,6 +201,10 @@ def table(self) -> t.Type[Table]: ) return self._table + @table.setter + def table(self, value: t.Type[Table]): + self._table = value + ########################################################################### # Used by Foreign Keys: diff --git a/piccolo/query/functions/__init__.py b/piccolo/query/functions/__init__.py index d0195cc40..f7c841d0b 100644 --- a/piccolo/query/functions/__init__.py +++ b/piccolo/query/functions/__init__.py @@ -1,8 +1,10 @@ from .aggregate import Avg, Count, Max, Min, Sum from .string import Length, Lower, Ltrim, Reverse, Rtrim, Upper +from .type_conversion import Cast __all__ = ( "Avg", + "Cast", "Count", "Length", "Lower", diff --git a/piccolo/query/functions/aggregate.py b/piccolo/query/functions/aggregate.py index 61dd36a46..c50e557db 100644 --- a/piccolo/query/functions/aggregate.py +++ b/piccolo/query/functions/aggregate.py @@ -12,17 +12,17 @@ class Avg(Function): .. code-block:: python - await Band.select(Avg(Band.popularity)).run() + await Band.select(Avg(Band.popularity)) # We can use an alias. These two are equivalent: await Band.select( Avg(Band.popularity, alias="popularity_avg") - ).run() + ) await Band.select( Avg(Band.popularity).as_alias("popularity_avg") - ).run() + ) """ @@ -103,17 +103,17 @@ class Min(Function): .. code-block:: python - await Band.select(Min(Band.popularity)).run() + await Band.select(Min(Band.popularity)) # We can use an alias. These two are equivalent: await Band.select( Min(Band.popularity, alias="popularity_min") - ).run() + ) await Band.select( Min(Band.popularity).as_alias("popularity_min") - ).run() + ) """ @@ -128,17 +128,17 @@ class Max(Function): await Band.select( Max(Band.popularity) - ).run() + ) # We can use an alias. These two are equivalent: await Band.select( Max(Band.popularity, alias="popularity_max") - ).run() + ) await Band.select( Max(Band.popularity).as_alias("popularity_max") - ).run() + ) """ @@ -153,17 +153,17 @@ class Sum(Function): await Band.select( Sum(Band.popularity) - ).run() + ) # We can use an alias. These two are equivalent: await Band.select( Sum(Band.popularity, alias="popularity_sum") - ).run() + ) await Band.select( Sum(Band.popularity).as_alias("popularity_sum") - ).run() + ) """ diff --git a/piccolo/query/functions/type_conversion.py b/piccolo/query/functions/type_conversion.py new file mode 100644 index 000000000..e8f1dfee5 --- /dev/null +++ b/piccolo/query/functions/type_conversion.py @@ -0,0 +1,82 @@ +import typing as t + +from piccolo.columns.base import Column +from piccolo.querystring import QueryString + + +class Cast(QueryString): + def __init__( + self, + identifier: t.Union[Column, QueryString], + as_type: Column, + alias: t.Optional[str] = None, + ): + """ + Cast a value to a different type. For example:: + + >>> from piccolo.query.functions import Cast + + >>> await Concert.select( + ... Cast(Concert.starts, Time(), "start_time") + ... ) + [{"start_time": datetime.time(19, 0)}] + + :param identifier: + Identifies what is being converted (e.g. a column). + :param as_type: + The type to be converted to. + + """ + # Make sure the identifier is a supported type. + + if not isinstance(identifier, (Column, QueryString)): + raise ValueError( + "The identifier is an unsupported type - only Column and " + "QueryString instances are allowed." + ) + + ####################################################################### + # Convert `as_type` to a string which can be used in the query. + + if not isinstance(as_type, Column): + raise ValueError("The `as_type` value must be a Column instance.") + + # We need to give the column a reference to a table, and hence + # the database engine, as the column type is sometimes dependent + # on which database is being used. + from piccolo.table import Table, create_table_class + + table: t.Optional[t.Type[Table]] = None + + if isinstance(identifier, Column): + table = identifier._meta.table + elif isinstance(identifier, QueryString): + table = ( + identifier.columns[0]._meta.table + if identifier.columns + else None + ) + + as_type._meta.table = table or create_table_class("Table") + as_type_string = as_type.column_type + + ####################################################################### + # Preserve the original alias from the column. + + if isinstance(identifier, Column): + alias = ( + alias + or identifier._alias + or identifier._meta.get_default_alias() + ) + + ####################################################################### + + super().__init__( + f"CAST({{}} AS {as_type_string})", + identifier, + alias=alias, + ) + + +__all__ = ("Cast",) diff --git a/piccolo/querystring.py b/piccolo/querystring.py index 7f3f3e42a..c2fc0f80f 100644 --- a/piccolo/querystring.py +++ b/piccolo/querystring.py @@ -270,6 +270,18 @@ def __add__(self, value) -> QueryString: def __sub__(self, value) -> QueryString: return QueryString("{} - {}", self, value) + def __gt__(self, value) -> QueryString: + return QueryString("{} > {}", self, value) + + def __ge__(self, value) -> QueryString: + return QueryString("{} >= {}", self, value) + + def __lt__(self, value) -> QueryString: + return QueryString("{} < {}", self, value) + + def __le__(self, value) -> QueryString: + return QueryString("{} <= {}", self, value) + def is_in(self, value) -> QueryString: return QueryString("{} IN {}", self, value) diff --git a/tests/query/test_functions.py b/tests/query/test_functions.py index abe9a5f01..a970d9fa1 100644 --- a/tests/query/test_functions.py +++ b/tests/query/test_functions.py @@ -1,6 +1,7 @@ from unittest import TestCase -from piccolo.query.functions.string import Reverse, Upper +from piccolo.columns import Integer, Text, Varchar +from piccolo.query.functions import Cast, Length, Reverse, Upper from piccolo.querystring import QueryString from piccolo.table import create_db_tables_sync, drop_db_tables_sync from tests.base import engines_skip @@ -16,7 +17,13 @@ def setUp(self) -> None: manager = Manager({Manager.name: "Guido"}) manager.save().run_sync() - band = Band({Band.name: "Pythonistas", Band.manager: manager}) + band = Band( + { + Band.name: "Pythonistas", + Band.manager: manager, + Band.popularity: 1000, + } + ) band.save().run_sync() def tearDown(self) -> None: @@ -100,3 +107,132 @@ def test_where_with_joined_column(self): .run_sync() ) self.assertListEqual(response, [{"name": "Pythonistas"}]) + + +class TestCast(FunctionTest): + def test_varchar(self): + """ + Make sure that casting to ``Varchar`` works. + """ + response = Band.select( + Cast( + Band.popularity, + as_type=Varchar(), + ) + ).run_sync() + + self.assertListEqual( + response, + [{"popularity": "1000"}], + ) + + def test_text(self): + """ + Make sure that casting to ``Text`` works. + """ + response = Band.select( + Cast( + Band.popularity, + as_type=Text(), + ) + ).run_sync() + + self.assertListEqual( + response, + [{"popularity": "1000"}], + ) + + def test_integer(self): + """ + Make sure that casting to ``Integer`` works. + """ + Band.update({Band.name: "1111"}, force=True).run_sync() + + response = Band.select( + Cast( + Band.name, + as_type=Integer(), + ) + ).run_sync() + + self.assertListEqual( + response, + [{"name": 1111}], + ) + + def test_join(self): + """ + Make sure that casting works with joins. + """ + Manager.update({Manager.name: "1111"}, force=True).run_sync() + + response = Band.select( + Band.name, + Cast( + Band.manager.name, + as_type=Integer(), + ), + ).run_sync() + + self.assertListEqual( + response, + [ + { + "name": "Pythonistas", + "manager.name": 1111, + } + ], + ) + + def test_nested_inner(self): + """ + Make sure ``Cast`` can be passed into other functions. + """ + Band.update({Band.name: "1111"}, force=True).run_sync() + + response = Band.select( + Length( + Cast( + Band.popularity, + as_type=Varchar(), + ) + ) + ).run_sync() + + self.assertListEqual( + response, + [{"length": 4}], + ) + + def test_nested_outer(self): + """ + Make sure a querystring can be passed into ``Cast`` (meaning it can be + nested). + """ + response = Band.select( + Cast( + Length(Band.name), + as_type=Varchar(), + alias="length", + ) + ).run_sync() + + self.assertListEqual( + response, + [{"length": str(len("Pythonistas"))}], + ) + + def test_where_clause(self): + """ + Make sure ``Cast`` works in a where clause. + """ + response = ( + Band.select(Band.name, Band.popularity) + .where(Cast(Band.popularity, Varchar()) == "1000") + .run_sync() + ) + + self.assertListEqual( + response, + [{"name": "Pythonistas", "popularity": 1000}], + ) From 3e14d9c8c8a84e4fb40e1117ad51305207510574 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 7 Jun 2024 21:57:44 +0100 Subject: [PATCH 573/727] bumped version --- CHANGES.rst | 19 +++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index dd77fba43..dec4930a5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,25 @@ Changes ======= +1.8.0 +----- + +Added the ``Cast`` function, for performing type conversion. + +Here's an example, where we convert a ``timestamp`` to ``time``: + +.. code-block:: python + + >>> from piccolo.columns import Time + >>> from piccolo.query.functions import Cast + + >>> await Concert.select(Cast(Concert.starts, Time())) + [{'starts': datetime.time(19, 0)}] + +A new section was also added to the docs describing functions in more detail. + +------------------------------------------------------------------------------- + 1.7.0 ----- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 64888e7c6..a407d5275 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.7.0" +__VERSION__ = "1.8.0" From 28f871d0aa1f72f64d8d5ea1616ceee390a8750f Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 10 Jun 2024 18:18:14 +0100 Subject: [PATCH 574/727] 1005 CockroachDB - enable array tests (#1012) * start refactoring array tests, so they work with CockroachDB * mark slow array tests --- pyproject.toml | 3 +- scripts/test-cockroach.sh | 2 +- tests/columns/test_array.py | 82 ++++++++++++++++++++++++++++--------- 3 files changed, 66 insertions(+), 21 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ec6a0655e..eed50963a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,8 @@ ignore_missing_imports = true [tool.pytest.ini_options] markers = [ "integration", - "speed" + "speed", + "cockroach_array_slow" ] [tool.coverage.run] diff --git a/scripts/test-cockroach.sh b/scripts/test-cockroach.sh index 1c944aeae..9c8d23ca8 100755 --- a/scripts/test-cockroach.sh +++ b/scripts/test-cockroach.sh @@ -10,5 +10,5 @@ python3 -m pytest \ --cov-report=xml \ --cov-report=html \ --cov-fail-under=80 \ - -m "not integration" \ + -m "not integration and not cockroach_array_slow" \ -s $@ diff --git a/tests/columns/test_array.py b/tests/columns/test_array.py index 4677ef995..0b96e176c 100644 --- a/tests/columns/test_array.py +++ b/tests/columns/test_array.py @@ -1,6 +1,8 @@ import datetime from unittest import TestCase +import pytest + from piccolo.columns.column_types import ( Array, BigInt, @@ -10,8 +12,9 @@ Timestamp, Timestamptz, ) +from piccolo.querystring import QueryString from piccolo.table import Table -from tests.base import engines_only, sqlite_only +from tests.base import engines_only, engines_skip, sqlite_only class MyTable(Table): @@ -40,12 +43,18 @@ def setUp(self): def tearDown(self): MyTable.alter().drop_table().run_sync() - @engines_only("postgres", "sqlite") + @pytest.mark.cockroach_array_slow def test_storage(self): """ Make sure data can be stored and retrieved. - 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + In CockroachDB <= v22.2.0 we had this error: + + * https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + + In newer CockroachDB versions, it runs but is very slow: + + * https://github.com/piccolo-orm/piccolo/issues/1005 """ # noqa: E501 MyTable(value=[1, 2, 3]).save().run_sync() @@ -54,12 +63,19 @@ def test_storage(self): assert row is not None self.assertEqual(row.value, [1, 2, 3]) - @engines_only("postgres") + @engines_skip("sqlite") + @pytest.mark.cockroach_array_slow def test_index(self): """ Indexes should allow individual array elements to be queried. - 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + In CockroachDB <= v22.2.0 we had this error: + + * https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + + In newer CockroachDB versions, it runs but is very slow: + + * https://github.com/piccolo-orm/piccolo/issues/1005 """ # noqa: E501 MyTable(value=[1, 2, 3]).save().run_sync() @@ -68,66 +84,92 @@ def test_index(self): MyTable.select(MyTable.value[0]).first().run_sync(), {"value": 1} ) - @engines_only("postgres") + @engines_skip("sqlite") + @pytest.mark.cockroach_array_slow def test_all(self): """ Make sure rows can be retrieved where all items in an array match a given value. - 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + In CockroachDB <= v22.2.0 we had this error: + + * https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + + In newer CockroachDB versions, it runs but is very slow: + + * https://github.com/piccolo-orm/piccolo/issues/1005 """ # noqa: E501 MyTable(value=[1, 1, 1]).save().run_sync() + # We have to explicitly specify the type, so CockroachDB works. self.assertEqual( MyTable.select(MyTable.value) - .where(MyTable.value.all(1)) + .where(MyTable.value.all(QueryString("{}::INTEGER", 1))) .first() .run_sync(), {"value": [1, 1, 1]}, ) + # We have to explicitly specify the type, so CockroachDB works. self.assertEqual( MyTable.select(MyTable.value) - .where(MyTable.value.all(0)) + .where(MyTable.value.all(QueryString("{}::INTEGER", 0))) .first() .run_sync(), None, ) - @engines_only("postgres") + @engines_skip("sqlite") + @pytest.mark.cockroach_array_slow def test_any(self): """ Make sure rows can be retrieved where any items in an array match a given value. - 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + In CockroachDB <= v22.2.0 we had this error: + + * https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + + In newer CockroachDB versions, it runs but is very slow: + + * https://github.com/piccolo-orm/piccolo/issues/1005 """ # noqa: E501 + MyTable(value=[1, 2, 3]).save().run_sync() + # We have to explicitly specify the type, so CockroachDB works. self.assertEqual( MyTable.select(MyTable.value) - .where(MyTable.value.any(1)) + .where(MyTable.value.any(QueryString("{}::INTEGER", 1))) .first() .run_sync(), {"value": [1, 2, 3]}, ) + # We have to explicitly specify the type, so CockroachDB works. self.assertEqual( MyTable.select(MyTable.value) - .where(MyTable.value.any(0)) + .where(MyTable.value.any(QueryString("{}::INTEGER", 0))) .first() .run_sync(), None, ) - @engines_only("postgres") + @engines_skip("sqlite") + @pytest.mark.cockroach_array_slow def test_cat(self): """ Make sure values can be appended to an array. - 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + In CockroachDB <= v22.2.0 we had this error: + + * https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + + In newer CockroachDB versions, it runs but is very slow: + + * https://github.com/piccolo-orm/piccolo/issues/1005 """ # noqa: E501 MyTable(value=[1, 1, 1]).save().run_sync() @@ -137,7 +179,8 @@ def test_cat(self): ).run_sync() self.assertEqual( - MyTable.select().run_sync(), [{"id": 1, "value": [1, 1, 1, 2]}] + MyTable.select(MyTable.value).run_sync(), + [{"value": [1, 1, 1, 2]}], ) # Try plus symbol @@ -147,7 +190,8 @@ def test_cat(self): ).run_sync() self.assertEqual( - MyTable.select().run_sync(), [{"id": 1, "value": [1, 1, 1, 2, 3]}] + MyTable.select(MyTable.value).run_sync(), + [{"value": [1, 1, 1, 2, 3]}], ) # Make sure non-list values work @@ -157,8 +201,8 @@ def test_cat(self): ).run_sync() self.assertEqual( - MyTable.select().run_sync(), - [{"id": 1, "value": [1, 1, 1, 2, 3, 4]}], + MyTable.select(MyTable.value).run_sync(), + [{"value": [1, 1, 1, 2, 3, 4]}], ) @sqlite_only From cfb674a2d490b48143c7e07091bf1f091a6ca09c Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 10 Jun 2024 22:07:57 +0100 Subject: [PATCH 575/727] 1013 Add more operators to `QueryString` (#1014) * more quertstring operators * add modulus method * add tests * add more tests --- piccolo/querystring.py | 18 +++++ tests/query/test_querystring.py | 136 ++++++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+) diff --git a/piccolo/querystring.py b/piccolo/querystring.py index c2fc0f80f..1f7282a7d 100644 --- a/piccolo/querystring.py +++ b/piccolo/querystring.py @@ -282,12 +282,30 @@ def __lt__(self, value) -> QueryString: def __le__(self, value) -> QueryString: return QueryString("{} <= {}", self, value) + def __truediv__(self, value) -> QueryString: + return QueryString("{} / {}", self, value) + + def __mul__(self, value) -> QueryString: + return QueryString("{} * {}", self, value) + + def __pow__(self, value) -> QueryString: + return QueryString("{} ^ {}", self, value) + + def __mod__(self, value) -> QueryString: + return QueryString("{} % {}", self, value) + def is_in(self, value) -> QueryString: return QueryString("{} IN {}", self, value) def not_in(self, value) -> QueryString: return QueryString("{} NOT IN {}", self, value) + def like(self, value: str) -> QueryString: + return QueryString("{} LIKE {}", self, value) + + def ilike(self, value: str) -> QueryString: + return QueryString("{} ILIKE {}", self, value) + class Unquoted(QueryString): """ diff --git a/tests/query/test_querystring.py b/tests/query/test_querystring.py index 59631ab7f..58ca29495 100644 --- a/tests/query/test_querystring.py +++ b/tests/query/test_querystring.py @@ -1,6 +1,7 @@ from unittest import TestCase from piccolo.querystring import QueryString +from tests.base import postgres_only # TODO - add more extensive tests (increased nesting and argument count). @@ -28,3 +29,138 @@ def test_string(self): def test_querystring_with_no_args(self): qs = QueryString("SELECT name FROM band") self.assertEqual(qs.compile_string(), ("SELECT name FROM band", [])) + + +@postgres_only +class TestQueryStringOperators(TestCase): + """ + Make sure basic operations can be used on ``QueryString``. + """ + + def test_add(self): + query = QueryString("SELECT price") + 1 + self.assertIsInstance(query, QueryString) + self.assertEqual( + query.compile_string(), + ("SELECT price + $1", [1]), + ) + + def test_multiply(self): + query = QueryString("SELECT price") * 2 + self.assertIsInstance(query, QueryString) + self.assertEqual( + query.compile_string(), + ("SELECT price * $1", [2]), + ) + + def test_divide(self): + query = QueryString("SELECT price") / 1 + self.assertIsInstance(query, QueryString) + self.assertEqual( + query.compile_string(), + ("SELECT price / $1", [1]), + ) + + def test_power(self): + query = QueryString("SELECT price") ** 2 + self.assertIsInstance(query, QueryString) + self.assertEqual( + query.compile_string(), + ("SELECT price ^ $1", [2]), + ) + + def test_subtract(self): + query = QueryString("SELECT price") - 1 + self.assertIsInstance(query, QueryString) + self.assertEqual( + query.compile_string(), + ("SELECT price - $1", [1]), + ) + + def test_modulus(self): + query = QueryString("SELECT price") % 1 + self.assertIsInstance(query, QueryString) + self.assertEqual( + query.compile_string(), + ("SELECT price % $1", [1]), + ) + + def test_like(self): + query = QueryString("strip(name)").like("Python%") + self.assertIsInstance(query, QueryString) + self.assertEqual( + query.compile_string(), + ("strip(name) LIKE $1", ["Python%"]), + ) + + def test_ilike(self): + query = QueryString("strip(name)").ilike("Python%") + self.assertIsInstance(query, QueryString) + self.assertEqual( + query.compile_string(), + ("strip(name) ILIKE $1", ["Python%"]), + ) + + def test_greater_than(self): + query = QueryString("SELECT price") > 10 + self.assertIsInstance(query, QueryString) + self.assertEqual( + query.compile_string(), + ("SELECT price > $1", [10]), + ) + + def test_greater_equal_than(self): + query = QueryString("SELECT price") >= 10 + self.assertIsInstance(query, QueryString) + self.assertEqual( + query.compile_string(), + ("SELECT price >= $1", [10]), + ) + + def test_less_than(self): + query = QueryString("SELECT price") < 10 + self.assertIsInstance(query, QueryString) + self.assertEqual( + query.compile_string(), + ("SELECT price < $1", [10]), + ) + + def test_less_equal_than(self): + query = QueryString("SELECT price") <= 10 + self.assertIsInstance(query, QueryString) + self.assertEqual( + query.compile_string(), + ("SELECT price <= $1", [10]), + ) + + def test_equals(self): + query = QueryString("SELECT price") == 10 + self.assertIsInstance(query, QueryString) + self.assertEqual( + query.compile_string(), + ("SELECT price = $1", [10]), + ) + + def test_not_equals(self): + query = QueryString("SELECT price") != 10 + self.assertIsInstance(query, QueryString) + self.assertEqual( + query.compile_string(), + ("SELECT price != $1", [10]), + ) + + def test_is_in(self): + query = QueryString("SELECT price").is_in([10, 20, 30]) + self.assertIsInstance(query, QueryString) + self.assertEqual( + query.compile_string(), + ("SELECT price IN $1", [[10, 20, 30]]), + ) + + def test_not_in(self): + query = QueryString("SELECT price").not_in([10, 20, 30]) + self.assertIsInstance(query, QueryString) + self.assertEqual( + query.compile_string(), + ("SELECT price NOT IN $1", [[10, 20, 30]]), + ) From fb7a5ed7f74de3571a1c22df3c0f551850d27a42 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 13 Jun 2024 08:46:55 +0100 Subject: [PATCH 576/727] add basic math functions (#1016) --- docs/src/piccolo/functions/index.rst | 1 + docs/src/piccolo/functions/math.rst | 28 +++ piccolo/query/functions/__init__.py | 5 + piccolo/query/functions/math.py | 48 ++++ tests/query/functions/__init__.py | 0 tests/query/functions/base.py | 34 +++ tests/query/functions/test_functions.py | 64 +++++ tests/query/functions/test_math.py | 39 +++ tests/query/functions/test_string.py | 25 ++ tests/query/functions/test_type_conversion.py | 134 ++++++++++ tests/query/test_functions.py | 238 ------------------ 11 files changed, 378 insertions(+), 238 deletions(-) create mode 100644 docs/src/piccolo/functions/math.rst create mode 100644 piccolo/query/functions/math.py create mode 100644 tests/query/functions/__init__.py create mode 100644 tests/query/functions/base.py create mode 100644 tests/query/functions/test_functions.py create mode 100644 tests/query/functions/test_math.py create mode 100644 tests/query/functions/test_string.py create mode 100644 tests/query/functions/test_type_conversion.py delete mode 100644 tests/query/test_functions.py diff --git a/docs/src/piccolo/functions/index.rst b/docs/src/piccolo/functions/index.rst index d9a412bbc..93b3fe4f7 100644 --- a/docs/src/piccolo/functions/index.rst +++ b/docs/src/piccolo/functions/index.rst @@ -8,5 +8,6 @@ Functions can be used to modify how queries are run, and what is returned. ./basic_usage ./string + ./math ./type_conversion ./aggregate diff --git a/docs/src/piccolo/functions/math.rst b/docs/src/piccolo/functions/math.rst new file mode 100644 index 000000000..6b9472764 --- /dev/null +++ b/docs/src/piccolo/functions/math.rst @@ -0,0 +1,28 @@ +Math functions +============== + +.. currentmodule:: piccolo.query.functions.math + +Abs +--- + +.. autoclass:: Abs + :class-doc-from: class + +Ceil +---- + +.. autoclass:: Ceil + :class-doc-from: class + +Floor +----- + +.. autoclass:: Floor + :class-doc-from: class + +Round +----- + +.. autoclass:: Round + :class-doc-from: class diff --git a/piccolo/query/functions/__init__.py b/piccolo/query/functions/__init__.py index f7c841d0b..9b83eca7b 100644 --- a/piccolo/query/functions/__init__.py +++ b/piccolo/query/functions/__init__.py @@ -1,17 +1,22 @@ from .aggregate import Avg, Count, Max, Min, Sum +from .math import Abs, Ceil, Floor, Round from .string import Length, Lower, Ltrim, Reverse, Rtrim, Upper from .type_conversion import Cast __all__ = ( + "Abs", "Avg", "Cast", + "Ceil", "Count", + "Floor", "Length", "Lower", "Ltrim", "Max", "Min", "Reverse", + "Round", "Rtrim", "Sum", "Upper", diff --git a/piccolo/query/functions/math.py b/piccolo/query/functions/math.py new file mode 100644 index 000000000..e0ebaf70f --- /dev/null +++ b/piccolo/query/functions/math.py @@ -0,0 +1,48 @@ +""" +These functions mirror their counterparts in the Postgresql docs: + +https://www.postgresql.org/docs/current/functions-math.html + +""" + +from .base import Function + + +class Abs(Function): + """ + Absolute value. + """ + + function_name = "ABS" + + +class Ceil(Function): + """ + Nearest integer greater than or equal to argument. + """ + + function_name = "CEIL" + + +class Floor(Function): + """ + Nearest integer less than or equal to argument. + """ + + function_name = "FLOOR" + + +class Round(Function): + """ + Rounds to nearest integer. + """ + + function_name = "ROUND" + + +__all__ = ( + "Abs", + "Ceil", + "Floor", + "Round", +) diff --git a/tests/query/functions/__init__.py b/tests/query/functions/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/query/functions/base.py b/tests/query/functions/base.py new file mode 100644 index 000000000..168f5528b --- /dev/null +++ b/tests/query/functions/base.py @@ -0,0 +1,34 @@ +import typing as t +from unittest import TestCase + +from piccolo.table import Table, create_db_tables_sync, drop_db_tables_sync +from tests.example_apps.music.tables import Band, Manager + + +class FunctionTest(TestCase): + tables: t.List[t.Type[Table]] + + def setUp(self) -> None: + create_db_tables_sync(*self.tables) + + def tearDown(self) -> None: + drop_db_tables_sync(*self.tables) + + +class BandTest(FunctionTest): + tables = [Band, Manager] + + def setUp(self) -> None: + super().setUp() + + manager = Manager({Manager.name: "Guido"}) + manager.save().run_sync() + + band = Band( + { + Band.name: "Pythonistas", + Band.manager: manager, + Band.popularity: 1000, + } + ) + band.save().run_sync() diff --git a/tests/query/functions/test_functions.py b/tests/query/functions/test_functions.py new file mode 100644 index 000000000..cb306dcc4 --- /dev/null +++ b/tests/query/functions/test_functions.py @@ -0,0 +1,64 @@ +from piccolo.query.functions import Reverse, Upper +from piccolo.querystring import QueryString +from tests.base import engines_skip +from tests.example_apps.music.tables import Band + +from .base import BandTest + + +@engines_skip("sqlite") +class TestNested(BandTest): + """ + Skip the the test for SQLite, as it doesn't support ``Reverse``. + """ + + def test_nested(self): + """ + Make sure we can nest functions. + """ + response = Band.select(Upper(Reverse(Band.name))).run_sync() + self.assertListEqual(response, [{"upper": "SATSINOHTYP"}]) + + def test_nested_with_joined_column(self): + """ + Make sure nested functions can be used on a column from a joined table. + """ + response = Band.select(Upper(Reverse(Band.manager._.name))).run_sync() + self.assertListEqual(response, [{"upper": "ODIUG"}]) + + def test_nested_within_querystring(self): + """ + If we wrap a function in a custom QueryString - make sure the columns + are still accessible, so joins are successful. + """ + response = Band.select( + QueryString("CONCAT({}, '!')", Upper(Band.manager._.name)), + ).run_sync() + + self.assertListEqual(response, [{"concat": "GUIDO!"}]) + + +class TestWhereClause(BandTest): + + def test_where(self): + """ + Make sure where clauses work with functions. + """ + response = ( + Band.select(Band.name) + .where(Upper(Band.name) == "PYTHONISTAS") + .run_sync() + ) + self.assertListEqual(response, [{"name": "Pythonistas"}]) + + def test_where_with_joined_column(self): + """ + Make sure where clauses work with functions, when a joined column is + used. + """ + response = ( + Band.select(Band.name) + .where(Upper(Band.manager._.name) == "GUIDO") + .run_sync() + ) + self.assertListEqual(response, [{"name": "Pythonistas"}]) diff --git a/tests/query/functions/test_math.py b/tests/query/functions/test_math.py new file mode 100644 index 000000000..1c82f9426 --- /dev/null +++ b/tests/query/functions/test_math.py @@ -0,0 +1,39 @@ +import decimal + +from piccolo.columns import Numeric +from piccolo.query.functions.math import Abs, Ceil, Floor, Round +from piccolo.table import Table + +from .base import FunctionTest + + +class Ticket(Table): + price = Numeric(digits=(5, 2)) + + +class TestMath(FunctionTest): + + tables = [Ticket] + + def setUp(self): + super().setUp() + self.ticket = Ticket({Ticket.price: decimal.Decimal("36.50")}) + self.ticket.save().run_sync() + + def test_floor(self): + response = Ticket.select(Floor(Ticket.price, alias="price")).run_sync() + self.assertListEqual(response, [{"price": decimal.Decimal("36.00")}]) + + def test_ceil(self): + response = Ticket.select(Ceil(Ticket.price, alias="price")).run_sync() + self.assertListEqual(response, [{"price": decimal.Decimal("37.00")}]) + + def test_abs(self): + self.ticket.price = decimal.Decimal("-1.50") + self.ticket.save().run_sync() + response = Ticket.select(Abs(Ticket.price, alias="price")).run_sync() + self.assertListEqual(response, [{"price": decimal.Decimal("1.50")}]) + + def test_round(self): + response = Ticket.select(Round(Ticket.price, alias="price")).run_sync() + self.assertListEqual(response, [{"price": decimal.Decimal("37.00")}]) diff --git a/tests/query/functions/test_string.py b/tests/query/functions/test_string.py new file mode 100644 index 000000000..b87952634 --- /dev/null +++ b/tests/query/functions/test_string.py @@ -0,0 +1,25 @@ +from piccolo.query.functions.string import Upper +from tests.example_apps.music.tables import Band + +from .base import BandTest + + +class TestUpperFunction(BandTest): + + def test_column(self): + """ + Make sure we can uppercase a column's value. + """ + response = Band.select(Upper(Band.name)).run_sync() + self.assertListEqual(response, [{"upper": "PYTHONISTAS"}]) + + def test_alias(self): + response = Band.select(Upper(Band.name, alias="name")).run_sync() + self.assertListEqual(response, [{"name": "PYTHONISTAS"}]) + + def test_joined_column(self): + """ + Make sure we can uppercase a column's value from a joined table. + """ + response = Band.select(Upper(Band.manager._.name)).run_sync() + self.assertListEqual(response, [{"upper": "GUIDO"}]) diff --git a/tests/query/functions/test_type_conversion.py b/tests/query/functions/test_type_conversion.py new file mode 100644 index 000000000..598d9d37c --- /dev/null +++ b/tests/query/functions/test_type_conversion.py @@ -0,0 +1,134 @@ +from piccolo.columns import Integer, Text, Varchar +from piccolo.query.functions import Cast, Length +from tests.example_apps.music.tables import Band, Manager + +from .base import BandTest + + +class TestCast(BandTest): + def test_varchar(self): + """ + Make sure that casting to ``Varchar`` works. + """ + response = Band.select( + Cast( + Band.popularity, + as_type=Varchar(), + ) + ).run_sync() + + self.assertListEqual( + response, + [{"popularity": "1000"}], + ) + + def test_text(self): + """ + Make sure that casting to ``Text`` works. + """ + response = Band.select( + Cast( + Band.popularity, + as_type=Text(), + ) + ).run_sync() + + self.assertListEqual( + response, + [{"popularity": "1000"}], + ) + + def test_integer(self): + """ + Make sure that casting to ``Integer`` works. + """ + Band.update({Band.name: "1111"}, force=True).run_sync() + + response = Band.select( + Cast( + Band.name, + as_type=Integer(), + ) + ).run_sync() + + self.assertListEqual( + response, + [{"name": 1111}], + ) + + def test_join(self): + """ + Make sure that casting works with joins. + """ + Manager.update({Manager.name: "1111"}, force=True).run_sync() + + response = Band.select( + Band.name, + Cast( + Band.manager.name, + as_type=Integer(), + ), + ).run_sync() + + self.assertListEqual( + response, + [ + { + "name": "Pythonistas", + "manager.name": 1111, + } + ], + ) + + def test_nested_inner(self): + """ + Make sure ``Cast`` can be passed into other functions. + """ + Band.update({Band.name: "1111"}, force=True).run_sync() + + response = Band.select( + Length( + Cast( + Band.popularity, + as_type=Varchar(), + ) + ) + ).run_sync() + + self.assertListEqual( + response, + [{"length": 4}], + ) + + def test_nested_outer(self): + """ + Make sure a querystring can be passed into ``Cast`` (meaning it can be + nested). + """ + response = Band.select( + Cast( + Length(Band.name), + as_type=Varchar(), + alias="length", + ) + ).run_sync() + + self.assertListEqual( + response, + [{"length": str(len("Pythonistas"))}], + ) + + def test_where_clause(self): + """ + Make sure ``Cast`` works in a where clause. + """ + response = ( + Band.select(Band.name, Band.popularity) + .where(Cast(Band.popularity, Varchar()) == "1000") + .run_sync() + ) + + self.assertListEqual( + response, + [{"name": "Pythonistas", "popularity": 1000}], + ) diff --git a/tests/query/test_functions.py b/tests/query/test_functions.py deleted file mode 100644 index a970d9fa1..000000000 --- a/tests/query/test_functions.py +++ /dev/null @@ -1,238 +0,0 @@ -from unittest import TestCase - -from piccolo.columns import Integer, Text, Varchar -from piccolo.query.functions import Cast, Length, Reverse, Upper -from piccolo.querystring import QueryString -from piccolo.table import create_db_tables_sync, drop_db_tables_sync -from tests.base import engines_skip -from tests.example_apps.music.tables import Band, Manager - - -class FunctionTest(TestCase): - tables = (Band, Manager) - - def setUp(self) -> None: - create_db_tables_sync(*self.tables) - - manager = Manager({Manager.name: "Guido"}) - manager.save().run_sync() - - band = Band( - { - Band.name: "Pythonistas", - Band.manager: manager, - Band.popularity: 1000, - } - ) - band.save().run_sync() - - def tearDown(self) -> None: - drop_db_tables_sync(*self.tables) - - -class TestUpperFunction(FunctionTest): - - def test_column(self): - """ - Make sure we can uppercase a column's value. - """ - response = Band.select(Upper(Band.name)).run_sync() - self.assertListEqual(response, [{"upper": "PYTHONISTAS"}]) - - def test_alias(self): - response = Band.select(Upper(Band.name, alias="name")).run_sync() - self.assertListEqual(response, [{"name": "PYTHONISTAS"}]) - - def test_joined_column(self): - """ - Make sure we can uppercase a column's value from a joined table. - """ - response = Band.select(Upper(Band.manager._.name)).run_sync() - self.assertListEqual(response, [{"upper": "GUIDO"}]) - - -@engines_skip("sqlite") -class TestNested(FunctionTest): - """ - Skip the the test for SQLite, as it doesn't support ``Reverse``. - """ - - def test_nested(self): - """ - Make sure we can nest functions. - """ - response = Band.select(Upper(Reverse(Band.name))).run_sync() - self.assertListEqual(response, [{"upper": "SATSINOHTYP"}]) - - def test_nested_with_joined_column(self): - """ - Make sure nested functions can be used on a column from a joined table. - """ - response = Band.select(Upper(Reverse(Band.manager._.name))).run_sync() - self.assertListEqual(response, [{"upper": "ODIUG"}]) - - def test_nested_within_querystring(self): - """ - If we wrap a function in a custom QueryString - make sure the columns - are still accessible, so joins are successful. - """ - response = Band.select( - QueryString("CONCAT({}, '!')", Upper(Band.manager._.name)), - ).run_sync() - - self.assertListEqual(response, [{"concat": "GUIDO!"}]) - - -class TestWhereClause(FunctionTest): - - def test_where(self): - """ - Make sure where clauses work with functions. - """ - response = ( - Band.select(Band.name) - .where(Upper(Band.name) == "PYTHONISTAS") - .run_sync() - ) - self.assertListEqual(response, [{"name": "Pythonistas"}]) - - def test_where_with_joined_column(self): - """ - Make sure where clauses work with functions, when a joined column is - used. - """ - response = ( - Band.select(Band.name) - .where(Upper(Band.manager._.name) == "GUIDO") - .run_sync() - ) - self.assertListEqual(response, [{"name": "Pythonistas"}]) - - -class TestCast(FunctionTest): - def test_varchar(self): - """ - Make sure that casting to ``Varchar`` works. - """ - response = Band.select( - Cast( - Band.popularity, - as_type=Varchar(), - ) - ).run_sync() - - self.assertListEqual( - response, - [{"popularity": "1000"}], - ) - - def test_text(self): - """ - Make sure that casting to ``Text`` works. - """ - response = Band.select( - Cast( - Band.popularity, - as_type=Text(), - ) - ).run_sync() - - self.assertListEqual( - response, - [{"popularity": "1000"}], - ) - - def test_integer(self): - """ - Make sure that casting to ``Integer`` works. - """ - Band.update({Band.name: "1111"}, force=True).run_sync() - - response = Band.select( - Cast( - Band.name, - as_type=Integer(), - ) - ).run_sync() - - self.assertListEqual( - response, - [{"name": 1111}], - ) - - def test_join(self): - """ - Make sure that casting works with joins. - """ - Manager.update({Manager.name: "1111"}, force=True).run_sync() - - response = Band.select( - Band.name, - Cast( - Band.manager.name, - as_type=Integer(), - ), - ).run_sync() - - self.assertListEqual( - response, - [ - { - "name": "Pythonistas", - "manager.name": 1111, - } - ], - ) - - def test_nested_inner(self): - """ - Make sure ``Cast`` can be passed into other functions. - """ - Band.update({Band.name: "1111"}, force=True).run_sync() - - response = Band.select( - Length( - Cast( - Band.popularity, - as_type=Varchar(), - ) - ) - ).run_sync() - - self.assertListEqual( - response, - [{"length": 4}], - ) - - def test_nested_outer(self): - """ - Make sure a querystring can be passed into ``Cast`` (meaning it can be - nested). - """ - response = Band.select( - Cast( - Length(Band.name), - as_type=Varchar(), - alias="length", - ) - ).run_sync() - - self.assertListEqual( - response, - [{"length": str(len("Pythonistas"))}], - ) - - def test_where_clause(self): - """ - Make sure ``Cast`` works in a where clause. - """ - response = ( - Band.select(Band.name, Band.popularity) - .where(Cast(Band.popularity, Varchar()) == "1000") - .run_sync() - ) - - self.assertListEqual( - response, - [{"name": "Pythonistas", "popularity": 1000}], - ) From 70dac99c1040d6b1ec09a13e01caed06b7443d09 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 13 Jun 2024 12:23:45 +0100 Subject: [PATCH 577/727] 1017 Improve array serialisation in `get_sql_value` (#1018) * fix arrays * make multidimensional arrays work, and SQLite edgecases * add tests * fix sqlite tests on certain Python versions --- piccolo/columns/base.py | 84 +++++++++++++++++++---------- tests/columns/test_get_sql_value.py | 66 +++++++++++++++++++++++ 2 files changed, 121 insertions(+), 29 deletions(-) create mode 100644 tests/columns/test_get_sql_value.py diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 2520502d6..9d3e2b1cc 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -830,7 +830,11 @@ def get_where_string(self, engine_type: str) -> QueryString: engine_type=engine_type, with_alias=False ) - def get_sql_value(self, value: t.Any) -> t.Any: + def get_sql_value( + self, + value: t.Any, + delimiter: str = "'", + ) -> str: """ When using DDL statements, we can't parameterise the values. An example is when setting the default for a column. So we have to convert from @@ -839,11 +843,18 @@ def get_sql_value(self, value: t.Any) -> t.Any: :param value: The Python value to convert to a string usable in a DDL statement - e.g. 1. + e.g. ``1``. + :param delimiter: + The string returned by this function is wrapped in delimiters, + ready to be added to a DDL statement. For example: + ``'hello world'``. :returns: - The string usable in the DDL statement e.g. '1'. + The string usable in the DDL statement e.g. ``'1'``. """ + from piccolo.engine.sqlite import ADAPTERS as sqlite_adapters + + # Common across all DB engines if isinstance(value, Default): return getattr(value, self._meta.engine_type) elif value is None: @@ -851,37 +862,52 @@ def get_sql_value(self, value: t.Any) -> t.Any: elif isinstance(value, (float, decimal.Decimal)): return str(value) elif isinstance(value, str): - return f"'{value}'" + return f"{delimiter}{value}{delimiter}" elif isinstance(value, bool): return str(value).lower() - elif isinstance(value, datetime.datetime): - return f"'{value.isoformat().replace('T', ' ')}'" - elif isinstance(value, datetime.date): - return f"'{value.isoformat()}'" - elif isinstance(value, datetime.time): - return f"'{value.isoformat()}'" - elif isinstance(value, datetime.timedelta): - interval = IntervalCustom.from_timedelta(value) - return getattr(interval, self._meta.engine_type) elif isinstance(value, bytes): - return f"'{value.hex()}'" - elif isinstance(value, uuid.UUID): - return f"'{value}'" - elif isinstance(value, list): - # Convert to the array syntax. - return ( - "'{" - + ", ".join( - ( - f'"{i}"' - if isinstance(i, str) - else str(self.get_sql_value(i)) + return f"{delimiter}{value.hex()}{delimiter}" + + # SQLite specific + if self._meta.engine_type == "sqlite": + if adapter := sqlite_adapters.get(type(value)): + sqlite_value = adapter(value) + return ( + f"{delimiter}{sqlite_value}{delimiter}" + if isinstance(sqlite_value, str) + else sqlite_value + ) + + # Postgres and Cockroach + if self._meta.engine_type in ["postgres", "cockroach"]: + if isinstance(value, datetime.datetime): + return f"{delimiter}{value.isoformat().replace('T', ' ')}{delimiter}" # noqa: E501 + elif isinstance(value, datetime.date): + return f"{delimiter}{value.isoformat()}{delimiter}" + elif isinstance(value, datetime.time): + return f"{delimiter}{value.isoformat()}{delimiter}" + elif isinstance(value, datetime.timedelta): + interval = IntervalCustom.from_timedelta(value) + return getattr(interval, self._meta.engine_type) + elif isinstance(value, uuid.UUID): + return f"{delimiter}{value}{delimiter}" + elif isinstance(value, list): + # Convert to the array syntax. + return ( + delimiter + + "{" + + ",".join( + self.get_sql_value( + i, + delimiter="" if isinstance(i, list) else '"', + ) + for i in value ) - for i in value + + "}" + + delimiter ) - ) + "}'" - else: - return value + + return str(value) @property def column_type(self): diff --git a/tests/columns/test_get_sql_value.py b/tests/columns/test_get_sql_value.py new file mode 100644 index 000000000..9a5d1c7d8 --- /dev/null +++ b/tests/columns/test_get_sql_value.py @@ -0,0 +1,66 @@ +import datetime +from unittest import TestCase + +from tests.base import engines_only +from tests.example_apps.music.tables import Band + + +@engines_only("postgres", "cockroach") +class TestArrayPostgres(TestCase): + + def test_string(self): + self.assertEqual( + Band.name.get_sql_value(["a", "b", "c"]), + '\'{"a","b","c"}\'', + ) + + def test_int(self): + self.assertEqual( + Band.name.get_sql_value([1, 2, 3]), + "'{1,2,3}'", + ) + + def test_nested(self): + self.assertEqual( + Band.name.get_sql_value([1, 2, 3, [4, 5, 6]]), + "'{1,2,3,{4,5,6}}'", + ) + + def test_time(self): + self.assertEqual( + Band.name.get_sql_value([datetime.time(hour=8, minute=0)]), + "'{\"08:00:00\"}'", + ) + + +@engines_only("sqlite") +class TestArraySQLite(TestCase): + """ + Note, we use ``.replace(" ", "")`` because we serialise arrays using + Python's json library, and there is inconsistency between Python versions + (some output ``["a", "b", "c"]``, and others ``["a","b","c"]``). + """ + + def test_string(self): + self.assertEqual( + Band.name.get_sql_value(["a", "b", "c"]).replace(" ", ""), + '\'["a","b","c"]\'', + ) + + def test_int(self): + self.assertEqual( + Band.name.get_sql_value([1, 2, 3]).replace(" ", ""), + "'[1,2,3]'", + ) + + def test_nested(self): + self.assertEqual( + Band.name.get_sql_value([1, 2, 3, [4, 5, 6]]).replace(" ", ""), + "'[1,2,3,[4,5,6]]'", + ) + + def test_time(self): + self.assertEqual( + Band.name.get_sql_value([datetime.time(hour=8, minute=0)]), + "'[\"08:00:00\"]'", + ) From 157bb13816219c65baf7a0838cdf52544f2b1533 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 13 Jun 2024 13:03:43 +0100 Subject: [PATCH 578/727] Replace deprecated @abstractproperty (#1020) --- piccolo/columns/defaults/base.py | 8 +++++--- piccolo/schema.py | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/piccolo/columns/defaults/base.py b/piccolo/columns/defaults/base.py index eceb6f0f1..9ef45ec93 100644 --- a/piccolo/columns/defaults/base.py +++ b/piccolo/columns/defaults/base.py @@ -1,17 +1,19 @@ from __future__ import annotations import typing as t -from abc import ABC, abstractmethod, abstractproperty +from abc import ABC, abstractmethod from piccolo.utils.repr import repr_class_instance class Default(ABC): - @abstractproperty + @property + @abstractmethod def postgres(self) -> str: pass - @abstractproperty + @property + @abstractmethod def sqlite(self) -> str: pass diff --git a/piccolo/schema.py b/piccolo/schema.py index 7899238d5..ef0bd6ab4 100644 --- a/piccolo/schema.py +++ b/piccolo/schema.py @@ -13,7 +13,8 @@ class SchemaDDLBase(abc.ABC): db: Engine - @abc.abstractproperty + @property + @abc.abstractmethod def ddl(self) -> str: pass From 5859917e572e0ee37656f5e2092b9b7434c7ba7c Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 13 Jun 2024 13:14:10 +0100 Subject: [PATCH 579/727] bumped version --- CHANGES.rst | 35 +++++++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index dec4930a5..8d0790f25 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,41 @@ Changes ======= +1.9.0 +----- + +Added some math functions, for example ``Abs``, ``Ceil``, ``Floor`` and +``Round``. + +.. code-block:: python + + >>> from piccolo.query.functions import Round + >>> await Ticket.select(Round(Ticket.price, alias="price")) + [{'price': 50.0}] + +Added more operators to ``QueryString`` (multiply, divide, modulus, power), so +we can do things like: + +.. code-block:: python + + >>> await Ticket.select(Round(Ticket.price) * 2) + [{'price': 100.0}] + +Fixed some edge cases around defaults for ``Array`` columns. + +.. code-block:: python + + def get_default(): + # This used to fail: + return [datetime.time(hour=8, minute=0)] + + class MyTable(Table): + times = Array(Time(), default=get_default) + +Fixed some deprecation warnings, and improved CockroachDB array tests. + +------------------------------------------------------------------------------- + 1.8.0 ----- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index a407d5275..26de05016 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.8.0" +__VERSION__ = "1.9.0" From 2b79717d4127886e64a0ccbaffc2cc5cd135141e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 14 Jun 2024 10:06:59 +0100 Subject: [PATCH 580/727] add `not_any` method to `Array` column (#1022) --- docs/src/piccolo/schema/column_types.rst | 6 +++++ piccolo/columns/column_types.py | 24 +++++++++++++++++++- piccolo/columns/operators/comparison.py | 4 ++++ tests/columns/test_array.py | 28 ++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 1 deletion(-) diff --git a/docs/src/piccolo/schema/column_types.rst b/docs/src/piccolo/schema/column_types.rst index 4b7fd1315..37484978a 100644 --- a/docs/src/piccolo/schema/column_types.rst +++ b/docs/src/piccolo/schema/column_types.rst @@ -335,6 +335,12 @@ any .. automethod:: Array.any +======= +not_any +======= + +.. automethod:: Array.not_any + === all === diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index e16ffe6ab..c248d3b1b 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -57,7 +57,11 @@ class Band(Table): TimestamptzNow, ) from piccolo.columns.defaults.uuid import UUID4, UUIDArg -from piccolo.columns.operators.comparison import ArrayAll, ArrayAny +from piccolo.columns.operators.comparison import ( + ArrayAll, + ArrayAny, + ArrayNotAny, +) from piccolo.columns.operators.string import Concat from piccolo.columns.reference import LazyTableReference from piccolo.querystring import QueryString @@ -2670,6 +2674,24 @@ def any(self, value: t.Any) -> Where: else: raise ValueError("Unrecognised engine type") + def not_any(self, value: t.Any) -> Where: + """ + Check if the given value isn't in the array. + + .. code-block:: python + + >>> await Ticket.select().where(Ticket.seat_numbers.not_any(510)) + + """ + engine_type = self._meta.engine_type + + if engine_type in ("postgres", "cockroach"): + return Where(column=self, value=value, operator=ArrayNotAny) + elif engine_type == "sqlite": + return self.not_like(f"%{value}%") + else: + raise ValueError("Unrecognised engine type") + def all(self, value: t.Any) -> Where: """ Check if all of the items in the array match the given value. diff --git a/piccolo/columns/operators/comparison.py b/piccolo/columns/operators/comparison.py index 2811c3836..91b565361 100644 --- a/piccolo/columns/operators/comparison.py +++ b/piccolo/columns/operators/comparison.py @@ -62,5 +62,9 @@ class ArrayAny(ComparisonOperator): template = "{value} = ANY ({name})" +class ArrayNotAny(ComparisonOperator): + template = "NOT {value} = ANY ({name})" + + class ArrayAll(ComparisonOperator): template = "{value} = ALL ({name})" diff --git a/tests/columns/test_array.py b/tests/columns/test_array.py index 0b96e176c..604aecd51 100644 --- a/tests/columns/test_array.py +++ b/tests/columns/test_array.py @@ -157,6 +157,34 @@ def test_any(self): None, ) + @engines_skip("sqlite") + @pytest.mark.cockroach_array_slow + def test_not_any(self): + """ + Make sure rows can be retrieved where the array doesn't contain a + certain value. + + In CockroachDB <= v22.2.0 we had this error: + + * https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + + In newer CockroachDB versions, it runs but is very slow: + + * https://github.com/piccolo-orm/piccolo/issues/1005 + + """ # noqa: E501 + + MyTable(value=[1, 2, 3]).save().run_sync() + MyTable(value=[4, 5, 6]).save().run_sync() + + # We have to explicitly specify the type, so CockroachDB works. + self.assertEqual( + MyTable.select(MyTable.value) + .where(MyTable.value.not_any(QueryString("{}::INTEGER", 4))) + .run_sync(), + [{"value": [1, 2, 3]}], + ) + @engines_skip("sqlite") @pytest.mark.cockroach_array_slow def test_cat(self): From e1550b43c06c8d7d0a3dcdb3c8891b0d7ec9629f Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 14 Jun 2024 12:49:16 +0100 Subject: [PATCH 581/727] Pylance fixes (#906) * add `id: Serial` * replace `pass` with `... ` in abstract properties * improve types for Postgres' `AsyncBatch` * improve types for SQLite's `AsyncBatch` * ignore deliberate type errors * add assertions for nullable values * fix type annotation for `test_function` in `MigrationTestCase` * more null assertions * make sure the error code is a string * add `id: Serial` to a table * fix iPython import path * add type annotations to `COLUMN_DEFAULT_PARSER` * add pyright to CI * check that engine is `PostgresEngine` * ignore deliberate type error * more null assertions * more null assertions * fix database checks * ignore unbound value errors * make sure `Engine` has `log_queries` and `log_responses` values * remove test cases which don't match literal values * null assertions * ignore deliberate type error * add `id: Serial` annotations * fix type annotation for `Time` default value allow it to receive a function which returns a `time` value * fix type annotation for `Date` offset It can accept a function which returns a `date` value * fix type annotation for `Timestamp` default * fix remaining errors in `test_migrations.py` * refactor `test_add_column` * add test methods to `M2MBase` * ignore some type errors * fix pydantic types * fix type warnings with engines * fix `fixtures/commands/load.py` warnings * fix `schema.py` warnings * fix `table.py` warnings * make assertion more explicit * refactor `M2MBase` * fix `test_m2m.py` warnings * fixes for `migration_manager.py` * ignore some errors * foreign key improvements * fix formatting * fixes for `test_join.py` and `test_objects.py` * fix remaining issues in `test_join.py` * ignore deliberate error in `test_slots.py` * fix issues in `test_all_related.py` * fix errors in `test_pydantic.py` * fix errors in `schema.py` * fix errors in `test_select.py` * fix errors in `test_get_related_readable.py` * fix errors in `test_all_columns.py` * fix errors in `user/test_tables.py` * fix errors in `test_indexes.py` * fix errors in `select.py` * remove unused `Self` * fix errors with `M2MBase` * fix errors in `test_foreign_key_self.py` * fix errors in `test_primary_key.py` * fix errors with `m2m.py` * fix errors with `test_objects.py` * ignore errors with update * fix migration errors with `migrations_folder_path` * fix array errors * added `resolved_migrations_folder_path` * try fixing `Batch` * add an extra overload for `output` * fix errors in `WhereDelegate` and `AddDelegate` * fix errors in `test_attribute_access.py` * fix errors in `test_schema.py` * use `resolved_migrations_folder_path` in `test_apps.py` * fix errors in `test_columns_delegate.py` * ignore error in `test_output.py` * add explicit exports to `migrations/auto/__init__.py` * improve `atomic` * disable pyright in linter for now * reformat with latest black * remove unused import * remove duplicate import * fix test * rename `Batch` to `BaseBatch` * upgrade pyright and move to a separate script * fix linter warnings * remove some unnecessary type casting * remove unnecessary type casts * use `pass` for consistency with other methods in class * fix some more pyright warnings * fix pyright warnings for `BaseBatch. __aenter__` * fix `BaseTransaction. __aexit__` return type * check length of self.columns in M2MSelect --- piccolo/apps/fixtures/commands/load.py | 2 +- piccolo/apps/migrations/auto/__init__.py | 8 ++ .../apps/migrations/auto/migration_manager.py | 3 +- piccolo/apps/migrations/commands/backwards.py | 4 +- piccolo/apps/migrations/commands/base.py | 2 +- piccolo/apps/migrations/commands/check.py | 2 +- piccolo/apps/migrations/commands/clean.py | 2 +- piccolo/apps/migrations/commands/forwards.py | 4 +- piccolo/apps/migrations/commands/new.py | 6 +- piccolo/apps/schema/commands/generate.py | 4 +- piccolo/apps/shell/commands/run.py | 2 +- piccolo/columns/column_types.py | 8 +- piccolo/columns/defaults/base.py | 2 +- piccolo/columns/defaults/date.py | 10 +- piccolo/columns/defaults/interval.py | 1 + piccolo/columns/defaults/time.py | 10 +- piccolo/columns/defaults/timestamp.py | 1 + piccolo/columns/defaults/uuid.py | 2 +- piccolo/columns/m2m.py | 14 +-- piccolo/conf/apps.py | 13 ++- piccolo/engine/base.py | 89 +++++++++++++---- piccolo/engine/cockroach.py | 5 +- piccolo/engine/postgres.py | 52 ++++++---- piccolo/engine/sqlite.py | 49 ++++----- piccolo/query/methods/create_index.py | 2 +- piccolo/query/methods/drop_index.py | 2 +- piccolo/query/methods/objects.py | 14 +-- piccolo/query/methods/select.py | 20 ++-- piccolo/query/mixins.py | 13 +-- piccolo/schema.py | 29 +++--- piccolo/table.py | 43 ++++---- piccolo/utils/encoding.py | 8 +- requirements/dev-requirements.txt | 1 + scripts/pyright.sh | 14 +++ .../auto/integration/test_migrations.py | 2 +- tests/conf/test_apps.py | 2 +- tests/engine/test_nested_transaction.py | 2 + tests/engine/test_transaction.py | 3 +- tests/table/test_indexes.py | 6 +- tests/utils/test_pydantic.py | 99 +++++++++++++------ 40 files changed, 365 insertions(+), 190 deletions(-) create mode 100755 scripts/pyright.sh diff --git a/piccolo/apps/fixtures/commands/load.py b/piccolo/apps/fixtures/commands/load.py index 1de1d5a44..64e4c3334 100644 --- a/piccolo/apps/fixtures/commands/load.py +++ b/piccolo/apps/fixtures/commands/load.py @@ -51,7 +51,7 @@ async def load_json_string( finder = Finder() engine = engine_finder() - if not engine: + if engine is None: raise Exception("Unable to find the engine.") # This is what we want to the insert into the database: diff --git a/piccolo/apps/migrations/auto/__init__.py b/piccolo/apps/migrations/auto/__init__.py index cdffc6c1c..1df58816c 100644 --- a/piccolo/apps/migrations/auto/__init__.py +++ b/piccolo/apps/migrations/auto/__init__.py @@ -2,3 +2,11 @@ from .migration_manager import MigrationManager from .schema_differ import AlterStatements, SchemaDiffer from .schema_snapshot import SchemaSnapshot + +__all__ = [ + "DiffableTable", + "MigrationManager", + "AlterStatements", + "SchemaDiffer", + "SchemaSnapshot", +] diff --git a/piccolo/apps/migrations/auto/migration_manager.py b/piccolo/apps/migrations/auto/migration_manager.py index fca36e8e7..e8f4931cb 100644 --- a/piccolo/apps/migrations/auto/migration_manager.py +++ b/piccolo/apps/migrations/auto/migration_manager.py @@ -261,7 +261,8 @@ def add_column( cleaned_params = deserialise_params(params=params) column = column_class(**cleaned_params) column._meta.name = column_name - column._meta.db_column_name = db_column_name + if db_column_name: + column._meta.db_column_name = db_column_name self.add_columns.append( AddColumnClass( diff --git a/piccolo/apps/migrations/commands/backwards.py b/piccolo/apps/migrations/commands/backwards.py index 6627fe8af..363992510 100644 --- a/piccolo/apps/migrations/commands/backwards.py +++ b/piccolo/apps/migrations/commands/backwards.py @@ -32,7 +32,9 @@ def __init__( async def run_migrations_backwards(self, app_config: AppConfig): migration_modules: t.Dict[str, MigrationModule] = ( - self.get_migration_modules(app_config.migrations_folder_path) + self.get_migration_modules( + app_config.resolved_migrations_folder_path + ) ) ran_migration_ids = await Migration.get_migrations_which_ran( diff --git a/piccolo/apps/migrations/commands/base.py b/piccolo/apps/migrations/commands/base.py index a3966f7c3..bcc5cbc55 100644 --- a/piccolo/apps/migrations/commands/base.py +++ b/piccolo/apps/migrations/commands/base.py @@ -86,7 +86,7 @@ async def get_migration_managers( """ migration_managers: t.List[MigrationManager] = [] - migrations_folder = app_config.migrations_folder_path + migrations_folder = app_config.resolved_migrations_folder_path migration_modules: t.Dict[str, MigrationModule] = ( self.get_migration_modules(migrations_folder) diff --git a/piccolo/apps/migrations/commands/check.py b/piccolo/apps/migrations/commands/check.py index fd2b49c3d..53e20840a 100644 --- a/piccolo/apps/migrations/commands/check.py +++ b/piccolo/apps/migrations/commands/check.py @@ -36,7 +36,7 @@ async def get_migration_statuses(self) -> t.List[MigrationStatus]: continue migration_modules = self.get_migration_modules( - app_config.migrations_folder_path + app_config.resolved_migrations_folder_path ) ids = self.get_migration_ids(migration_modules) for _id in ids: diff --git a/piccolo/apps/migrations/commands/clean.py b/piccolo/apps/migrations/commands/clean.py index e7ef22091..687ff64e9 100644 --- a/piccolo/apps/migrations/commands/clean.py +++ b/piccolo/apps/migrations/commands/clean.py @@ -20,7 +20,7 @@ def get_migration_ids_to_remove(self) -> t.List[str]: app_config = self.get_app_config(app_name=self.app_name) migration_module_dict = self.get_migration_modules( - folder_path=app_config.migrations_folder_path + folder_path=app_config.resolved_migrations_folder_path ) # The migration IDs which are in migration modules. diff --git a/piccolo/apps/migrations/commands/forwards.py b/piccolo/apps/migrations/commands/forwards.py index 6d967dd5e..62278060d 100644 --- a/piccolo/apps/migrations/commands/forwards.py +++ b/piccolo/apps/migrations/commands/forwards.py @@ -33,7 +33,9 @@ async def run_migrations(self, app_config: AppConfig) -> MigrationResult: ) migration_modules: t.Dict[str, MigrationModule] = ( - self.get_migration_modules(app_config.migrations_folder_path) + self.get_migration_modules( + app_config.resolved_migrations_folder_path + ) ) ids = self.get_migration_ids(migration_modules) diff --git a/piccolo/apps/migrations/commands/new.py b/piccolo/apps/migrations/commands/new.py index ff123aaa2..082868435 100644 --- a/piccolo/apps/migrations/commands/new.py +++ b/piccolo/apps/migrations/commands/new.py @@ -98,7 +98,9 @@ def _generate_migration_meta(app_config: AppConfig) -> NewMigrationMeta: filename = f"{cleaned_app_name}_{cleaned_id}" - path = os.path.join(app_config.migrations_folder_path, f"{filename}.py") + path = os.path.join( + app_config.resolved_migrations_folder_path, f"{filename}.py" + ) return NewMigrationMeta( migration_id=_id, migration_filename=filename, migration_path=path @@ -255,7 +257,7 @@ async def new( app_config = Finder().get_app_config(app_name=app_name) - _create_migrations_folder(app_config.migrations_folder_path) + _create_migrations_folder(app_config.resolved_migrations_folder_path) try: await _create_new_migration( diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index 20aea360d..da97d247b 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -313,7 +313,7 @@ def __add__(self, value: OutputSchema) -> OutputSchema: **{"integer": BigInt, "json": JSONB}, } -COLUMN_DEFAULT_PARSER = { +COLUMN_DEFAULT_PARSER: t.Dict[t.Type[Column], t.Any] = { BigInt: re.compile(r"^'?(?P-?[0-9]\d*)'?(?:::bigint)?$"), Boolean: re.compile(r"^(?Ptrue|false)$"), Bytea: re.compile(r"'(?P.*)'::bytea$"), @@ -373,7 +373,7 @@ def __add__(self, value: OutputSchema) -> OutputSchema: } # Re-map for Cockroach compatibility. -COLUMN_DEFAULT_PARSER_COCKROACH = { +COLUMN_DEFAULT_PARSER_COCKROACH: t.Dict[t.Type[Column], t.Any] = { **COLUMN_DEFAULT_PARSER, BigInt: re.compile(r"^(?P-?\d+)$"), } diff --git a/piccolo/apps/shell/commands/run.py b/piccolo/apps/shell/commands/run.py index 38cd1af66..4f86cc23c 100644 --- a/piccolo/apps/shell/commands/run.py +++ b/piccolo/apps/shell/commands/run.py @@ -24,7 +24,7 @@ def start_ipython_shell(**tables: t.Type[Table]): # pragma: no cover if table_class_name not in existing_global_names: globals()[table_class_name] = table_class - IPython.embed(using=_asyncio_runner, colors="neutral") + IPython.embed(using=_asyncio_runner, colors="neutral") # type: ignore def run() -> None: diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index c248d3b1b..9c74b4a52 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -1956,7 +1956,9 @@ def _setup(self, table_class: t.Type[Table]) -> ForeignKeySetupResponse: if is_table_class: # Record the reverse relationship on the target table. - references._meta._foreign_key_references.append(self) + t.cast( + t.Type[Table], references + )._meta._foreign_key_references.append(self) # Allow columns on the referenced table to be accessed via # auto completion. @@ -2710,7 +2712,7 @@ def all(self, value: t.Any) -> Where: else: raise ValueError("Unrecognised engine type") - def cat(self, value: t.List[t.Any]) -> QueryString: + def cat(self, value: t.Union[t.Any, t.List[t.Any]]) -> QueryString: """ Used in an ``update`` query to append items to an array. @@ -2741,7 +2743,7 @@ def cat(self, value: t.List[t.Any]) -> QueryString: db_column_name = self._meta.db_column_name return QueryString(f'array_cat("{db_column_name}", {{}})', value) - def __add__(self, value: t.List[t.Any]) -> QueryString: + def __add__(self, value: t.Union[t.Any, t.List[t.Any]]) -> QueryString: return self.cat(value) ########################################################################### diff --git a/piccolo/columns/defaults/base.py b/piccolo/columns/defaults/base.py index 9ef45ec93..062162032 100644 --- a/piccolo/columns/defaults/base.py +++ b/piccolo/columns/defaults/base.py @@ -18,7 +18,7 @@ def sqlite(self) -> str: pass @abstractmethod - def python(self): + def python(self) -> t.Any: pass def get_postgres_interval_string(self, attributes: t.List[str]) -> str: diff --git a/piccolo/columns/defaults/date.py b/piccolo/columns/defaults/date.py index 87e431390..423f112ca 100644 --- a/piccolo/columns/defaults/date.py +++ b/piccolo/columns/defaults/date.py @@ -102,7 +102,15 @@ def from_date(cls, instance: datetime.date): # Might add an enum back which encapsulates all of the options. -DateArg = t.Union[DateOffset, DateCustom, DateNow, Enum, None, datetime.date] +DateArg = t.Union[ + DateOffset, + DateCustom, + DateNow, + Enum, + None, + datetime.date, + t.Callable[[], datetime.date], +] __all__ = ["DateArg", "DateOffset", "DateCustom", "DateNow"] diff --git a/piccolo/columns/defaults/interval.py b/piccolo/columns/defaults/interval.py index f3daba639..4d5f72ae8 100644 --- a/piccolo/columns/defaults/interval.py +++ b/piccolo/columns/defaults/interval.py @@ -80,6 +80,7 @@ def from_timedelta(cls, instance: datetime.timedelta): Enum, None, datetime.timedelta, + t.Callable[[], datetime.timedelta], ] diff --git a/piccolo/columns/defaults/time.py b/piccolo/columns/defaults/time.py index 25535cb5d..9b72416ea 100644 --- a/piccolo/columns/defaults/time.py +++ b/piccolo/columns/defaults/time.py @@ -89,7 +89,15 @@ def from_time(cls, instance: datetime.time): ) -TimeArg = t.Union[TimeCustom, TimeNow, TimeOffset, Enum, None, datetime.time] +TimeArg = t.Union[ + TimeCustom, + TimeNow, + TimeOffset, + Enum, + None, + datetime.time, + t.Callable[[], datetime.time], +] __all__ = ["TimeArg", "TimeCustom", "TimeNow", "TimeOffset"] diff --git a/piccolo/columns/defaults/timestamp.py b/piccolo/columns/defaults/timestamp.py index 9558f4100..73e06d1ef 100644 --- a/piccolo/columns/defaults/timestamp.py +++ b/piccolo/columns/defaults/timestamp.py @@ -138,6 +138,7 @@ class DatetimeDefault: None, datetime.datetime, DatetimeDefault, + t.Callable[[], datetime.datetime], ] diff --git a/piccolo/columns/defaults/uuid.py b/piccolo/columns/defaults/uuid.py index 176d282ec..17b07021c 100644 --- a/piccolo/columns/defaults/uuid.py +++ b/piccolo/columns/defaults/uuid.py @@ -22,7 +22,7 @@ def python(self): return uuid.uuid4() -UUIDArg = t.Union[UUID4, uuid.UUID, str, Enum, None] +UUIDArg = t.Union[UUID4, uuid.UUID, str, Enum, None, t.Callable[[], uuid.UUID]] __all__ = ["UUIDArg", "UUID4"] diff --git a/piccolo/columns/m2m.py b/piccolo/columns/m2m.py index 90469fc1f..29bafe9b5 100644 --- a/piccolo/columns/m2m.py +++ b/piccolo/columns/m2m.py @@ -131,6 +131,7 @@ def get_select_string( if len(self.columns) > 1 or not self.serialisation_safe: column_name = table_2_pk_name else: + assert len(self.columns) > 0 column_name = self.columns[0]._meta.db_column_name return QueryString( @@ -256,15 +257,14 @@ def secondary_table(self) -> t.Type[Table]: @dataclass class M2MAddRelated: - target_row: Table m2m: M2M rows: t.Sequence[Table] extra_column_values: t.Dict[t.Union[Column, str], t.Any] - def __post_init__(self) -> None: - # Normalise `extra_column_values`, so we just have the column names. - self.extra_column_values: t.Dict[str, t.Any] = { + @property + def resolved_extra_column_values(self) -> t.Dict[str, t.Any]: + return { i._meta.name if isinstance(i, Column) else i: j for i, j in self.extra_column_values.items() } @@ -281,7 +281,9 @@ async def _run(self): joining_table_rows = [] for row in rows: - joining_table_row = joining_table(**self.extra_column_values) + joining_table_row = joining_table( + **self.resolved_extra_column_values + ) setattr( joining_table_row, self.m2m._meta.primary_foreign_key._meta.name, @@ -323,7 +325,6 @@ def __await__(self): @dataclass class M2MRemoveRelated: - target_row: Table m2m: M2M rows: t.Sequence[Table] @@ -363,7 +364,6 @@ def __await__(self): @dataclass class M2MGetRelated: - row: Table m2m: M2M diff --git a/piccolo/conf/apps.py b/piccolo/conf/apps.py index 47631c478..6c16c9e81 100644 --- a/piccolo/conf/apps.py +++ b/piccolo/conf/apps.py @@ -157,17 +157,22 @@ class AppConfig: """ app_name: str - migrations_folder_path: str + migrations_folder_path: t.Union[str, pathlib.Path] table_classes: t.List[t.Type[Table]] = field(default_factory=list) migration_dependencies: t.List[str] = field(default_factory=list) commands: t.List[t.Union[t.Callable, Command]] = field( default_factory=list ) - def __post_init__(self) -> None: - if isinstance(self.migrations_folder_path, pathlib.Path): - self.migrations_folder_path = str(self.migrations_folder_path) + @property + def resolved_migrations_folder_path(self) -> str: + return ( + str(self.migrations_folder_path) + if isinstance(self.migrations_folder_path, pathlib.Path) + else self.migrations_folder_path + ) + def __post_init__(self) -> None: self._migration_dependency_app_configs: t.Optional[ t.List[AppConfig] ] = None diff --git a/piccolo/engine/base.py b/piccolo/engine/base.py index 95d1b8a24..bf59426ad 100644 --- a/piccolo/engine/base.py +++ b/piccolo/engine/base.py @@ -7,12 +7,14 @@ import typing as t from abc import ABCMeta, abstractmethod +from typing_extensions import Self + from piccolo.querystring import QueryString from piccolo.utils.sync import run_sync from piccolo.utils.warnings import Level, colored_string, colored_warning if t.TYPE_CHECKING: # pragma: no cover - from piccolo.query.base import Query + from piccolo.query.base import DDL, Query logger = logging.getLogger(__name__) @@ -32,31 +34,76 @@ def validate_savepoint_name(savepoint_name: str) -> None: ) -class Batch: - pass +class BaseBatch(metaclass=ABCMeta): + @abstractmethod + async def __aenter__(self: Self, *args, **kwargs) -> Self: ... + @abstractmethod + async def __aexit__(self, *args, **kwargs): ... -TransactionClass = t.TypeVar("TransactionClass") + @abstractmethod + def __aiter__(self: Self) -> Self: ... + @abstractmethod + async def __anext__(self) -> t.List[t.Dict]: ... -class Engine(t.Generic[TransactionClass], metaclass=ABCMeta): - __slots__ = ("query_id",) +class BaseTransaction(metaclass=ABCMeta): - def __init__(self): - run_sync(self.check_version()) - run_sync(self.prep_database()) - self.query_id = 0 + __slots__: t.Tuple[str, ...] = tuple() - @property @abstractmethod - def engine_type(self) -> str: - pass + async def __aenter__(self, *args, **kwargs): ... - @property @abstractmethod - def min_version_number(self) -> float: - pass + async def __aexit__(self, *args, **kwargs) -> bool: ... + + +class BaseAtomic(metaclass=ABCMeta): + + __slots__: t.Tuple[str, ...] = tuple() + + @abstractmethod + def add(self, *query: t.Union[Query, DDL]): ... + + @abstractmethod + async def run(self): ... + + @abstractmethod + def run_sync(self): ... + + @abstractmethod + def __await__(self): ... + + +TransactionClass = t.TypeVar("TransactionClass", bound=BaseTransaction) + + +class Engine(t.Generic[TransactionClass], metaclass=ABCMeta): + __slots__ = ( + "query_id", + "log_queries", + "log_responses", + "engine_type", + "min_version_number", + "current_transaction", + ) + + def __init__( + self, + engine_type: str, + min_version_number: t.Union[int, float], + log_queries: bool = False, + log_responses: bool = False, + ): + self.log_queries = log_queries + self.log_responses = log_responses + self.engine_type = engine_type + self.min_version_number = min_version_number + + run_sync(self.check_version()) + run_sync(self.prep_database()) + self.query_id = 0 @abstractmethod async def get_version(self) -> float: @@ -76,11 +123,13 @@ async def batch( query: Query, batch_size: int = 100, node: t.Optional[str] = None, - ) -> Batch: + ) -> BaseBatch: pass @abstractmethod - async def run_querystring(self, querystring: QueryString, in_pool: bool): + async def run_querystring( + self, querystring: QueryString, in_pool: bool = True + ): pass @abstractmethod @@ -88,11 +137,11 @@ async def run_ddl(self, ddl: str, in_pool: bool = True): pass @abstractmethod - def transaction(self): + def transaction(self, *args, **kwargs) -> TransactionClass: pass @abstractmethod - def atomic(self): + def atomic(self) -> BaseAtomic: pass async def check_version(self): diff --git a/piccolo/engine/cockroach.py b/piccolo/engine/cockroach.py index ecbb74ad8..d091527bb 100644 --- a/piccolo/engine/cockroach.py +++ b/piccolo/engine/cockroach.py @@ -16,9 +16,6 @@ class CockroachEngine(PostgresEngine): :class:`PostgresEngine `. """ - engine_type = "cockroach" - min_version_number = 0 # Doesn't seem to work with cockroach versioning. - def __init__( self, config: t.Dict[str, t.Any], @@ -34,6 +31,8 @@ def __init__( log_responses=log_responses, extra_nodes=extra_nodes, ) + self.engine_type = "cockroach" + self.min_version_number = 0 async def prep_database(self): try: diff --git a/piccolo/engine/postgres.py b/piccolo/engine/postgres.py index 06b8ffb4b..970623535 100644 --- a/piccolo/engine/postgres.py +++ b/piccolo/engine/postgres.py @@ -4,7 +4,15 @@ import typing as t from dataclasses import dataclass -from piccolo.engine.base import Batch, Engine, validate_savepoint_name +from typing_extensions import Self + +from piccolo.engine.base import ( + BaseAtomic, + BaseBatch, + BaseTransaction, + Engine, + validate_savepoint_name, +) from piccolo.engine.exceptions import TransactionError from piccolo.query.base import DDL, Query from piccolo.querystring import QueryString @@ -18,16 +26,17 @@ from asyncpg.connection import Connection from asyncpg.cursor import Cursor from asyncpg.pool import Pool + from asyncpg.transaction import Transaction @dataclass -class AsyncBatch(Batch): +class AsyncBatch(BaseBatch): connection: Connection query: Query batch_size: int # Set internally - _transaction = None + _transaction: t.Optional[Transaction] = None _cursor: t.Optional[Cursor] = None @property @@ -36,20 +45,26 @@ def cursor(self) -> Cursor: raise ValueError("_cursor not set") return self._cursor + @property + def transaction(self) -> Transaction: + if not self._transaction: + raise ValueError("The transaction can't be found.") + return self._transaction + async def next(self) -> t.List[t.Dict]: data = await self.cursor.fetch(self.batch_size) return await self.query._process_results(data) - def __aiter__(self): + def __aiter__(self: Self) -> Self: return self - async def __anext__(self): + async def __anext__(self) -> t.List[t.Dict]: response = await self.next() if response == []: raise StopAsyncIteration() return response - async def __aenter__(self): + async def __aenter__(self: Self) -> Self: self._transaction = self.connection.transaction() await self._transaction.start() querystring = self.query.querystrings[0] @@ -60,9 +75,9 @@ async def __aenter__(self): async def __aexit__(self, exception_type, exception, traceback): if exception: - await self._transaction.rollback() + await self.transaction.rollback() else: - await self._transaction.commit() + await self.transaction.commit() await self.connection.close() @@ -72,7 +87,7 @@ async def __aexit__(self, exception_type, exception, traceback): ############################################################################### -class Atomic: +class Atomic(BaseAtomic): """ This is useful if you want to build up a transaction programatically, by adding queries to it. @@ -140,7 +155,7 @@ async def release(self): ) -class PostgresTransaction: +class PostgresTransaction(BaseTransaction): """ Used for wrapping queries in a transaction, using a context manager. Currently it's async only. @@ -243,7 +258,7 @@ async def savepoint(self, name: t.Optional[str] = None) -> Savepoint: ########################################################################### - async def __aexit__(self, exception_type, exception, traceback): + async def __aexit__(self, exception_type, exception, traceback) -> bool: if self._parent: return exception is None @@ -269,7 +284,7 @@ async def __aexit__(self, exception_type, exception, traceback): ############################################################################### -class PostgresEngine(Engine[t.Optional[PostgresTransaction]]): +class PostgresEngine(Engine[PostgresTransaction]): """ Used to connect to PostgreSQL. @@ -331,16 +346,10 @@ class PostgresEngine(Engine[t.Optional[PostgresTransaction]]): __slots__ = ( "config", "extensions", - "log_queries", - "log_responses", "extra_nodes", "pool", - "current_transaction", ) - engine_type = "postgres" - min_version_number = 10 - def __init__( self, config: t.Dict[str, t.Any], @@ -362,7 +371,12 @@ def __init__( self.current_transaction = contextvars.ContextVar( f"pg_current_transaction_{database_name}", default=None ) - super().__init__() + super().__init__( + engine_type="postgres", + log_queries=log_queries, + log_responses=log_responses, + min_version_number=10, + ) @staticmethod def _parse_raw_version_string(version_string: str) -> float: diff --git a/piccolo/engine/sqlite.py b/piccolo/engine/sqlite.py index f6fbd4e38..3f7649d76 100644 --- a/piccolo/engine/sqlite.py +++ b/piccolo/engine/sqlite.py @@ -11,7 +11,15 @@ from decimal import Decimal from functools import partial, wraps -from piccolo.engine.base import Batch, Engine, validate_savepoint_name +from typing_extensions import Self + +from piccolo.engine.base import ( + BaseAtomic, + BaseBatch, + BaseTransaction, + Engine, + validate_savepoint_name, +) from piccolo.engine.exceptions import TransactionError from piccolo.query.base import DDL, Query from piccolo.querystring import QueryString @@ -309,7 +317,7 @@ def convert_M2M_out(value: str) -> t.List: @dataclass -class AsyncBatch(Batch): +class AsyncBatch(BaseBatch): connection: Connection query: Query batch_size: int @@ -327,16 +335,16 @@ async def next(self) -> t.List[t.Dict]: data = await self.cursor.fetchmany(self.batch_size) return await self.query._process_results(data) - def __aiter__(self): + def __aiter__(self: Self) -> Self: return self - async def __anext__(self): + async def __anext__(self) -> t.List[t.Dict]: response = await self.next() if response == []: raise StopAsyncIteration() return response - async def __aenter__(self): + async def __aenter__(self: Self) -> Self: querystring = self.query.querystrings[0] template, template_args = querystring.compile_string() @@ -344,7 +352,7 @@ async def __aenter__(self): return self async def __aexit__(self, exception_type, exception, traceback): - await self._cursor.close() + await self.cursor.close() await self.connection.close() return exception is not None @@ -363,7 +371,7 @@ class TransactionType(enum.Enum): exclusive = "EXCLUSIVE" -class Atomic: +class Atomic(BaseAtomic): """ Usage: @@ -384,9 +392,9 @@ def __init__( ): self.engine = engine self.transaction_type = transaction_type - self.queries: t.List[Query] = [] + self.queries: t.List[t.Union[Query, DDL]] = [] - def add(self, *query: Query): + def add(self, *query: t.Union[Query, DDL]): self.queries += list(query) async def run(self): @@ -434,7 +442,7 @@ async def release(self): ) -class SQLiteTransaction: +class SQLiteTransaction(BaseTransaction): """ Used for wrapping queries in a transaction, using a context manager. Currently it's async only. @@ -534,7 +542,7 @@ async def savepoint(self, name: t.Optional[str] = None) -> Savepoint: ########################################################################### - async def __aexit__(self, exception_type, exception, traceback): + async def __aexit__(self, exception_type, exception, traceback) -> bool: if self._parent: return exception is None @@ -560,16 +568,8 @@ def dict_factory(cursor, row) -> t.Dict: return {col[0]: row[idx] for idx, col in enumerate(cursor.description)} -class SQLiteEngine(Engine[t.Optional[SQLiteTransaction]]): - __slots__ = ( - "connection_kwargs", - "current_transaction", - "log_queries", - "log_responses", - ) - - engine_type = "sqlite" - min_version_number = 3.25 +class SQLiteEngine(Engine[SQLiteTransaction]): + __slots__ = ("connection_kwargs",) def __init__( self, @@ -613,7 +613,12 @@ def __init__( f"sqlite_current_transaction_{path}", default=None ) - super().__init__() + super().__init__( + engine_type="sqlite", + min_version_number=3.25, + log_queries=log_queries, + log_responses=log_responses, + ) @property def path(self): diff --git a/piccolo/query/methods/create_index.py b/piccolo/query/methods/create_index.py index b10e0c203..c81d38b9d 100644 --- a/piccolo/query/methods/create_index.py +++ b/piccolo/query/methods/create_index.py @@ -14,7 +14,7 @@ class CreateIndex(DDL): def __init__( self, table: t.Type[Table], - columns: t.List[t.Union[Column, str]], + columns: t.Union[t.List[Column], t.List[str]], method: IndexMethod = IndexMethod.btree, if_not_exists: bool = False, **kwargs, diff --git a/piccolo/query/methods/drop_index.py b/piccolo/query/methods/drop_index.py index 437728437..049a066dd 100644 --- a/piccolo/query/methods/drop_index.py +++ b/piccolo/query/methods/drop_index.py @@ -14,7 +14,7 @@ class DropIndex(Query): def __init__( self, table: t.Type[Table], - columns: t.List[t.Union[Column, str]], + columns: t.Union[t.List[Column], t.List[str]], if_exists: bool = True, **kwargs, ): diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index f11f78e8e..7f2b5aaed 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -5,7 +5,7 @@ from piccolo.columns.column_types import ForeignKey from piccolo.columns.combination import And, Where from piccolo.custom_types import Combinable, TableInstance -from piccolo.engine.base import Batch +from piccolo.engine.base import BaseBatch from piccolo.query.base import Query from piccolo.query.methods.select import Select from piccolo.query.mixins import ( @@ -268,17 +268,17 @@ def where(self: Self, *where: t.Union[Combinable, QueryString]) -> Self: ########################################################################### - def first(self: Self) -> First[TableInstance]: + def first(self) -> First[TableInstance]: self.limit_delegate.limit(1) return First[TableInstance](query=self) - def get(self: Self, where: Combinable) -> Get[TableInstance]: + def get(self, where: Combinable) -> Get[TableInstance]: self.where_delegate.where(where) self.limit_delegate.limit(1) return Get[TableInstance](query=First[TableInstance](query=self)) def get_or_create( - self: Self, + self, where: Combinable, defaults: t.Optional[t.Dict[Column, t.Any]] = None, ) -> GetOrCreate[TableInstance]: @@ -288,17 +288,17 @@ def get_or_create( query=self, table_class=self.table, where=where, defaults=defaults ) - def create(self: Self, **columns: t.Any) -> Create[TableInstance]: + def create(self, **columns: t.Any) -> Create[TableInstance]: return Create[TableInstance](table_class=self.table, columns=columns) ########################################################################### async def batch( - self: Self, + self, batch_size: t.Optional[int] = None, node: t.Optional[str] = None, **kwargs, - ) -> Batch: + ) -> BaseBatch: if batch_size: kwargs.update(batch_size=batch_size) if node: diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index fdb929f8a..05302455f 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -5,11 +5,11 @@ from collections import OrderedDict from piccolo.columns import Column, Selectable -from piccolo.columns.column_types import JSON, JSONB, PrimaryKey +from piccolo.columns.column_types import JSON, JSONB from piccolo.columns.m2m import M2MSelect from piccolo.columns.readable import Readable from piccolo.custom_types import TableInstance -from piccolo.engine.base import Batch +from piccolo.engine.base import BaseBatch from piccolo.query.base import Query from piccolo.query.mixins import ( AsOfDelegate, @@ -217,7 +217,7 @@ async def _splice_m2m_rows( self, response: t.List[t.Dict[str, t.Any]], secondary_table: t.Type[Table], - secondary_table_pk: PrimaryKey, + secondary_table_pk: Column, m2m_name: str, m2m_select: M2MSelect, as_list: bool = False, @@ -386,14 +386,20 @@ def order_by( return self @t.overload - def output(self: Self, *, as_list: bool) -> SelectList: ... + def output(self: Self, *, as_list: bool) -> SelectList: # type: ignore + ... @t.overload - def output(self: Self, *, as_json: bool) -> SelectJSON: ... + def output(self: Self, *, as_json: bool) -> SelectJSON: # type: ignore + ... @t.overload def output(self: Self, *, load_json: bool) -> Self: ... + @t.overload + def output(self: Self, *, load_json: bool, as_list: bool) -> SelectJSON: # type: ignore # noqa: E501 + ... + @t.overload def output(self: Self, *, nested: bool) -> Self: ... @@ -404,7 +410,7 @@ def output( as_json: bool = False, load_json: bool = False, nested: bool = False, - ): + ) -> t.Union[Self, SelectJSON, SelectList]: self.output_delegate.output( as_list=as_list, as_json=as_json, @@ -436,7 +442,7 @@ async def batch( batch_size: t.Optional[int] = None, node: t.Optional[str] = None, **kwargs, - ) -> Batch: + ) -> BaseBatch: if batch_size: kwargs.update(batch_size=batch_size) if node: diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index 214d1b8d7..d9d5f84ca 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -207,7 +207,6 @@ def __str__(self): @dataclass class Output: - as_json: bool = False as_list: bool = False as_objects: bool = False @@ -236,7 +235,6 @@ class Callback: @dataclass class WhereDelegate: - _where: t.Optional[Combinable] = None _where_columns: t.List[Column] = field(default_factory=list) @@ -246,7 +244,8 @@ def get_where_columns(self): needed. """ self._where_columns = [] - self._extract_columns(self._where) + if self._where is not None: + self._extract_columns(self._where) return self._where_columns def _extract_columns(self, combinable: Combinable): @@ -277,7 +276,6 @@ def where(self, *where: t.Union[Combinable, QueryString]): @dataclass class OrderByDelegate: - _order_by: OrderBy = field(default_factory=OrderBy) def get_order_by_columns(self) -> t.List[Column]: @@ -303,7 +301,6 @@ def order_by(self, *columns: t.Union[Column, OrderByRaw], ascending=True): @dataclass class LimitDelegate: - _limit: t.Optional[Limit] = None _first: bool = False @@ -330,7 +327,6 @@ def as_of(self, interval: str = "-1s"): @dataclass class DistinctDelegate: - _distinct: Distinct = field( default_factory=lambda: Distinct(enabled=False, on=None) ) @@ -356,7 +352,6 @@ def returning(self, columns: t.Sequence[Column]): @dataclass class CountDelegate: - _count: bool = False def count(self): @@ -365,7 +360,6 @@ def count(self): @dataclass class AddDelegate: - _add: t.List[Table] = field(default_factory=list) def add(self, *instances: Table, table_class: t.Type[Table]): @@ -421,8 +415,7 @@ def output( self._output.nested = bool(nested) def copy(self) -> OutputDelegate: - _output = self._output.copy() if self._output is not None else None - return self.__class__(_output=_output) + return self.__class__(_output=self._output.copy()) @dataclass diff --git a/piccolo/schema.py b/piccolo/schema.py index ef0bd6ab4..01cd0bd91 100644 --- a/piccolo/schema.py +++ b/piccolo/schema.py @@ -10,7 +10,6 @@ class SchemaDDLBase(abc.ABC): - db: Engine @property @@ -132,16 +131,19 @@ def __init__(self, db: Engine, schema_name: str): self.db = db self.schema_name = schema_name - async def run(self): - response = await self.db.run_querystring( - QueryString( - """ + async def run(self) -> t.List[str]: + response = t.cast( + t.List[t.Dict], + await self.db.run_querystring( + QueryString( + """ SELECT table_name FROM information_schema.tables WHERE table_schema = {} """, - self.schema_name, - ) + self.schema_name, + ) + ), ) return [i["table_name"] for i in response] @@ -156,9 +158,14 @@ class ListSchemas: def __init__(self, db: Engine): self.db = db - async def run(self): - response = await self.db.run_querystring( - QueryString("SELECT schema_name FROM information_schema.schemata") + async def run(self) -> t.List[str]: + response = t.cast( + t.List[t.Dict], + await self.db.run_querystring( + QueryString( + "SELECT schema_name FROM information_schema.schemata" + ) + ), ) return [i["schema_name"] for i in response] @@ -180,7 +187,7 @@ def __init__(self, db: t.Optional[Engine] = None): """ db = db or engine_finder() - if not db: + if db is None: raise ValueError("The DB can't be found.") self.db = db diff --git a/piccolo/table.py b/piccolo/table.py index 7882db95e..3b3ff4853 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -143,8 +143,11 @@ def db(self) -> Engine: def db(self, value: Engine): self._db = value - def refresh_db(self): - self.db = engine_finder() + def refresh_db(self) -> None: + engine = engine_finder() + if engine is None: + raise ValueError("The engine can't be found") + self.db = engine def get_column_by_name(self, name: str) -> Column: """ @@ -184,8 +187,8 @@ def get_auto_update_values(self) -> t.Dict[Column, t.Any]: class TableMetaclass(type): - def __str__(cls): - return cls._table_str() + def __str__(cls) -> str: + return cls._table_str() # type: ignore def __repr__(cls): """ @@ -822,7 +825,7 @@ def __repr__(self) -> str: @classmethod def all_related( cls, exclude: t.Optional[t.List[t.Union[str, ForeignKey]]] = None - ) -> t.List[Column]: + ) -> t.List[ForeignKey]: """ Used in conjunction with ``objects`` queries. Just as we can use ``all_related`` on a ``ForeignKey``, you can also use it for the table @@ -1251,7 +1254,7 @@ def indexes(cls) -> Indexes: @classmethod def create_index( cls, - columns: t.List[t.Union[Column, str]], + columns: t.Union[t.List[Column], t.List[str]], method: IndexMethod = IndexMethod.btree, if_not_exists: bool = False, ) -> CreateIndex: @@ -1273,7 +1276,9 @@ def create_index( @classmethod def drop_index( - cls, columns: t.List[t.Union[Column, str]], if_exists: bool = True + cls, + columns: t.Union[t.List[Column], t.List[str]], + if_exists: bool = True, ) -> DropIndex: """ Drop a table index. If multiple columns are specified, this refers @@ -1464,22 +1469,18 @@ async def drop_db_tables(*tables: t.Type[Table]) -> None: # SQLite doesn't support CASCADE, so we have to drop them in the # correct order. sorted_table_classes = reversed(sort_table_classes(list(tables))) - atomic = engine.atomic() - atomic.add( - *[ - Alter(table=table).drop_table(if_exists=True) - for table in sorted_table_classes - ] - ) + ddl_statements = [ + Alter(table=table).drop_table(if_exists=True) + for table in sorted_table_classes + ] else: - atomic = engine.atomic() - atomic.add( - *[ - table.alter().drop_table(cascade=True, if_exists=True) - for table in tables - ] - ) + ddl_statements = [ + table.alter().drop_table(cascade=True, if_exists=True) + for table in tables + ] + atomic = engine.atomic() + atomic.add(*ddl_statements) await atomic.run() diff --git a/piccolo/utils/encoding.py b/piccolo/utils/encoding.py index 48a131dc5..97fde4683 100644 --- a/piccolo/utils/encoding.py +++ b/piccolo/utils/encoding.py @@ -19,13 +19,15 @@ def dump_json(data: t.Any, pretty: bool = False) -> str: orjson_params["option"] = ( orjson.OPT_INDENT_2 | orjson.OPT_APPEND_NEWLINE # type: ignore ) - return orjson.dumps(data, **orjson_params).decode("utf8") + return orjson.dumps(data, **orjson_params).decode( # type: ignore + "utf8" + ) else: params: t.Dict[str, t.Any] = {"default": str} if pretty: params["indent"] = 2 - return json.dumps(data, **params) + return json.dumps(data, **params) # type: ignore def load_json(data: str) -> t.Any: - return orjson.loads(data) if ORJSON else json.loads(data) + return orjson.loads(data) if ORJSON else json.loads(data) # type: ignore diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt index 6fbda712d..726e62994 100644 --- a/requirements/dev-requirements.txt +++ b/requirements/dev-requirements.txt @@ -7,4 +7,5 @@ slotscheck==0.17.1 twine==3.8.0 mypy==1.7.1 pip-upgrader==1.4.15 +pyright==1.1.367 wheel==0.38.1 diff --git a/scripts/pyright.sh b/scripts/pyright.sh new file mode 100755 index 000000000..616652eb8 --- /dev/null +++ b/scripts/pyright.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# We have a separate script for pyright vs lint.sh, as it's hard to get 100% +# success in pyright. In the future we might merge them. + +set -e + +MODULES="piccolo" +SOURCES="$MODULES tests" + +echo "Running pyright..." +pyright $sources +echo "-----" + +echo "All passed!" diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index 2851aaee9..84b194ea8 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -145,7 +145,7 @@ def _test_migrations( """ app_config = self._get_app_config() - migrations_folder_path = app_config.migrations_folder_path + migrations_folder_path = app_config.resolved_migrations_folder_path if os.path.exists(migrations_folder_path): shutil.rmtree(migrations_folder_path) diff --git a/tests/conf/test_apps.py b/tests/conf/test_apps.py index 44b2d4a4a..0749f5f25 100644 --- a/tests/conf/test_apps.py +++ b/tests/conf/test_apps.py @@ -85,7 +85,7 @@ def test_pathlib(self): config = AppConfig( app_name="music", migrations_folder_path=pathlib.Path(__file__) ) - self.assertEqual(config.migrations_folder_path, __file__) + self.assertEqual(config.resolved_migrations_folder_path, __file__) def test_get_table_with_name(self): """ diff --git a/tests/engine/test_nested_transaction.py b/tests/engine/test_nested_transaction.py index 23bee59a4..71d519b79 100644 --- a/tests/engine/test_nested_transaction.py +++ b/tests/engine/test_nested_transaction.py @@ -45,10 +45,12 @@ async def run_nested(self): self.assertTrue(await Musician.table_exists().run()) musician = await Musician.select("name").first().run() + assert musician is not None self.assertEqual(musician["name"], "Bob") self.assertTrue(await Roadie.table_exists().run()) roadie = await Roadie.select("name").first().run() + assert roadie is not None self.assertEqual(roadie["name"], "Dave") def test_nested(self): diff --git a/tests/engine/test_transaction.py b/tests/engine/test_transaction.py index 4b47f8759..88e4cff15 100644 --- a/tests/engine/test_transaction.py +++ b/tests/engine/test_transaction.py @@ -4,7 +4,6 @@ import pytest -from piccolo.engine.postgres import Atomic from piccolo.engine.sqlite import SQLiteEngine, TransactionType from piccolo.table import drop_db_tables_sync from piccolo.utils.sync import run_sync @@ -58,7 +57,7 @@ async def run() -> None: engine = Band._meta.db await engine.start_connection_pool() - atomic: Atomic = engine.atomic() + atomic = engine.atomic() atomic.add( Manager.create_table(), Band.create_table(), diff --git a/tests/table/test_indexes.py b/tests/table/test_indexes.py index 6aebd350c..13d1758de 100644 --- a/tests/table/test_indexes.py +++ b/tests/table/test_indexes.py @@ -1,5 +1,7 @@ +import typing as t from unittest import TestCase +from piccolo.columns.base import Column from piccolo.columns.column_types import Integer from piccolo.table import Table from tests.example_apps.music.tables import Manager @@ -45,12 +47,12 @@ def setUp(self): def tearDown(self): Concert.alter().drop_table().run_sync() - def test_problematic_name(self): + def test_problematic_name(self) -> None: """ Make sure we can add an index to a column with a problematic name (which clashes with a SQL keyword). """ - columns = [Concert.order] + columns: t.List[Column] = [Concert.order] Concert.create_index(columns=columns).run_sync() index_name = Concert._get_index_name([i._meta.name for i in columns]) diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index 5447361e9..82096603d 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -274,6 +274,7 @@ class Ticket(Table): # We'll also fetch it from the DB in case the database adapter's UUID # is used. ticket_from_db = Ticket.objects().first().run_sync() + assert ticket_from_db is not None for ticket_ in (ticket, ticket_from_db): json = pydantic_model(**ticket_.to_dict()).model_dump_json() @@ -368,8 +369,8 @@ class Movie(Table): json_string = '{"code": 12345}' model_instance = pydantic_model(meta=json_string, meta_b=json_string) - self.assertEqual(model_instance.meta, json_string) - self.assertEqual(model_instance.meta_b, json_string) + self.assertEqual(model_instance.meta, json_string) # type: ignore + self.assertEqual(model_instance.meta_b, json_string) # type: ignore def test_deserialize_json(self): class Movie(Table): @@ -384,8 +385,8 @@ class Movie(Table): output = {"code": 12345} model_instance = pydantic_model(meta=json_string, meta_b=json_string) - self.assertEqual(model_instance.meta, output) - self.assertEqual(model_instance.meta_b, output) + self.assertEqual(model_instance.meta, output) # type: ignore + self.assertEqual(model_instance.meta_b, output) # type: ignore def test_validation(self): class Movie(Table): @@ -428,8 +429,8 @@ class Movie(Table): pydantic_model = create_pydantic_model(table=Movie) movie = pydantic_model(meta=None, meta_b=None) - self.assertIsNone(movie.meta) - self.assertIsNone(movie.meta_b) + self.assertIsNone(movie.meta) # type: ignore + self.assertIsNone(movie.meta_b) # type: ignore class TestExcludeColumns(TestCase): @@ -490,7 +491,7 @@ class Computer(Table): with self.assertRaises(ValueError): create_pydantic_model( Computer, - exclude_columns=("CPU",), + exclude_columns=("CPU",), # type: ignore ) def test_invalid_column_different_table(self): @@ -629,7 +630,10 @@ class Band(Table): ####################################################################### - ManagerModel = BandModel.model_fields["manager"].annotation + ManagerModel = t.cast( + t.Type[pydantic.BaseModel], + BandModel.model_fields["manager"].annotation, + ) self.assertTrue(issubclass(ManagerModel, pydantic.BaseModel)) self.assertEqual( [i for i in ManagerModel.model_fields.keys()], ["name", "country"] @@ -637,7 +641,10 @@ class Band(Table): ####################################################################### - CountryModel = ManagerModel.model_fields["country"].annotation + CountryModel = t.cast( + t.Type[pydantic.BaseModel], + ManagerModel.model_fields["country"].annotation, + ) self.assertTrue(issubclass(CountryModel, pydantic.BaseModel)) self.assertEqual( [i for i in CountryModel.model_fields.keys()], ["name"] @@ -674,7 +681,10 @@ class Concert(Table): BandModel = create_pydantic_model(table=Band, nested=(Band.manager,)) - ManagerModel = BandModel.model_fields["manager"].annotation + ManagerModel = t.cast( + t.Type[pydantic.BaseModel], + BandModel.model_fields["manager"].annotation, + ) self.assertTrue(issubclass(ManagerModel, pydantic.BaseModel)) self.assertEqual( [i for i in ManagerModel.model_fields.keys()], ["name", "country"] @@ -690,22 +700,29 @@ class Concert(Table): # Test two levels deep BandModel = create_pydantic_model( - table=Band, nested=(Band.manager.country,) + table=Band, nested=(Band.manager._.country,) ) - ManagerModel = BandModel.model_fields["manager"].annotation + ManagerModel = t.cast( + t.Type[pydantic.BaseModel], + BandModel.model_fields["manager"].annotation, + ) self.assertTrue(issubclass(ManagerModel, pydantic.BaseModel)) self.assertEqual( [i for i in ManagerModel.model_fields.keys()], ["name", "country"] ) self.assertEqual(ManagerModel.__qualname__, "Band.manager") - AssistantManagerType = BandModel.model_fields[ - "assistant_manager" - ].annotation + AssistantManagerType = t.cast( + t.Type[pydantic.BaseModel], + BandModel.model_fields["assistant_manager"].annotation, + ) self.assertIs(AssistantManagerType, t.Optional[int]) - CountryModel = ManagerModel.model_fields["country"].annotation + CountryModel = t.cast( + t.Type[pydantic.BaseModel], + ManagerModel.model_fields["country"].annotation, + ) self.assertTrue(issubclass(CountryModel, pydantic.BaseModel)) self.assertEqual( [i for i in CountryModel.model_fields.keys()], ["name"] @@ -716,13 +733,16 @@ class Concert(Table): # Test three levels deep ConcertModel = create_pydantic_model( - Concert, nested=(Concert.band_1.manager,) + Concert, nested=(Concert.band_1._.manager,) ) VenueModel = ConcertModel.model_fields["venue"].annotation self.assertIs(VenueModel, t.Optional[int]) - BandModel = ConcertModel.model_fields["band_1"].annotation + BandModel = t.cast( + t.Type[pydantic.BaseModel], + ConcertModel.model_fields["band_1"].annotation, + ) self.assertTrue(issubclass(BandModel, pydantic.BaseModel)) self.assertEqual( [i for i in BandModel.model_fields.keys()], @@ -730,7 +750,10 @@ class Concert(Table): ) self.assertEqual(BandModel.__qualname__, "Concert.band_1") - ManagerModel = BandModel.model_fields["manager"].annotation + ManagerModel = t.cast( + t.Type[pydantic.BaseModel], + BandModel.model_fields["manager"].annotation, + ) self.assertTrue(issubclass(ManagerModel, pydantic.BaseModel)) self.assertEqual( [i for i in ManagerModel.model_fields.keys()], @@ -751,11 +774,14 @@ class Concert(Table): MyConcertModel = create_pydantic_model( Concert, - nested=(Concert.band_1.manager,), + nested=(Concert.band_1._.manager,), model_name="MyConcertModel", ) - BandModel = MyConcertModel.model_fields["band_1"].annotation + BandModel = t.cast( + t.Type[pydantic.BaseModel], + MyConcertModel.model_fields["band_1"].annotation, + ) self.assertEqual(BandModel.__qualname__, "MyConcertModel.band_1") ManagerModel = BandModel.model_fields["manager"].annotation @@ -763,7 +789,7 @@ class Concert(Table): ManagerModel.__qualname__, "MyConcertModel.band_1.manager" ) - def test_cascaded_args(self): + def test_cascaded_args(self) -> None: """ Make sure that arguments passed to ``create_pydantic_model`` are cascaded to nested models. @@ -784,14 +810,20 @@ class Band(Table): table=Band, nested=True, include_default_columns=True ) - ManagerModel = BandModel.model_fields["manager"].annotation + ManagerModel = t.cast( + t.Type[pydantic.BaseModel], + BandModel.model_fields["manager"].annotation, + ) self.assertTrue(issubclass(ManagerModel, pydantic.BaseModel)) self.assertEqual( [i for i in ManagerModel.model_fields.keys()], ["id", "name", "country"], ) - CountryModel = ManagerModel.model_fields["country"].annotation + CountryModel = t.cast( + t.Type[pydantic.BaseModel], + ManagerModel.model_fields["country"].annotation, + ) self.assertTrue(issubclass(CountryModel, pydantic.BaseModel)) self.assertEqual( [i for i in CountryModel.model_fields.keys()], ["id", "name"] @@ -823,13 +855,22 @@ class Concert(Table): table=Concert, nested=True, max_recursion_depth=2 ) - VenueModel = ConcertModel.model_fields["venue"].annotation + VenueModel = t.cast( + t.Type[pydantic.BaseModel], + ConcertModel.model_fields["venue"].annotation, + ) self.assertTrue(issubclass(VenueModel, pydantic.BaseModel)) - BandModel = ConcertModel.model_fields["band"].annotation + BandModel = t.cast( + t.Type[pydantic.BaseModel], + ConcertModel.model_fields["band"].annotation, + ) self.assertTrue(issubclass(BandModel, pydantic.BaseModel)) - ManagerModel = BandModel.model_fields["manager"].annotation + ManagerModel = t.cast( + t.Type[pydantic.BaseModel], + BandModel.model_fields["manager"].annotation, + ) self.assertTrue(issubclass(ManagerModel, pydantic.BaseModel)) # We should have hit the recursion depth: @@ -851,7 +892,7 @@ class Band(Table): model = BandModel(regrettable_column_name="test") - self.assertEqual(model.name, "test") + self.assertEqual(model.name, "test") # type: ignore class TestJSONSchemaExtra(TestCase): @@ -885,7 +926,7 @@ class Band(Table): config: pydantic.config.ConfigDict = {"extra": "forbid"} model = create_pydantic_model(Band, pydantic_config=config) - self.assertEqual(model.model_config["extra"], "forbid") + self.assertEqual(model.model_config.get("extra"), "forbid") def test_pydantic_invalid_extra_fields(self) -> None: """ From 2e40e68084c43edc06a78f527f3aaa5fc2680f58 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 14 Jun 2024 12:56:37 +0100 Subject: [PATCH 582/727] bumped version --- CHANGES.rst | 22 ++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 8d0790f25..2f0958783 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,28 @@ Changes ======= +1.10.0 +------ + +Added ``not_any`` method for ``Array`` columns. This will return rows where an +array doesn't contain the given value. For example: + +.. code-block:: python + + class MyTable(Table): + array_column = Array(Integer()) + + >>> await MyTable.select( + ... MyTable.array_column + ... ).where( + ... MyTable.array_column.not_any(1) + ... ) + [{"array_column": [4, 5, 6]}] + +Also fixed a bunch of Pylance linter warnings across the codebase. + +------------------------------------------------------------------------------- + 1.9.0 ----- diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 26de05016..f900dabe9 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.9.0" +__VERSION__ = "1.10.0" From bf36f50feed0d12f68129471fe35614f9d50c4a6 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 15 Jun 2024 00:33:52 +0100 Subject: [PATCH 583/727] 856 Be able to extract parts of timestamps / dates (#1023) * added extract function * add `Strftime` and database agnostic functions * reduce repetition * move alias to outer * add tests * add tests for `Extract` and `Strftime` --- docs/src/piccolo/functions/datetime.rst | 67 ++++++ docs/src/piccolo/functions/index.rst | 1 + piccolo/query/functions/__init__.py | 9 + piccolo/query/functions/datetime.py | 260 ++++++++++++++++++++++++ requirements/doc-requirements.txt | 4 +- tests/query/functions/test_datetime.py | 114 +++++++++++ 6 files changed, 453 insertions(+), 2 deletions(-) create mode 100644 docs/src/piccolo/functions/datetime.rst create mode 100644 piccolo/query/functions/datetime.py create mode 100644 tests/query/functions/test_datetime.py diff --git a/docs/src/piccolo/functions/datetime.rst b/docs/src/piccolo/functions/datetime.rst new file mode 100644 index 000000000..ee33e8c4a --- /dev/null +++ b/docs/src/piccolo/functions/datetime.rst @@ -0,0 +1,67 @@ +Datetime functions +================== + +.. currentmodule:: piccolo.query.functions.datetime + +Postgres / Cockroach +-------------------- + +Extract +~~~~~~~ + +.. autoclass:: Extract + + +SQLite +------ + +Strftime +~~~~~~~~ + +.. autoclass:: Strftime + + +Database agnostic +----------------- + +These convenience functions work consistently across database engines. + +They all work very similarly, for example: + +.. code-block:: python + + >>> from piccolo.query.functions import Year + >>> await Concert.select( + ... Year(Concert.starts, alias="start_year") + ... ) + [{"start_year": 2024}] + +Year +~~~~ + +.. autofunction:: Year + +Month +~~~~~ + +.. autofunction:: Month + +Day +~~~ + +.. autofunction:: Day + +Hour +~~~~ + +.. autofunction:: Hour + +Minute +~~~~~~ + +.. autofunction:: Minute + +Second +~~~~~~ + +.. autofunction:: Second diff --git a/docs/src/piccolo/functions/index.rst b/docs/src/piccolo/functions/index.rst index 93b3fe4f7..da3dfe43d 100644 --- a/docs/src/piccolo/functions/index.rst +++ b/docs/src/piccolo/functions/index.rst @@ -9,5 +9,6 @@ Functions can be used to modify how queries are run, and what is returned. ./basic_usage ./string ./math + ./datetime ./type_conversion ./aggregate diff --git a/piccolo/query/functions/__init__.py b/piccolo/query/functions/__init__.py index 9b83eca7b..8f8944d32 100644 --- a/piccolo/query/functions/__init__.py +++ b/piccolo/query/functions/__init__.py @@ -1,4 +1,5 @@ from .aggregate import Avg, Count, Max, Min, Sum +from .datetime import Day, Extract, Hour, Month, Second, Strftime, Year from .math import Abs, Ceil, Floor, Round from .string import Length, Lower, Ltrim, Reverse, Rtrim, Upper from .type_conversion import Cast @@ -9,15 +10,23 @@ "Cast", "Ceil", "Count", + "Day", + "Extract", + "Extract", "Floor", + "Hour", "Length", "Lower", "Ltrim", "Max", "Min", + "Month", "Reverse", "Round", "Rtrim", + "Second", + "Strftime", "Sum", "Upper", + "Year", ) diff --git a/piccolo/query/functions/datetime.py b/piccolo/query/functions/datetime.py new file mode 100644 index 000000000..130f846ce --- /dev/null +++ b/piccolo/query/functions/datetime.py @@ -0,0 +1,260 @@ +import typing as t + +from piccolo.columns.base import Column +from piccolo.columns.column_types import ( + Date, + Integer, + Time, + Timestamp, + Timestamptz, +) +from piccolo.querystring import QueryString + +from .type_conversion import Cast + +############################################################################### +# Postgres / Cockroach + +ExtractComponent = t.Literal[ + "century", + "day", + "decade", + "dow", + "doy", + "epoch", + "hour", + "isodow", + "isoyear", + "julian", + "microseconds", + "millennium", + "milliseconds", + "minute", + "month", + "quarter", + "second", + "timezone", + "timezone_hour", + "timezone_minute", + "week", + "year", +] + + +class Extract(QueryString): + def __init__( + self, + identifier: t.Union[Date, Time, Timestamp, Timestamptz, QueryString], + datetime_component: ExtractComponent, + alias: t.Optional[str] = None, + ): + """ + .. note:: This is for Postgres / Cockroach only. + + Extract a date or time component from a ``Date`` / ``Time`` / + ``Timestamp`` / ``Timestamptz`` column. For example, getting the month + from a timestamp: + + .. code-block:: python + + >>> from piccolo.query.functions import Extract + >>> await Concert.select( + ... Extract(Concert.starts, "month", alias="start_month") + ... ) + [{"start_month": 12}] + + :param identifier: + Identifies the column. + :param datetime_component: + The date or time component to extract from the column. + + """ + if datetime_component.lower() not in t.get_args(ExtractComponent): + raise ValueError("The date time component isn't recognised.") + + super().__init__( + f"EXTRACT({datetime_component} FROM {{}})", + identifier, + alias=alias, + ) + + +############################################################################### +# SQLite + + +class Strftime(QueryString): + def __init__( + self, + identifier: t.Union[Date, Time, Timestamp, Timestamptz, QueryString], + datetime_format: str, + alias: t.Optional[str] = None, + ): + """ + .. note:: This is for SQLite only. + + Format a datetime value. For example: + + .. code-block:: python + + >>> from piccolo.query.functions import Strftime + >>> await Concert.select( + ... Strftime(Concert.starts, "%Y", alias="start_year") + ... ) + [{"start_month": "2024"}] + + :param identifier: + Identifies the column. + :param datetime_format: + A string describing the output format (see SQLite's + `documentation `_ + for more info). + + """ + super().__init__( + f"strftime('{datetime_format}', {{}})", + identifier, + alias=alias, + ) + + +############################################################################### +# Database agnostic + + +def _get_engine_type(identifier: t.Union[Column, QueryString]) -> str: + if isinstance(identifier, Column): + return identifier._meta.engine_type + elif isinstance(identifier, QueryString) and ( + columns := identifier.columns + ): + return columns[0]._meta.engine_type + else: + raise ValueError("Unable to determine the engine type") + + +def _extract_component( + identifier: t.Union[Date, Time, Timestamp, Timestamptz, QueryString], + sqlite_format: str, + postgres_format: ExtractComponent, + alias: t.Optional[str], +): + engine_type = _get_engine_type(identifier=identifier) + + return Cast( + ( + Strftime( + identifier=identifier, + datetime_format=sqlite_format, + ) + if engine_type == "sqlite" + else Extract( + identifier=identifier, + datetime_component=postgres_format, + ) + ), + Integer(), + alias=alias, + ) + + +def Year( + identifier: t.Union[Date, Timestamp, Timestamptz, QueryString], + alias: t.Optional[str] = None, +): + """ + Extract the year as an integer. + """ + return _extract_component( + identifier=identifier, + sqlite_format="%Y", + postgres_format="year", + alias=alias, + ) + + +def Month( + identifier: t.Union[Date, Timestamp, Timestamptz, QueryString], + alias: t.Optional[str] = None, +): + """ + Extract the month as an integer. + """ + return _extract_component( + identifier=identifier, + sqlite_format="%m", + postgres_format="month", + alias=alias, + ) + + +def Day( + identifier: t.Union[Date, Timestamp, Timestamptz, QueryString], + alias: t.Optional[str] = None, +): + """ + Extract the day as an integer. + """ + return _extract_component( + identifier=identifier, + sqlite_format="%d", + postgres_format="day", + alias=alias, + ) + + +def Hour( + identifier: t.Union[Time, Timestamp, Timestamptz, QueryString], + alias: t.Optional[str] = None, +): + """ + Extract the hour as an integer. + """ + return _extract_component( + identifier=identifier, + sqlite_format="%H", + postgres_format="hour", + alias=alias, + ) + + +def Minute( + identifier: t.Union[Time, Timestamp, Timestamptz, QueryString], + alias: t.Optional[str] = None, +): + """ + Extract the minute as an integer. + """ + return _extract_component( + identifier=identifier, + sqlite_format="%M", + postgres_format="minute", + alias=alias, + ) + + +def Second( + identifier: t.Union[Time, Timestamp, Timestamptz, QueryString], + alias: t.Optional[str] = None, +): + """ + Extract the second as an integer. + """ + return _extract_component( + identifier=identifier, + sqlite_format="%S", + postgres_format="second", + alias=alias, + ) + + +__all__ = ( + "Extract", + "Strftime", + "Year", + "Month", + "Day", + "Hour", + "Minute", + "Second", +) diff --git a/requirements/doc-requirements.txt b/requirements/doc-requirements.txt index 3c23f8905..9e407e150 100644 --- a/requirements/doc-requirements.txt +++ b/requirements/doc-requirements.txt @@ -1,3 +1,3 @@ -Sphinx==5.1.1 -piccolo-theme>=0.12.0 +Sphinx==7.3.7 +piccolo-theme==0.22.0 sphinx-autobuild==2021.3.14 diff --git a/tests/query/functions/test_datetime.py b/tests/query/functions/test_datetime.py new file mode 100644 index 000000000..3e2d33d0b --- /dev/null +++ b/tests/query/functions/test_datetime.py @@ -0,0 +1,114 @@ +import datetime + +from piccolo.columns import Timestamp +from piccolo.query.functions.datetime import ( + Day, + Extract, + Hour, + Minute, + Month, + Second, + Strftime, + Year, +) +from piccolo.table import Table +from tests.base import engines_only, sqlite_only + +from .base import FunctionTest + + +class Concert(Table): + starts = Timestamp() + + +class DatetimeTest(FunctionTest): + tables = [Concert] + + def setUp(self) -> None: + super().setUp() + self.concert = Concert( + { + Concert.starts: datetime.datetime( + year=2024, month=6, day=14, hour=23, minute=46, second=10 + ) + } + ) + self.concert.save().run_sync() + + +@engines_only("postgres", "cockroach") +class TestExtract(DatetimeTest): + def test_extract(self): + self.assertEqual( + Concert.select( + Extract(Concert.starts, "year", alias="starts_year") + ).run_sync(), + [{"starts_year": self.concert.starts.year}], + ) + + def test_invalid_format(self): + with self.assertRaises(ValueError): + Extract( + Concert.starts, + "abc123", # type: ignore + alias="starts_year", + ) + + +@sqlite_only +class TestStrftime(DatetimeTest): + def test_strftime(self): + self.assertEqual( + Concert.select( + Strftime(Concert.starts, "%Y", alias="starts_year") + ).run_sync(), + [{"starts_year": str(self.concert.starts.year)}], + ) + + +class TestDatabaseAgnostic(DatetimeTest): + def test_year(self): + self.assertEqual( + Concert.select( + Year(Concert.starts, alias="starts_year") + ).run_sync(), + [{"starts_year": self.concert.starts.year}], + ) + + def test_month(self): + self.assertEqual( + Concert.select( + Month(Concert.starts, alias="starts_month") + ).run_sync(), + [{"starts_month": self.concert.starts.month}], + ) + + def test_day(self): + self.assertEqual( + Concert.select(Day(Concert.starts, alias="starts_day")).run_sync(), + [{"starts_day": self.concert.starts.day}], + ) + + def test_hour(self): + self.assertEqual( + Concert.select( + Hour(Concert.starts, alias="starts_hour") + ).run_sync(), + [{"starts_hour": self.concert.starts.hour}], + ) + + def test_minute(self): + self.assertEqual( + Concert.select( + Minute(Concert.starts, alias="starts_minute") + ).run_sync(), + [{"starts_minute": self.concert.starts.minute}], + ) + + def test_second(self): + self.assertEqual( + Concert.select( + Second(Concert.starts, alias="starts_second") + ).run_sync(), + [{"starts_second": self.concert.starts.second}], + ) From 13479c0fe9e5f8d7e89d7cdc5f41e84302f790e0 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 15 Jun 2024 00:37:26 +0100 Subject: [PATCH 584/727] add missing return types --- piccolo/query/functions/datetime.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/piccolo/query/functions/datetime.py b/piccolo/query/functions/datetime.py index 130f846ce..678355164 100644 --- a/piccolo/query/functions/datetime.py +++ b/piccolo/query/functions/datetime.py @@ -138,7 +138,7 @@ def _extract_component( sqlite_format: str, postgres_format: ExtractComponent, alias: t.Optional[str], -): +) -> QueryString: engine_type = _get_engine_type(identifier=identifier) return Cast( @@ -161,7 +161,7 @@ def _extract_component( def Year( identifier: t.Union[Date, Timestamp, Timestamptz, QueryString], alias: t.Optional[str] = None, -): +) -> QueryString: """ Extract the year as an integer. """ @@ -176,7 +176,7 @@ def Year( def Month( identifier: t.Union[Date, Timestamp, Timestamptz, QueryString], alias: t.Optional[str] = None, -): +) -> QueryString: """ Extract the month as an integer. """ @@ -191,7 +191,7 @@ def Month( def Day( identifier: t.Union[Date, Timestamp, Timestamptz, QueryString], alias: t.Optional[str] = None, -): +) -> QueryString: """ Extract the day as an integer. """ @@ -206,7 +206,7 @@ def Day( def Hour( identifier: t.Union[Time, Timestamp, Timestamptz, QueryString], alias: t.Optional[str] = None, -): +) -> QueryString: """ Extract the hour as an integer. """ @@ -221,7 +221,7 @@ def Hour( def Minute( identifier: t.Union[Time, Timestamp, Timestamptz, QueryString], alias: t.Optional[str] = None, -): +) -> QueryString: """ Extract the minute as an integer. """ @@ -236,7 +236,7 @@ def Minute( def Second( identifier: t.Union[Time, Timestamp, Timestamptz, QueryString], alias: t.Optional[str] = None, -): +) -> QueryString: """ Extract the second as an integer. """ From 98ad5d5a4fe376770b667cd55023d9f1b5195001 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 15 Jun 2024 20:53:33 +0100 Subject: [PATCH 585/727] 1024 Add `Concat` function (#1025) * add `Concat` function * update concat delegate * go back to using the concat operator in concat delegate to keep behaviour consistent * add tests * fix cockroachdb test * fix mypy error * add notes that concat isn't available in older sqlite versions * move decorator to right file --- docs/src/piccolo/functions/string.rst | 5 +++ piccolo/columns/column_types.py | 63 ++++++++++++--------------- piccolo/query/functions/__init__.py | 3 +- piccolo/query/functions/string.py | 45 +++++++++++++++++++ tests/query/functions/test_string.py | 36 ++++++++++++++- 5 files changed, 114 insertions(+), 38 deletions(-) diff --git a/docs/src/piccolo/functions/string.rst b/docs/src/piccolo/functions/string.rst index dbc09125a..8d991c956 100644 --- a/docs/src/piccolo/functions/string.rst +++ b/docs/src/piccolo/functions/string.rst @@ -3,6 +3,11 @@ String functions .. currentmodule:: piccolo.query.functions.string +Concat +------ + +.. autoclass:: Concat + Length ------ diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 9c74b4a52..6140f6d13 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -86,47 +86,38 @@ class ConcatDelegate: def get_querystring( self, - column_name: str, - value: t.Union[str, Varchar, Text], + column: Column, + value: t.Union[str, Column, QueryString], reverse: bool = False, ) -> QueryString: - if isinstance(value, (Varchar, Text)): - column: Column = value + """ + :param reverse: + By default the value is appended to the column's value. If + ``reverse=True`` then the value is prepended to the column's + value instead. + + """ + if isinstance(value, Column): if len(column._meta.call_chain) > 0: raise ValueError( "Adding values across joins isn't currently supported." ) - other_column_name = column._meta.db_column_name - if reverse: - return QueryString( - Concat.template.format( - value_1=other_column_name, value_2=column_name - ) - ) - else: - return QueryString( - Concat.template.format( - value_1=column_name, value_2=other_column_name - ) - ) elif isinstance(value, str): - if reverse: - value_1 = QueryString("CAST({} AS text)", value) - return QueryString( - Concat.template.format(value_1="{}", value_2=column_name), - value_1, - ) - else: - value_2 = QueryString("CAST({} AS text)", value) - return QueryString( - Concat.template.format(value_1=column_name, value_2="{}"), - value_2, - ) - else: + value = QueryString("CAST({} AS TEXT)", value) + elif not isinstance(value, QueryString): raise ValueError( - "Only str, Varchar columns, and Text columns can be added." + "Only str, Column and QueryString values can be added." ) + args = [value, column] if reverse else [column, value] + + # We use the concat operator instead of the concat function, because + # this is what we historically used, and they treat null values + # differently. + return QueryString( + Concat.template.format(value_1="{}", value_2="{}"), *args + ) + class MathDelegate: """ @@ -340,12 +331,13 @@ def column_type(self): def __add__(self, value: t.Union[str, Varchar, Text]) -> QueryString: return self.concat_delegate.get_querystring( - column_name=self._meta.db_column_name, value=value + column=self, + value=value, ) def __radd__(self, value: t.Union[str, Varchar, Text]) -> QueryString: return self.concat_delegate.get_querystring( - column_name=self._meta.db_column_name, + column=self, value=value, reverse=True, ) @@ -442,12 +434,13 @@ def __init__( def __add__(self, value: t.Union[str, Varchar, Text]) -> QueryString: return self.concat_delegate.get_querystring( - column_name=self._meta.db_column_name, value=value + column=self, + value=value, ) def __radd__(self, value: t.Union[str, Varchar, Text]) -> QueryString: return self.concat_delegate.get_querystring( - column_name=self._meta.db_column_name, + column=self, value=value, reverse=True, ) diff --git a/piccolo/query/functions/__init__.py b/piccolo/query/functions/__init__.py index 8f8944d32..3163f6d1c 100644 --- a/piccolo/query/functions/__init__.py +++ b/piccolo/query/functions/__init__.py @@ -1,7 +1,7 @@ from .aggregate import Avg, Count, Max, Min, Sum from .datetime import Day, Extract, Hour, Month, Second, Strftime, Year from .math import Abs, Ceil, Floor, Round -from .string import Length, Lower, Ltrim, Reverse, Rtrim, Upper +from .string import Concat, Length, Lower, Ltrim, Reverse, Rtrim, Upper from .type_conversion import Cast __all__ = ( @@ -9,6 +9,7 @@ "Avg", "Cast", "Ceil", + "Concat", "Count", "Day", "Extract", diff --git a/piccolo/query/functions/string.py b/piccolo/query/functions/string.py index 556817a12..68b78219f 100644 --- a/piccolo/query/functions/string.py +++ b/piccolo/query/functions/string.py @@ -5,6 +5,12 @@ """ +import typing as t + +from piccolo.columns.base import Column +from piccolo.columns.column_types import Text, Varchar +from piccolo.querystring import QueryString + from .base import Function @@ -63,6 +69,44 @@ class Upper(Function): function_name = "UPPER" +class Concat(QueryString): + def __init__( + self, + *args: t.Union[Column, QueryString, str], + alias: t.Optional[str] = None, + ): + """ + Concatenate multiple values into a single string. + + .. note:: + Null values are ignored, so ``null + '!!!'`` returns ``!!!``, + not ``null``. + + .. warning:: + For SQLite, this is only available in version 3.44.0 and above. + + """ + if len(args) < 2: + raise ValueError("At least two values must be passed in.") + + placeholders = ", ".join("{}" for _ in args) + + processed_args: t.List[t.Union[QueryString, Column]] = [] + + for arg in args: + if isinstance(arg, str) or ( + isinstance(arg, Column) + and not isinstance(arg, (Varchar, Text)) + ): + processed_args.append(QueryString("CAST({} AS TEXT)", arg)) + else: + processed_args.append(arg) + + super().__init__( + f"CONCAT({placeholders})", *processed_args, alias=alias + ) + + __all__ = ( "Length", "Lower", @@ -70,4 +114,5 @@ class Upper(Function): "Reverse", "Rtrim", "Upper", + "Concat", ) diff --git a/tests/query/functions/test_string.py b/tests/query/functions/test_string.py index b87952634..bd3a8c2ab 100644 --- a/tests/query/functions/test_string.py +++ b/tests/query/functions/test_string.py @@ -1,10 +1,13 @@ -from piccolo.query.functions.string import Upper +import pytest + +from piccolo.query.functions.string import Concat, Upper +from tests.base import engine_version_lt, is_running_sqlite from tests.example_apps.music.tables import Band from .base import BandTest -class TestUpperFunction(BandTest): +class TestUpper(BandTest): def test_column(self): """ @@ -23,3 +26,32 @@ def test_joined_column(self): """ response = Band.select(Upper(Band.manager._.name)).run_sync() self.assertListEqual(response, [{"upper": "GUIDO"}]) + + +@pytest.mark.skipif( + is_running_sqlite() and engine_version_lt(3.44), + reason="SQLite version not supported", +) +class TestConcat(BandTest): + + def test_column_and_string(self): + response = Band.select( + Concat(Band.name, "!!!", alias="name") + ).run_sync() + self.assertListEqual(response, [{"name": "Pythonistas!!!"}]) + + def test_column_and_column(self): + response = Band.select( + Concat(Band.name, Band.popularity, alias="name") + ).run_sync() + self.assertListEqual(response, [{"name": "Pythonistas1000"}]) + + def test_join(self): + response = Band.select( + Concat(Band.name, "-", Band.manager._.name, alias="name") + ).run_sync() + self.assertListEqual(response, [{"name": "Pythonistas-Guido"}]) + + def test_min_args(self): + with self.assertRaises(ValueError): + Concat() From b4d9b26186eabf03a2d3c0ed584fb77e621fe88c Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 15 Jun 2024 21:19:05 +0100 Subject: [PATCH 586/727] bumped version --- CHANGES.rst | 27 +++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2f0958783..70f36016f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,33 @@ Changes ======= +1.11.0 +------ + +Added datetime functions, for example ``Year``: + +.. code-block:: python + + >>> from piccolo.query.functions import Year + >>> await Concert.select(Year(Concert.starts, alias="starts_year")) + [{'starts_year': 2024}] + +Added the ``Concat`` function, for concatenating strings: + +.. code-block:: python + + >>> await Band.select( + ... Concat( + ... Band.name, + ... '-', + ... Band.manager._.name, + ... alias="name_and_manager" + ... ) + ... ) + [{"name_and_manager": "Pythonistas-Guido"}] + +------------------------------------------------------------------------------- + 1.10.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index f900dabe9..05dd9013c 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.10.0" +__VERSION__ = "1.11.0" From c4f34f07cd96c6c0f3c1d0b3ee9dd85b08d35038 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 15 Jun 2024 21:22:13 +0100 Subject: [PATCH 587/727] add missing import in changelog --- CHANGES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.rst b/CHANGES.rst index 70f36016f..f451e0153 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -16,6 +16,7 @@ Added the ``Concat`` function, for concatenating strings: .. code-block:: python + >>> from piccolo.query.functions import Concat >>> await Band.select( ... Concat( ... Band.name, From 4e38cb7fbd1439620e0d9750399a328e97d36710 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 15 Jun 2024 21:54:52 +0100 Subject: [PATCH 588/727] Update doc-requirements.txt --- requirements/doc-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/doc-requirements.txt b/requirements/doc-requirements.txt index 9e407e150..493c9b46b 100644 --- a/requirements/doc-requirements.txt +++ b/requirements/doc-requirements.txt @@ -1,3 +1,3 @@ Sphinx==7.3.7 -piccolo-theme==0.22.0 +piccolo-theme==0.23.0 sphinx-autobuild==2021.3.14 From 2335b958e4a286181c05df07279dd7e4251c2efa Mon Sep 17 00:00:00 2001 From: sinisaos Date: Sun, 16 Jun 2024 13:12:28 +0200 Subject: [PATCH 589/727] replace deprecated events in asgi templates (#1027) --- .../templates/app/_fastapi_app.py.jinja | 65 ++++++++++--------- .../templates/app/_starlette_app.py.jinja | 50 ++++++++------ 2 files changed, 64 insertions(+), 51 deletions(-) diff --git a/piccolo/apps/asgi/commands/templates/app/_fastapi_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_fastapi_app.py.jinja index c2408223b..0fd9efac0 100644 --- a/piccolo/apps/asgi/commands/templates/app/_fastapi_app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/_fastapi_app.py.jinja @@ -1,11 +1,12 @@ import typing as t +from contextlib import asynccontextmanager from fastapi import FastAPI from fastapi.responses import JSONResponse +from piccolo.engine import engine_finder from piccolo_admin.endpoints import create_admin from piccolo_api.crud.serializers import create_pydantic_model -from piccolo.engine import engine_finder -from starlette.routing import Route, Mount +from starlette.routing import Mount, Route from starlette.staticfiles import StaticFiles from home.endpoints import HomeEndpoint @@ -13,6 +14,29 @@ from home.piccolo_app import APP_CONFIG from home.tables import Task +async def open_database_connection_pool(): + try: + engine = engine_finder() + await engine.start_connection_pool() + except Exception: + print("Unable to connect to the database") + + +async def close_database_connection_pool(): + try: + engine = engine_finder() + await engine.close_connection_pool() + except Exception: + print("Unable to connect to the database") + + +@asynccontextmanager +async def lifespan(app: FastAPI): + await open_database_connection_pool() + yield + await close_database_connection_pool() + + app = FastAPI( routes=[ Route("/", HomeEndpoint), @@ -22,21 +46,20 @@ app = FastAPI( tables=APP_CONFIG.table_classes, # Required when running under HTTPS: # allowed_hosts=['my_site.com'] - ) + ), ), Mount("/static/", StaticFiles(directory="static")), ], + lifespan=lifespan, ) TaskModelIn: t.Any = create_pydantic_model( table=Task, - model_name='TaskModelIn' + model_name="TaskModelIn", ) TaskModelOut: t.Any = create_pydantic_model( - table=Task, - include_default_columns=True, - model_name='TaskModelOut' + table=Task, include_default_columns=True, model_name="TaskModelOut" ) @@ -45,16 +68,16 @@ async def tasks(): return await Task.select().order_by(Task.id) -@app.post('/tasks/', response_model=TaskModelOut) +@app.post("/tasks/", response_model=TaskModelOut) async def create_task(task_model: TaskModelIn): task = Task(**task_model.dict()) await task.save() return task.to_dict() -@app.put('/tasks/{task_id}/', response_model=TaskModelOut) +@app.put("/tasks/{task_id}/", response_model=TaskModelOut) async def update_task(task_id: int, task_model: TaskModelIn): - task = await Task.objects().get(Task.id == task_id) + task = await Task.objects().get(Task._meta.primary_key == task_id) if not task: return JSONResponse({}, status_code=404) @@ -66,30 +89,12 @@ async def update_task(task_id: int, task_model: TaskModelIn): return task.to_dict() -@app.delete('/tasks/{task_id}/') +@app.delete("/tasks/{task_id}/") async def delete_task(task_id: int): - task = await Task.objects().get(Task.id == task_id) + task = await Task.objects().get(Task._meta.primary_key == task_id) if not task: return JSONResponse({}, status_code=404) await task.remove() return JSONResponse({}) - - -@app.on_event("startup") -async def open_database_connection_pool(): - try: - engine = engine_finder() - await engine.start_connection_pool() - except Exception: - print("Unable to connect to the database") - - -@app.on_event("shutdown") -async def close_database_connection_pool(): - try: - engine = engine_finder() - await engine.close_connection_pool() - except Exception: - print("Unable to connect to the database") diff --git a/piccolo/apps/asgi/commands/templates/app/_starlette_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_starlette_app.py.jinja index 1c401bf43..de99c8e15 100644 --- a/piccolo/apps/asgi/commands/templates/app/_starlette_app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/_starlette_app.py.jinja @@ -1,8 +1,10 @@ +from contextlib import asynccontextmanager + +from piccolo.engine import engine_finder from piccolo_admin.endpoints import create_admin from piccolo_api.crud.endpoints import PiccoloCRUD -from piccolo.engine import engine_finder -from starlette.routing import Route, Mount from starlette.applications import Starlette +from starlette.routing import Mount, Route from starlette.staticfiles import StaticFiles from home.endpoints import HomeEndpoint @@ -10,24 +12,6 @@ from home.piccolo_app import APP_CONFIG from home.tables import Task -app = Starlette( - routes=[ - Route("/", HomeEndpoint), - Mount( - "/admin/", - create_admin( - tables=APP_CONFIG.table_classes, - # Required when running under HTTPS: - # allowed_hosts=['my_site.com'] - ) - ), - Mount("/static/", StaticFiles(directory="static")), - Mount("/tasks/", PiccoloCRUD(table=Task)) - ], -) - - -@app.on_event("startup") async def open_database_connection_pool(): try: engine = engine_finder() @@ -36,10 +20,34 @@ async def open_database_connection_pool(): print("Unable to connect to the database") -@app.on_event("shutdown") async def close_database_connection_pool(): try: engine = engine_finder() await engine.close_connection_pool() except Exception: print("Unable to connect to the database") + + +@asynccontextmanager +async def lifespan(app: Starlette): + await open_database_connection_pool() + yield + await close_database_connection_pool() + + +app = Starlette( + routes=[ + Route("/", HomeEndpoint), + Mount( + "/admin/", + create_admin( + tables=APP_CONFIG.table_classes, + # Required when running under HTTPS: + # allowed_hosts=['my_site.com'] + ), + ), + Mount("/static/", StaticFiles(directory="static")), + Mount("/tasks/", PiccoloCRUD(table=Task)), + ], + lifespan=lifespan, +) From 4be8aecbc6bb92395b729f250514b7560479fc4c Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 19 Jun 2024 12:28:37 +0100 Subject: [PATCH 590/727] 1028 Document how to achieve a `OneToOneField` in Piccolo (#1029) * add reverse method * one to one field docs * add FanClub to playground * first attempt at making reverse work on multiple levels * make reverse work multiple levels deep * remove unused import * more on to one to its own page * break up the one to one docs into sections * final tweaks to docs --- .../piccolo/query_types/django_comparison.rst | 10 +++ docs/src/piccolo/schema/index.rst | 1 + docs/src/piccolo/schema/one_to_one.rst | 85 +++++++++++++++++++ piccolo/apps/playground/commands/run.py | 11 +++ piccolo/columns/column_types.py | 55 ++++++++++++ tests/base.py | 21 ++++- tests/columns/foreign_key/test_reverse.py | 56 ++++++++++++ tests/query/functions/base.py | 17 +--- tests/query/functions/test_datetime.py | 6 +- tests/query/functions/test_math.py | 5 +- 10 files changed, 244 insertions(+), 23 deletions(-) create mode 100644 docs/src/piccolo/schema/one_to_one.rst create mode 100644 tests/columns/foreign_key/test_reverse.py diff --git a/docs/src/piccolo/query_types/django_comparison.rst b/docs/src/piccolo/query_types/django_comparison.rst index 76734a3a5..d16c03d57 100644 --- a/docs/src/piccolo/query_types/django_comparison.rst +++ b/docs/src/piccolo/query_types/django_comparison.rst @@ -177,6 +177,16 @@ Piccolo has something similar: >>> band.manager +------------------------------------------------------------------------------- + +Schema +------ + +OneToOneField +~~~~~~~~~~~~~ + +To do this in Piccolo, use a ``ForeignKey`` with a unique constraint - see +:ref:`One to One`. ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/schema/index.rst b/docs/src/piccolo/schema/index.rst index 36ea8b807..ec9b887e6 100644 --- a/docs/src/piccolo/schema/index.rst +++ b/docs/src/piccolo/schema/index.rst @@ -9,4 +9,5 @@ The schema is how you define your database tables, columns and relationships. ./defining ./column_types ./m2m + ./one_to_one ./advanced diff --git a/docs/src/piccolo/schema/one_to_one.rst b/docs/src/piccolo/schema/one_to_one.rst new file mode 100644 index 000000000..f35433189 --- /dev/null +++ b/docs/src/piccolo/schema/one_to_one.rst @@ -0,0 +1,85 @@ +.. _OneToOne: + +One to One +========== + +Schema +------ + +A one to one relationship is basically just a foreign key with a unique +constraint. In Piccolo, you can do it like this: + +.. code-block:: python + + from piccolo.table import Table + from piccolo.columns import ForeignKey, Varchar, Text + + class Band(Table): + name = Varchar() + + class FanClub(Table): + band = ForeignKey(Band, unique=True) # <- Note the unique constraint + address = Text() + +Queries +------- + +Getting a related object +~~~~~~~~~~~~~~~~~~~~~~~~ + +If we have a ``Band`` object: + +.. code-block:: python + + band = await Band.objects().where(Band.name == "Pythonistas").first() + +To get the associated ``FanClub`` object, you could do this: + +.. code-block:: python + + fan_club = await FanClub.objects().where(FanClub.band == band).first() + +Or alternatively, using ``get_related``: + +.. code-block:: python + + fan_club = await band.get_related(Band.id.join_on(FanClub.band)) + +Instead of using ``join_on``, you can use ``reverse`` to traverse the foreign +key backwards if you prefer: + +.. code-block:: python + + fan_club = await band.get_related(FanClub.band.reverse()) + +Select +~~~~~~ + +If doing a select query, and you want data from the related table: + +.. code-block:: python + + >>> await Band.select( + ... Band.name, + ... Band.id.join_on(FanClub.band).address.as_alias("address") + ... ) + [{'name': 'Pythonistas', 'address': '1 Flying Circus, UK'}, ...] + +Where +~~~~~ + +If you want to filter by related tables in the ``where`` clause: + +.. code-block:: python + + >>> await Band.select( + ... Band.name, + ... ).where(Band.id.join_on(FanClub.band).address.like("%Flying%")) + [{'name': 'Pythonistas'}] + +Source +------ + +.. currentmodule:: piccolo.columns.column_types + +.. automethod:: ForeignKey.reverse diff --git a/piccolo/apps/playground/commands/run.py b/piccolo/apps/playground/commands/run.py index e1bc00130..3f435c44b 100644 --- a/piccolo/apps/playground/commands/run.py +++ b/piccolo/apps/playground/commands/run.py @@ -19,6 +19,7 @@ Interval, Numeric, Serial, + Text, Timestamp, Varchar, ) @@ -55,6 +56,12 @@ def get_readable(cls) -> Readable: ) +class FanClub(Table): + id: Serial + address = Text() + band = ForeignKey(Band, unique=True) + + class Venue(Table): id: Serial name = Varchar(length=100) @@ -154,6 +161,7 @@ def get_readable(cls) -> Readable: TABLES = ( Manager, Band, + FanClub, Venue, Concert, Ticket, @@ -185,6 +193,9 @@ def populate(): pythonistas = Band(name="Pythonistas", manager=guido.id, popularity=1000) pythonistas.save().run_sync() + fan_club = FanClub(address="1 Flying Circus, UK", band=pythonistas) + fan_club.save().run_sync() + graydon = Manager(name="Graydon") graydon.save().run_sync() diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 6140f6d13..40d0c945e 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -2016,6 +2016,61 @@ def all_columns( if column._meta.name not in excluded_column_names ] + def reverse(self) -> ForeignKey: + """ + If there's a unique foreign key, this function reverses it. + + .. code-block:: python + + class Band(Table): + name = Varchar() + + class FanClub(Table): + band = ForeignKey(Band, unique=True) + address = Text() + + class Treasurer(Table): + fan_club = ForeignKey(FanClub, unique=True) + name = Varchar() + + It's helpful with ``get_related``, for example: + + .. code-block:: python + + >>> band = await Band.objects().first() + >>> await band.get_related(FanClub.band.reverse()) + + + It works multiple levels deep: + + .. code-block:: python + + >>> await band.get_related(Treasurer.fan_club._.band.reverse()) + + + """ + if not self._meta.unique or any( + not i._meta.unique for i in self._meta.call_chain + ): + raise ValueError("Only reverse unique foreign keys.") + + foreign_keys = [*self._meta.call_chain, self] + + root_foreign_key = foreign_keys[0] + target_column = ( + root_foreign_key._foreign_key_meta.resolved_target_column + ) + foreign_key = target_column.join_on(root_foreign_key) + + call_chain = [] + for fk in reversed(foreign_keys[1:]): + target_column = fk._foreign_key_meta.resolved_target_column + call_chain.append(target_column.join_on(fk)) + + foreign_key._meta.call_chain = call_chain + + return foreign_key + def all_related( self, exclude: t.Optional[t.List[t.Union[ForeignKey, str]]] = None ) -> t.List[ForeignKey]: diff --git a/tests/base.py b/tests/base.py index b05f85622..f9f964c70 100644 --- a/tests/base.py +++ b/tests/base.py @@ -13,7 +13,12 @@ from piccolo.engine.finder import engine_finder from piccolo.engine.postgres import PostgresEngine from piccolo.engine.sqlite import SQLiteEngine -from piccolo.table import Table, create_table_class +from piccolo.table import ( + Table, + create_db_tables_sync, + create_table_class, + drop_db_tables_sync, +) from piccolo.utils.sync import run_sync ENGINE = engine_finder() @@ -454,3 +459,17 @@ def setUp(self): def tearDown(self): self.drop_tables() + + +class TableTest(TestCase): + """ + Used for tests where we need to create Piccolo tables. + """ + + tables: t.List[t.Type[Table]] + + def setUp(self) -> None: + create_db_tables_sync(*self.tables) + + def tearDown(self) -> None: + drop_db_tables_sync(*self.tables) diff --git a/tests/columns/foreign_key/test_reverse.py b/tests/columns/foreign_key/test_reverse.py new file mode 100644 index 000000000..2a90ac5ba --- /dev/null +++ b/tests/columns/foreign_key/test_reverse.py @@ -0,0 +1,56 @@ +from piccolo.columns import ForeignKey, Text, Varchar +from piccolo.table import Table +from tests.base import TableTest + + +class Band(Table): + name = Varchar() + + +class FanClub(Table): + address = Text() + band = ForeignKey(Band, unique=True) + + +class Treasurer(Table): + name = Varchar() + fan_club = ForeignKey(FanClub, unique=True) + + +class TestReverse(TableTest): + tables = [Band, FanClub, Treasurer] + + def setUp(self): + super().setUp() + + band = Band({Band.name: "Pythonistas"}) + band.save().run_sync() + + fan_club = FanClub( + {FanClub.band: band, FanClub.address: "1 Flying Circus, UK"} + ) + fan_club.save().run_sync() + + treasurer = Treasurer( + {Treasurer.fan_club: fan_club, Treasurer.name: "Bob"} + ) + treasurer.save().run_sync() + + def test_reverse(self): + response = Band.select( + Band.name, + FanClub.band.reverse().address.as_alias("address"), + Treasurer.fan_club._.band.reverse().name.as_alias( + "treasurer_name" + ), + ).run_sync() + self.assertListEqual( + response, + [ + { + "name": "Pythonistas", + "address": "1 Flying Circus, UK", + "treasurer_name": "Bob", + } + ], + ) diff --git a/tests/query/functions/base.py b/tests/query/functions/base.py index 168f5528b..623bc1a5a 100644 --- a/tests/query/functions/base.py +++ b/tests/query/functions/base.py @@ -1,21 +1,8 @@ -import typing as t -from unittest import TestCase - -from piccolo.table import Table, create_db_tables_sync, drop_db_tables_sync +from tests.base import TableTest from tests.example_apps.music.tables import Band, Manager -class FunctionTest(TestCase): - tables: t.List[t.Type[Table]] - - def setUp(self) -> None: - create_db_tables_sync(*self.tables) - - def tearDown(self) -> None: - drop_db_tables_sync(*self.tables) - - -class BandTest(FunctionTest): +class BandTest(TableTest): tables = [Band, Manager] def setUp(self) -> None: diff --git a/tests/query/functions/test_datetime.py b/tests/query/functions/test_datetime.py index 3e2d33d0b..360833dc4 100644 --- a/tests/query/functions/test_datetime.py +++ b/tests/query/functions/test_datetime.py @@ -12,16 +12,14 @@ Year, ) from piccolo.table import Table -from tests.base import engines_only, sqlite_only - -from .base import FunctionTest +from tests.base import TableTest, engines_only, sqlite_only class Concert(Table): starts = Timestamp() -class DatetimeTest(FunctionTest): +class DatetimeTest(TableTest): tables = [Concert] def setUp(self) -> None: diff --git a/tests/query/functions/test_math.py b/tests/query/functions/test_math.py index 1c82f9426..7029e7857 100644 --- a/tests/query/functions/test_math.py +++ b/tests/query/functions/test_math.py @@ -3,15 +3,14 @@ from piccolo.columns import Numeric from piccolo.query.functions.math import Abs, Ceil, Floor, Round from piccolo.table import Table - -from .base import FunctionTest +from tests.base import TableTest class Ticket(Table): price = Numeric(digits=(5, 2)) -class TestMath(FunctionTest): +class TestMath(TableTest): tables = [Ticket] From d91d59c7f7cb9592ec00e1e42641d8b82ff228bd Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 19 Jun 2024 13:06:07 +0100 Subject: [PATCH 591/727] use `TableTest` (#1031) --- tests/columns/test_array.py | 10 ++--- tests/columns/test_bigint.py | 12 ++---- tests/columns/test_boolean.py | 10 ++--- tests/columns/test_bytea.py | 19 +++------ tests/columns/test_choices.py | 19 +++------ tests/columns/test_date.py | 18 +++------ tests/columns/test_double_precision.py | 11 ++---- tests/columns/test_interval.py | 18 +++------ tests/columns/test_json.py | 35 +++++----------- tests/columns/test_jsonb.py | 14 ++----- tests/columns/test_numeric.py | 10 ++--- tests/columns/test_primary_key.py | 44 ++++++--------------- tests/columns/test_readable.py | 12 +++--- tests/columns/test_real.py | 11 ++---- tests/columns/test_reserved_column_names.py | 11 ++---- tests/columns/test_smallint.py | 11 ++---- tests/columns/test_time.py | 19 +++------ tests/columns/test_timestamp.py | 18 +++------ tests/columns/test_timestamptz.py | 18 +++------ tests/columns/test_uuid.py | 10 ++--- tests/columns/test_varchar.py | 12 ++---- 21 files changed, 93 insertions(+), 249 deletions(-) diff --git a/tests/columns/test_array.py b/tests/columns/test_array.py index 604aecd51..b28204231 100644 --- a/tests/columns/test_array.py +++ b/tests/columns/test_array.py @@ -14,7 +14,7 @@ ) from piccolo.querystring import QueryString from piccolo.table import Table -from tests.base import engines_only, engines_skip, sqlite_only +from tests.base import TableTest, engines_only, engines_skip, sqlite_only class MyTable(Table): @@ -32,16 +32,12 @@ def test_array_default(self): self.assertTrue(column.default is list) -class TestArray(TestCase): +class TestArray(TableTest): """ Make sure an Array column can be created, and works correctly. """ - def setUp(self): - MyTable.create_table().run_sync() - - def tearDown(self): - MyTable.alter().drop_table().run_sync() + tables = [MyTable] @pytest.mark.cockroach_array_slow def test_storage(self): diff --git a/tests/columns/test_bigint.py b/tests/columns/test_bigint.py index 4ebd37de5..7f418cced 100644 --- a/tests/columns/test_bigint.py +++ b/tests/columns/test_bigint.py @@ -1,10 +1,8 @@ import os -from unittest import TestCase from piccolo.columns.column_types import BigInt from piccolo.table import Table - -from ..base import engines_only +from tests.base import TableTest, engines_only class MyTable(Table): @@ -12,16 +10,12 @@ class MyTable(Table): @engines_only("postgres", "cockroach") -class TestBigIntPostgres(TestCase): +class TestBigIntPostgres(TableTest): """ Make sure a BigInt column in Postgres can store a large number. """ - def setUp(self): - MyTable.create_table().run_sync() - - def tearDown(self): - MyTable.alter().drop_table().run_sync() + tables = [MyTable] def _test_length(self): # Can store 8 bytes, but split between positive and negative values. diff --git a/tests/columns/test_boolean.py b/tests/columns/test_boolean.py index 4c67ef6db..57268b24a 100644 --- a/tests/columns/test_boolean.py +++ b/tests/columns/test_boolean.py @@ -1,20 +1,16 @@ import typing as t -from unittest import TestCase from piccolo.columns.column_types import Boolean from piccolo.table import Table +from tests.base import TableTest class MyTable(Table): boolean = Boolean(boolean=False, null=True) -class TestBoolean(TestCase): - def setUp(self): - MyTable.create_table().run_sync() - - def tearDown(self): - MyTable.alter().drop_table().run_sync() +class TestBoolean(TableTest): + tables = [MyTable] def test_return_type(self) -> None: for value in (True, False, None, ...): diff --git a/tests/columns/test_bytea.py b/tests/columns/test_bytea.py index 6976a8840..4b520083f 100644 --- a/tests/columns/test_bytea.py +++ b/tests/columns/test_bytea.py @@ -1,7 +1,6 @@ -from unittest import TestCase - from piccolo.columns.column_types import Bytea from piccolo.table import Table +from tests.base import TableTest class MyTable(Table): @@ -19,12 +18,8 @@ class MyTableDefault(Table): token_none = Bytea(default=None, null=True) -class TestBytea(TestCase): - def setUp(self): - MyTable.create_table().run_sync() - - def tearDown(self): - MyTable.alter().drop_table().run_sync() +class TestBytea(TableTest): + tables = [MyTable] def test_bytea(self): """ @@ -40,12 +35,8 @@ def test_bytea(self): ) -class TestByteaDefault(TestCase): - def setUp(self): - MyTableDefault.create_table().run_sync() - - def tearDown(self): - MyTableDefault.alter().drop_table().run_sync() +class TestByteaDefault(TableTest): + tables = [MyTableDefault] def test_json_default(self): row = MyTableDefault() diff --git a/tests/columns/test_choices.py b/tests/columns/test_choices.py index d127a87d0..05502fe0a 100644 --- a/tests/columns/test_choices.py +++ b/tests/columns/test_choices.py @@ -1,18 +1,13 @@ import enum -from unittest import TestCase from piccolo.columns.column_types import Array, Varchar from piccolo.table import Table -from tests.base import engines_only +from tests.base import TableTest, engines_only from tests.example_apps.music.tables import Shirt -class TestChoices(TestCase): - def setUp(self): - Shirt.create_table().run_sync() - - def tearDown(self): - Shirt.alter().drop_table().run_sync() +class TestChoices(TableTest): + tables = [Shirt] def _insert_shirts(self): Shirt.insert( @@ -87,16 +82,12 @@ class Extras(str, enum.Enum): @engines_only("postgres", "sqlite") -class TestArrayChoices(TestCase): +class TestArrayChoices(TableTest): """ 🐛 Cockroach bug: https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg """ # noqa: E501 - def setUp(self): - Ticket.create_table().run_sync() - - def tearDown(self): - Ticket.alter().drop_table().run_sync() + tables = [Ticket] def test_string(self): """ diff --git a/tests/columns/test_date.py b/tests/columns/test_date.py index 5c1016211..7cdfb89d9 100644 --- a/tests/columns/test_date.py +++ b/tests/columns/test_date.py @@ -1,9 +1,9 @@ import datetime -from unittest import TestCase from piccolo.columns.column_types import Date from piccolo.columns.defaults.date import DateNow from piccolo.table import Table +from tests.base import TableTest class MyTable(Table): @@ -14,12 +14,8 @@ class MyTableDefault(Table): created_on = Date(default=DateNow()) -class TestDate(TestCase): - def setUp(self): - MyTable.create_table().run_sync() - - def tearDown(self): - MyTable.alter().drop_table().run_sync() +class TestDate(TableTest): + tables = [MyTable] def test_timestamp(self): created_on = datetime.datetime.now().date() @@ -31,12 +27,8 @@ def test_timestamp(self): self.assertEqual(result.created_on, created_on) -class TestDateDefault(TestCase): - def setUp(self): - MyTableDefault.create_table().run_sync() - - def tearDown(self): - MyTableDefault.alter().drop_table().run_sync() +class TestDateDefault(TableTest): + tables = [MyTableDefault] def test_timestamp(self): created_on = datetime.datetime.now().date() diff --git a/tests/columns/test_double_precision.py b/tests/columns/test_double_precision.py index 99e411f11..bdc8ff387 100644 --- a/tests/columns/test_double_precision.py +++ b/tests/columns/test_double_precision.py @@ -1,19 +1,14 @@ -from unittest import TestCase - from piccolo.columns.column_types import DoublePrecision from piccolo.table import Table +from tests.base import TableTest class MyTable(Table): column_a = DoublePrecision() -class TestDoublePrecision(TestCase): - def setUp(self): - MyTable.create_table().run_sync() - - def tearDown(self): - MyTable.alter().drop_table().run_sync() +class TestDoublePrecision(TableTest): + tables = [MyTable] def test_creation(self): row = MyTable(column_a=1.23) diff --git a/tests/columns/test_interval.py b/tests/columns/test_interval.py index d9e7c6f10..2e56e6a84 100644 --- a/tests/columns/test_interval.py +++ b/tests/columns/test_interval.py @@ -1,9 +1,9 @@ import datetime -from unittest import TestCase from piccolo.columns.column_types import Interval from piccolo.columns.defaults.interval import IntervalCustom from piccolo.table import Table +from tests.base import TableTest class MyTable(Table): @@ -14,12 +14,8 @@ class MyTableDefault(Table): interval = Interval(default=IntervalCustom(days=1)) -class TestInterval(TestCase): - def setUp(self): - MyTable.create_table().run_sync() - - def tearDown(self): - MyTable.alter().drop_table().run_sync() +class TestInterval(TableTest): + tables = [MyTable] def test_interval(self): # Test a range of different timedeltas @@ -91,12 +87,8 @@ def test_interval_where_clause(self): self.assertTrue(result) -class TestIntervalDefault(TestCase): - def setUp(self): - MyTableDefault.create_table().run_sync() - - def tearDown(self): - MyTableDefault.alter().drop_table().run_sync() +class TestIntervalDefault(TableTest): + tables = [MyTableDefault] def test_interval(self): row = MyTableDefault() diff --git a/tests/columns/test_json.py b/tests/columns/test_json.py index 69808163c..8b4d8e6cb 100644 --- a/tests/columns/test_json.py +++ b/tests/columns/test_json.py @@ -1,7 +1,6 @@ -from unittest import TestCase - from piccolo.columns.column_types import JSON from piccolo.table import Table +from tests.base import TableTest class MyTable(Table): @@ -20,12 +19,8 @@ class MyTableDefault(Table): json_none = JSON(default=None, null=True) -class TestJSONSave(TestCase): - def setUp(self): - MyTable.create_table().run_sync() - - def tearDown(self): - MyTable.alter().drop_table().run_sync() +class TestJSONSave(TableTest): + tables = [MyTable] def test_json_string(self): """ @@ -58,12 +53,8 @@ def test_json_object(self): ) -class TestJSONDefault(TestCase): - def setUp(self): - MyTableDefault.create_table().run_sync() - - def tearDown(self): - MyTableDefault.alter().drop_table().run_sync() +class TestJSONDefault(TableTest): + tables = [MyTableDefault] def test_json_default(self): row = MyTableDefault() @@ -81,12 +72,8 @@ def test_invalid_default(self): JSON(default=value) # type: ignore -class TestJSONInsert(TestCase): - def setUp(self): - MyTable.create_table().run_sync() - - def tearDown(self): - MyTable.alter().drop_table().run_sync() +class TestJSONInsert(TableTest): + tables = [MyTable] def check_response(self): row = MyTable.select(MyTable.json).first().run_sync() @@ -112,12 +99,8 @@ def test_json_object(self): MyTable.insert(row).run_sync() -class TestJSONUpdate(TestCase): - def setUp(self): - MyTable.create_table().run_sync() - - def tearDown(self): - MyTable.alter().drop_table().run_sync() +class TestJSONUpdate(TableTest): + tables = [MyTable] def add_row(self): row = MyTable(json={"message": "original"}) diff --git a/tests/columns/test_jsonb.py b/tests/columns/test_jsonb.py index 7c2be3a5a..1e03aa10d 100644 --- a/tests/columns/test_jsonb.py +++ b/tests/columns/test_jsonb.py @@ -1,8 +1,6 @@ -from unittest import TestCase - from piccolo.columns.column_types import JSONB, ForeignKey, Varchar from piccolo.table import Table -from tests.base import engines_only, engines_skip +from tests.base import TableTest, engines_only, engines_skip class RecordingStudio(Table): @@ -16,14 +14,8 @@ class Instrument(Table): @engines_only("postgres", "cockroach") -class TestJSONB(TestCase): - def setUp(self): - RecordingStudio.create_table().run_sync() - Instrument.create_table().run_sync() - - def tearDown(self): - Instrument.alter().drop_table().run_sync() - RecordingStudio.alter().drop_table().run_sync() +class TestJSONB(TableTest): + tables = [RecordingStudio, Instrument] def test_json(self): """ diff --git a/tests/columns/test_numeric.py b/tests/columns/test_numeric.py index 872d739a9..cc8c8a604 100644 --- a/tests/columns/test_numeric.py +++ b/tests/columns/test_numeric.py @@ -1,8 +1,8 @@ from decimal import Decimal -from unittest import TestCase from piccolo.columns.column_types import Numeric from piccolo.table import Table +from tests.base import TableTest class MyTable(Table): @@ -10,12 +10,8 @@ class MyTable(Table): column_b = Numeric(digits=(3, 2)) -class TestNumeric(TestCase): - def setUp(self): - MyTable.create_table().run_sync() - - def tearDown(self): - MyTable.alter().drop_table().run_sync() +class TestNumeric(TableTest): + tables = [MyTable] def test_creation(self): row = MyTable(column_a=Decimal(1.23), column_b=Decimal(1.23)) diff --git a/tests/columns/test_primary_key.py b/tests/columns/test_primary_key.py index 1850944cc..df041111e 100644 --- a/tests/columns/test_primary_key.py +++ b/tests/columns/test_primary_key.py @@ -1,5 +1,4 @@ import uuid -from unittest import TestCase from piccolo.columns.column_types import ( UUID, @@ -9,6 +8,7 @@ Varchar, ) from piccolo.table import Table +from tests.base import TableTest class MyTableDefaultPrimaryKey(Table): @@ -30,12 +30,8 @@ class MyTablePrimaryKeyUUID(Table): name = Varchar() -class TestPrimaryKeyDefault(TestCase): - def setUp(self): - MyTableDefaultPrimaryKey.create_table().run_sync() - - def tearDown(self): - MyTableDefaultPrimaryKey.alter().drop_table().run_sync() +class TestPrimaryKeyDefault(TableTest): + tables = [MyTableDefaultPrimaryKey] def test_return_type(self): row = MyTableDefaultPrimaryKey() @@ -45,12 +41,8 @@ def test_return_type(self): self.assertIsInstance(row["id"], int) -class TestPrimaryKeyInteger(TestCase): - def setUp(self): - MyTablePrimaryKeySerial.create_table().run_sync() - - def tearDown(self): - MyTablePrimaryKeySerial.alter().drop_table().run_sync() +class TestPrimaryKeyInteger(TableTest): + tables = [MyTablePrimaryKeySerial] def test_return_type(self): row = MyTablePrimaryKeySerial() @@ -60,12 +52,8 @@ def test_return_type(self): self.assertIsInstance(row["pk"], int) -class TestPrimaryKeyBigSerial(TestCase): - def setUp(self): - MyTablePrimaryKeyBigSerial.create_table().run_sync() - - def tearDown(self): - MyTablePrimaryKeyBigSerial.alter().drop_table().run_sync() +class TestPrimaryKeyBigSerial(TableTest): + tables = [MyTablePrimaryKeyBigSerial] def test_return_type(self): row = MyTablePrimaryKeyBigSerial() @@ -75,12 +63,8 @@ def test_return_type(self): self.assertIsInstance(row["pk"], int) -class TestPrimaryKeyUUID(TestCase): - def setUp(self): - MyTablePrimaryKeyUUID.create_table().run_sync() - - def tearDown(self): - MyTablePrimaryKeyUUID.alter().drop_table().run_sync() +class TestPrimaryKeyUUID(TableTest): + tables = [MyTablePrimaryKeyUUID] def test_return_type(self): row = MyTablePrimaryKeyUUID() @@ -101,14 +85,8 @@ class Band(Table): manager = ForeignKey(Manager) -class TestPrimaryKeyQueries(TestCase): - def setUp(self): - Manager.create_table().run_sync() - Band.create_table().run_sync() - - def tearDown(self): - Band.alter().drop_table().run_sync() - Manager.alter().drop_table().run_sync() +class TestPrimaryKeyQueries(TableTest): + tables = [Manager, Band] def test_primary_key_queries(self): """ diff --git a/tests/columns/test_readable.py b/tests/columns/test_readable.py index 5b214896a..3d433b24c 100644 --- a/tests/columns/test_readable.py +++ b/tests/columns/test_readable.py @@ -1,8 +1,7 @@ -import unittest - from piccolo import columns from piccolo.columns.readable import Readable from piccolo.table import Table +from tests.base import TableTest class MyTable(Table): @@ -16,14 +15,13 @@ def get_readable(cls) -> Readable: ) -class TestReadable(unittest.TestCase): +class TestReadable(TableTest): + tables = [MyTable] + def setUp(self): - MyTable.create_table().run_sync() + super().setUp() MyTable(first_name="Guido", last_name="van Rossum").save().run_sync() def test_readable(self): response = MyTable.select(MyTable.get_readable()).run_sync() self.assertEqual(response[0]["readable"], "Guido van Rossum") - - def tearDown(self): - MyTable.alter().drop_table().run_sync() diff --git a/tests/columns/test_real.py b/tests/columns/test_real.py index 3257111de..63ab6a4fe 100644 --- a/tests/columns/test_real.py +++ b/tests/columns/test_real.py @@ -1,19 +1,14 @@ -from unittest import TestCase - from piccolo.columns.column_types import Real from piccolo.table import Table +from tests.base import TableTest class MyTable(Table): column_a = Real() -class TestReal(TestCase): - def setUp(self): - MyTable.create_table().run_sync() - - def tearDown(self): - MyTable.alter().drop_table().run_sync() +class TestReal(TableTest): + tables = [MyTable] def test_creation(self): row = MyTable(column_a=1.23) diff --git a/tests/columns/test_reserved_column_names.py b/tests/columns/test_reserved_column_names.py index 71cedacc8..b87c8b755 100644 --- a/tests/columns/test_reserved_column_names.py +++ b/tests/columns/test_reserved_column_names.py @@ -1,7 +1,6 @@ -from unittest import TestCase - from piccolo.columns.column_types import Integer, Varchar from piccolo.table import Table +from tests.base import TableTest class Concert(Table): @@ -16,17 +15,13 @@ class Concert(Table): order = Integer() -class TestReservedColumnNames(TestCase): +class TestReservedColumnNames(TableTest): """ Make sure the table works as expected, even though it has a problematic column name. """ - def setUp(self): - Concert.create_table().run_sync() - - def tearDown(self): - Concert.alter().drop_table().run_sync() + tables = [Concert] def test_common_operations(self): # Save / Insert diff --git a/tests/columns/test_smallint.py b/tests/columns/test_smallint.py index 4fc6c1a65..75beb4275 100644 --- a/tests/columns/test_smallint.py +++ b/tests/columns/test_smallint.py @@ -1,10 +1,9 @@ import os -from unittest import TestCase from piccolo.columns.column_types import SmallInt from piccolo.table import Table -from ..base import engines_only +from ..base import TableTest, engines_only class MyTable(Table): @@ -12,16 +11,12 @@ class MyTable(Table): @engines_only("postgres", "cockroach") -class TestSmallIntPostgres(TestCase): +class TestSmallIntPostgres(TableTest): """ Make sure a SmallInt column in Postgres can only store small numbers. """ - def setUp(self): - MyTable.create_table().run_sync() - - def tearDown(self): - MyTable.alter().drop_table().run_sync() + tables = [MyTable] def _test_length(self): # Can store 2 bytes, but split between positive and negative values. diff --git a/tests/columns/test_time.py b/tests/columns/test_time.py index a6d931448..f6ab5e7ef 100644 --- a/tests/columns/test_time.py +++ b/tests/columns/test_time.py @@ -1,11 +1,10 @@ import datetime from functools import partial -from unittest import TestCase from piccolo.columns.column_types import Time from piccolo.columns.defaults.time import TimeNow from piccolo.table import Table -from tests.base import engines_skip +from tests.base import TableTest, engines_skip class MyTable(Table): @@ -16,12 +15,8 @@ class MyTableDefault(Table): created_on = Time(default=TimeNow()) -class TestTime(TestCase): - def setUp(self): - MyTable.create_table().run_sync() - - def tearDown(self): - MyTable.alter().drop_table().run_sync() +class TestTime(TableTest): + tables = [MyTable] @engines_skip("cockroach") def test_timestamp(self): @@ -34,12 +29,8 @@ def test_timestamp(self): self.assertEqual(result.created_on, created_on) -class TestTimeDefault(TestCase): - def setUp(self): - MyTableDefault.create_table().run_sync() - - def tearDown(self): - MyTableDefault.alter().drop_table().run_sync() +class TestTimeDefault(TableTest): + tables = [MyTableDefault] @engines_skip("cockroach") def test_timestamp(self): diff --git a/tests/columns/test_timestamp.py b/tests/columns/test_timestamp.py index ad1fa01f0..0b82c6e42 100644 --- a/tests/columns/test_timestamp.py +++ b/tests/columns/test_timestamp.py @@ -1,9 +1,9 @@ import datetime -from unittest import TestCase from piccolo.columns.column_types import Timestamp from piccolo.columns.defaults.timestamp import TimestampNow from piccolo.table import Table +from tests.base import TableTest class MyTable(Table): @@ -19,12 +19,8 @@ class MyTableDefault(Table): created_on = Timestamp(default=TimestampNow()) -class TestTimestamp(TestCase): - def setUp(self): - MyTable.create_table().run_sync() - - def tearDown(self): - MyTable.alter().drop_table().run_sync() +class TestTimestamp(TableTest): + tables = [MyTable] def test_timestamp(self): """ @@ -46,12 +42,8 @@ def test_timezone_aware(self): Timestamp(default=datetime.datetime.now(tz=datetime.timezone.utc)) -class TestTimestampDefault(TestCase): - def setUp(self): - MyTableDefault.create_table().run_sync() - - def tearDown(self): - MyTableDefault.alter().drop_table().run_sync() +class TestTimestampDefault(TableTest): + tables = [MyTableDefault] def test_timestamp(self): """ diff --git a/tests/columns/test_timestamptz.py b/tests/columns/test_timestamptz.py index 8e239900b..62d83a52b 100644 --- a/tests/columns/test_timestamptz.py +++ b/tests/columns/test_timestamptz.py @@ -1,5 +1,4 @@ import datetime -from unittest import TestCase from dateutil import tz @@ -10,6 +9,7 @@ TimestamptzOffset, ) from piccolo.table import Table +from tests.base import TableTest class MyTable(Table): @@ -34,12 +34,8 @@ class CustomTimezone(datetime.tzinfo): pass -class TestTimestamptz(TestCase): - def setUp(self): - MyTable.create_table().run_sync() - - def tearDown(self): - MyTable.alter().drop_table().run_sync() +class TestTimestamptz(TableTest): + tables = [MyTable] def test_timestamptz_timezone_aware(self): """ @@ -78,12 +74,8 @@ def test_timestamptz_timezone_aware(self): self.assertEqual(result.created_on.tzinfo, datetime.timezone.utc) -class TestTimestamptzDefault(TestCase): - def setUp(self): - MyTableDefault.create_table().run_sync() - - def tearDown(self): - MyTableDefault.alter().drop_table().run_sync() +class TestTimestamptzDefault(TableTest): + tables = [MyTableDefault] def test_timestamptz_default(self): """ diff --git a/tests/columns/test_uuid.py b/tests/columns/test_uuid.py index 64a197fc6..28be64c8a 100644 --- a/tests/columns/test_uuid.py +++ b/tests/columns/test_uuid.py @@ -1,20 +1,16 @@ import uuid -from unittest import TestCase from piccolo.columns.column_types import UUID from piccolo.table import Table +from tests.base import TableTest class MyTable(Table): uuid = UUID() -class TestUUID(TestCase): - def setUp(self): - MyTable.create_table().run_sync() - - def tearDown(self): - MyTable.alter().drop_table().run_sync() +class TestUUID(TableTest): + tables = [MyTable] def test_return_type(self): row = MyTable() diff --git a/tests/columns/test_varchar.py b/tests/columns/test_varchar.py index 89850d801..f433cb4c7 100644 --- a/tests/columns/test_varchar.py +++ b/tests/columns/test_varchar.py @@ -1,9 +1,7 @@ -from unittest import TestCase - from piccolo.columns.column_types import Varchar from piccolo.table import Table -from ..base import engines_only +from ..base import TableTest, engines_only class MyTable(Table): @@ -11,7 +9,7 @@ class MyTable(Table): @engines_only("postgres", "cockroach") -class TestVarchar(TestCase): +class TestVarchar(TableTest): """ SQLite doesn't enforce any constraints on max character length. @@ -20,11 +18,7 @@ class TestVarchar(TestCase): Might consider enforcing this at the ORM level instead in the future. """ - def setUp(self): - MyTable.create_table().run_sync() - - def tearDown(self): - MyTable.alter().drop_table().run_sync() + tables = [MyTable] def test_length(self): row = MyTable(name="bob") From bf6393b4cb761f0bcd8cc74ef0ea163ec3fa6354 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 19 Jun 2024 18:46:56 +0100 Subject: [PATCH 592/727] allow migrations to be hardcoded as fake (#1034) --- docs/src/piccolo/migrations/running.rst | 20 +++++++++++ .../tutorials/migrate_existing_project.rst | 2 +- .../apps/migrations/auto/migration_manager.py | 1 + piccolo/apps/migrations/commands/forwards.py | 15 +++++---- .../commands/test_forwards_backwards.py | 33 ++++++++++++++++++- .../music_2024_06_19t18_11_05_793132.py | 20 +++++++++++ 6 files changed, 82 insertions(+), 9 deletions(-) create mode 100644 tests/example_apps/music/piccolo_migrations/music_2024_06_19t18_11_05_793132.py diff --git a/docs/src/piccolo/migrations/running.rst b/docs/src/piccolo/migrations/running.rst index 74bc9cd50..8979a70b1 100644 --- a/docs/src/piccolo/migrations/running.rst +++ b/docs/src/piccolo/migrations/running.rst @@ -23,16 +23,36 @@ If you have multiple apps you can run them all using: piccolo migrations forwards all +.. _FakeMigration: + Fake ~~~~ We can 'fake' running a migration - we record that it ran in the database without actually running it. +There are two ways to do this - by passing in the ``--fake`` flag on the +command line: + .. code-block:: bash piccolo migrations forwards my_app 2022-09-04T19:44:09 --fake +Or by setting ``fake=True`` on the ``MigrationManager`` within the migration +file. + +.. code-block:: python + + async def forwards(): + manager = MigrationManager( + migration_id=ID, + app_name="app", + description=DESCRIPTION, + fake=True + ) + ... + + This is useful if we started from an existing database using ``piccolo schema generate``, and the initial migration we generated is for tables which already exist, hence we fake run it. diff --git a/docs/src/piccolo/tutorials/migrate_existing_project.rst b/docs/src/piccolo/tutorials/migrate_existing_project.rst index 7ead3dcbd..e4810d25a 100644 --- a/docs/src/piccolo/tutorials/migrate_existing_project.rst +++ b/docs/src/piccolo/tutorials/migrate_existing_project.rst @@ -99,7 +99,7 @@ This will create a new file in ``my_app/piccolo_migrations``: piccolo migrations new my_app --auto These tables already exist in the database, as it's an existing project, so -you need to fake apply this initial migration: +you need to :ref:`fake apply ` this initial migration: .. code-block:: bash diff --git a/piccolo/apps/migrations/auto/migration_manager.py b/piccolo/apps/migrations/auto/migration_manager.py index e8f4931cb..ededa3a25 100644 --- a/piccolo/apps/migrations/auto/migration_manager.py +++ b/piccolo/apps/migrations/auto/migration_manager.py @@ -165,6 +165,7 @@ class MigrationManager: raw_backwards: t.List[t.Union[t.Callable, AsyncFunction]] = field( default_factory=list ) + fake: bool = False def add_table( self, diff --git a/piccolo/apps/migrations/commands/forwards.py b/piccolo/apps/migrations/commands/forwards.py index 62278060d..560f117b2 100644 --- a/piccolo/apps/migrations/commands/forwards.py +++ b/piccolo/apps/migrations/commands/forwards.py @@ -72,18 +72,19 @@ async def run_migrations(self, app_config: AppConfig) -> MigrationResult: print(f"🚀 Running {n} migration{'s' if n != 1 else ''}:") for _id in subset: - if self.fake: - print(f"- {_id}: faked! ⏭️") - else: - migration_module = migration_modules[_id] - response = await migration_module.forwards() + migration_module = migration_modules[_id] + response = await migration_module.forwards() - if isinstance(response, MigrationManager): + if isinstance(response, MigrationManager): + if self.fake or response.fake: + print(f"- {_id}: faked! ⏭️") + else: if self.preview: response.preview = True await response.run() - print("ok! ✔️") + print("ok! ✔️") + if not self.preview: await Migration.insert().add( Migration(name=_id, app_name=app_config.app_name) diff --git a/tests/apps/migrations/commands/test_forwards_backwards.py b/tests/apps/migrations/commands/test_forwards_backwards.py index c3c2c6592..e88747eda 100644 --- a/tests/apps/migrations/commands/test_forwards_backwards.py +++ b/tests/apps/migrations/commands/test_forwards_backwards.py @@ -192,7 +192,7 @@ def test_forwards_no_migrations(self, print_: MagicMock): @engines_only("postgres") def test_forwards_fake(self): """ - Test running the migrations if they've already run. + Make sure migrations can be faked on the command line. """ run_sync(forwards(app_name="music", migration_id="all", fake=True)) @@ -214,9 +214,40 @@ def test_forwards_fake(self): "2021-09-06T13:58:23:024723", "2021-11-13T14:01:46:114725", "2024-05-28T23:15:41:018844", + "2024-06-19T18:11:05:793132", ], ) + @engines_only("postgres") + @patch("piccolo.apps.migrations.commands.forwards.print") + def test_hardcoded_fake_migrations(self, print_: MagicMock): + """ + Make sure that migrations that have been hardcoded as fake aren't + executed, even without the ``--fake`` command line flag. + + See tests/example_apps/music/piccolo_migrations/music_2024_06_19t18_11_05_793132.py + + """ # noqa: E501 + run_sync(forwards(app_name="music", migration_id="all")) + + # The migration which is hardcoded as fake: + migration_name = "2024-06-19T18:11:05:793132" + + self.assertTrue( + Migration.exists() + .where(Migration.name == migration_name) + .run_sync() + ) + + self.assertNotIn( + call("Running fake migration"), + print_.mock_calls, + ) + self.assertIn( + call(f"- {migration_name}: faked! ⏭️"), + print_.mock_calls, + ) + def tearDown(self): for table_class in TABLE_CLASSES + [Migration]: table_class.alter().drop_table( diff --git a/tests/example_apps/music/piccolo_migrations/music_2024_06_19t18_11_05_793132.py b/tests/example_apps/music/piccolo_migrations/music_2024_06_19t18_11_05_793132.py new file mode 100644 index 000000000..5d884210d --- /dev/null +++ b/tests/example_apps/music/piccolo_migrations/music_2024_06_19t18_11_05_793132.py @@ -0,0 +1,20 @@ +from piccolo.apps.migrations.auto.migration_manager import MigrationManager + +ID = "2024-06-19T18:11:05:793132" +VERSION = "1.11.0" +DESCRIPTION = "An example fake migration" + + +async def forwards(): + manager = MigrationManager( + migration_id=ID, app_name="", description=DESCRIPTION, fake=True + ) + + def run(): + # This should never run, as this migrations is `fake=True`. It's here + # for testing purposes (to make sure it never gets triggered). + print("Running fake migration") + + manager.add_raw(run) + + return manager From 17c0a8859c19da48b33a19bb5a15d44d91bb6278 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 19 Jun 2024 18:53:14 +0100 Subject: [PATCH 593/727] bumped version --- CHANGES.rst | 11 +++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index f451e0153..0adb3341f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,17 @@ Changes ======= +1.12.0 +------ + +* Added documentation for one to one fields. +* Upgraded ASGI templates (thanks to @sinisaos for this). +* Migrations can now be hardcoded as fake. +* Refactored tests to reduce boilerplate code. +* Updated documentation dependencies. + +------------------------------------------------------------------------------- + 1.11.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 05dd9013c..6a66f5591 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.11.0" +__VERSION__ = "1.12.0" From 85dd176b9ab94d5b438cd25e28a3a51dbf48beaa Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 26 Jun 2024 17:23:41 +0100 Subject: [PATCH 594/727] fix typo in `Email` docstring (#1038) --- piccolo/columns/column_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 40d0c945e..7038bf6f2 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -363,7 +363,7 @@ class Email(Varchar): Used for storing email addresses. It's identical to :class:`Varchar`, except when using :func:`create_pydantic_model ` - we add email validation to the Pydantic model. This means that :ref:`PiccoloAdmin` - also validates emails addresses. + also validates email addresses. """ # noqa: E501 pass From 76737dca4d7da174e6c874d17d165fcddb8c3a5b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 27 Jun 2024 12:58:50 +0100 Subject: [PATCH 595/727] 1039 Improve `LazyTableReference` (#1040) * add test * improve lazy table refs * improve docs for `LazyTableReference` * add docs for circular imports --- .../projects_and_apps/included_apps.rst | 2 + docs/src/piccolo/schema/m2m.rst | 5 +- .../tutorials/avoiding_circular_imports.rst | 82 +++++++++++++++++++ .../avoiding_circular_imports_src/tables.py | 22 +++++ docs/src/piccolo/tutorials/deployment.rst | 10 ++- docs/src/piccolo/tutorials/index.rst | 1 + piccolo/columns/column_types.py | 9 +- piccolo/columns/reference.py | 3 + tests/columns/test_reference.py | 35 +++++++- 9 files changed, 159 insertions(+), 10 deletions(-) create mode 100644 docs/src/piccolo/tutorials/avoiding_circular_imports.rst create mode 100644 docs/src/piccolo/tutorials/avoiding_circular_imports_src/tables.py diff --git a/docs/src/piccolo/projects_and_apps/included_apps.rst b/docs/src/piccolo/projects_and_apps/included_apps.rst index 620e20194..5966c92af 100644 --- a/docs/src/piccolo/projects_and_apps/included_apps.rst +++ b/docs/src/piccolo/projects_and_apps/included_apps.rst @@ -38,6 +38,8 @@ Lets you scaffold an ASGI web app. See :ref:`ASGICommand`. ------------------------------------------------------------------------------- +.. _Fixtures: + fixtures ~~~~~~~~ diff --git a/docs/src/piccolo/schema/m2m.rst b/docs/src/piccolo/schema/m2m.rst index 5413389ec..b7c44188a 100644 --- a/docs/src/piccolo/schema/m2m.rst +++ b/docs/src/piccolo/schema/m2m.rst @@ -54,8 +54,9 @@ We create it in Piccolo like this: .. note:: - We use ``LazyTableReference`` because when Python evaluates ``Band`` and - ``Genre``, the ``GenreToBand`` class doesn't exist yet. + We use :class:`LazyTableReference ` + because when Python evaluates ``Band`` and ``Genre``, the ``GenreToBand`` + class doesn't exist yet. By using ``M2M`` it unlocks some powerful and convenient features. diff --git a/docs/src/piccolo/tutorials/avoiding_circular_imports.rst b/docs/src/piccolo/tutorials/avoiding_circular_imports.rst new file mode 100644 index 000000000..fd587abb6 --- /dev/null +++ b/docs/src/piccolo/tutorials/avoiding_circular_imports.rst @@ -0,0 +1,82 @@ +Avoiding circular imports +========================= + +How Python imports work +----------------------- + +When Python imports a file, it evaluates it from top to bottom. + +With :class:`ForeignKey ` columns we +sometimes have to reference tables lower down in the file (which haven't been +evaluated yet). + +The solutions are: + +* Try and move the referenced table to a different Python file. +* Use :class:`LazyTableReference ` + +Import ``Table`` definitions as early as possible +------------------------------------------------- + +In the entrypoint to your app, at the top of the file, it's recommended to +import your tables. + +.. code-block:: python + + # main.py + from my_app.tables import Manager, Band + +This ensures that the tables are imported, and setup correctly. + +Keep table files focused +------------------------ + +You should try and keep your ``tables.py`` files pretty focused (i.e. +just contain your ``Table`` definitions). + +If you have lots of logic alongside your ``Table`` definitions, it might cause +your ``LazyTableReference`` references to evaluate too soon (causing circular +import errors). An example of this is with +:func:`create_pydantic_model `: + +.. literalinclude:: avoiding_circular_imports_src/tables.py + +Simplify your schema if possible +-------------------------------- + +Even with :class:`LazyTableReference `, +you may run into some problems if your schema is really complicated. + +An example is when you have two tables, and they have foreign keys to each other. + +.. code-block:: python + + class Band(Table): + name = Varchar() + manager = ForeignKey("Manager") + + + class Manager(Table): + name = Varchar() + favourite_band = ForeignKey(Band) + + +Piccolo should be able to create these tables, and query them. However, some +Piccolo tooling may struggle - for example when loading :ref:`fixtures `. + +A joining table can help in these situations: + +.. code-block:: python + + class Band(Table): + name = Varchar() + manager = ForeignKey("Manager") + + + class Manager(Table): + name = Varchar() + + + class ManagerFavouriteBand(Table): + manager = ForeignKey(Manager, unique=True) + band = ForeignKey(Band) diff --git a/docs/src/piccolo/tutorials/avoiding_circular_imports_src/tables.py b/docs/src/piccolo/tutorials/avoiding_circular_imports_src/tables.py new file mode 100644 index 000000000..6d1021deb --- /dev/null +++ b/docs/src/piccolo/tutorials/avoiding_circular_imports_src/tables.py @@ -0,0 +1,22 @@ +# tables.py + +from piccolo.columns import ForeignKey, Varchar +from piccolo.table import Table +from piccolo.utils.pydantic import create_pydantic_model + + +class Band(Table): + name = Varchar() + # This automatically gets converted into a LazyTableReference, because a + # string is passed in: + manager = ForeignKey("Manager") + + +# This is not recommended, as it will cause the LazyTableReference to be +# evaluated before Manager has imported. +# Instead, move this to a separate file, or below Manager. +BandModel = create_pydantic_model(Band) + + +class Manager(Table): + name = Varchar() diff --git a/docs/src/piccolo/tutorials/deployment.rst b/docs/src/piccolo/tutorials/deployment.rst index 2a1b25b68..3b5352d58 100644 --- a/docs/src/piccolo/tutorials/deployment.rst +++ b/docs/src/piccolo/tutorials/deployment.rst @@ -35,7 +35,7 @@ This is a very simple Dockerfile, and illustrates the basics: .. code-block:: dockerfile # Specify the base image: - FROM python:3.10-slim-bullseye + FROM python:3.12-bookworm # Install the pip requirements: RUN pip install --upgrade pip @@ -77,3 +77,11 @@ When we run the container (usually via `Kubernetes `_, `Docker Compose `_, or similar), we can specify the database credentials using environment variables, which will be used by our application. + +Accessing a local Postgres database +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Bear in mind that if you have Postgres running locally on the server (i.e. on +``localhost``), your Docker container won't automatically be able to access it. +You can try Docker's host based networking, or just run Postgres within a +Docker container. diff --git a/docs/src/piccolo/tutorials/index.rst b/docs/src/piccolo/tutorials/index.rst index 5c9d66989..1fae535a0 100644 --- a/docs/src/piccolo/tutorials/index.rst +++ b/docs/src/piccolo/tutorials/index.rst @@ -11,3 +11,4 @@ help you solve common problems: ./using_sqlite_and_asyncio_effectively ./deployment ./fastapi + ./avoiding_circular_imports diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 7038bf6f2..d16329b49 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -2183,7 +2183,7 @@ def __getattribute__(self, name: str) -> t.Union[Column, t.Any]: # If the ForeignKey is using a lazy reference, we need to set the # attributes here. Attributes starting with an underscore are # unlikely to be column names. - if not name.startswith("__"): + if not name.startswith("_") and name not in dir(self): try: _foreign_key_meta = object.__getattribute__( self, "_foreign_key_meta" @@ -2196,12 +2196,9 @@ def __getattribute__(self, name: str) -> t.Union[Column, t.Any]: ): object.__getattribute__(self, "set_proxy_columns")() - try: - value = object.__getattribute__(self, name) - except AttributeError: - raise AttributeError + value = object.__getattribute__(self, name) - if name == "_": + if name.startswith("_"): return value foreignkey_class: t.Type[ForeignKey] = object.__getattribute__( diff --git a/piccolo/columns/reference.py b/piccolo/columns/reference.py index f6edcdd56..9870f119a 100644 --- a/piccolo/columns/reference.py +++ b/piccolo/columns/reference.py @@ -30,6 +30,9 @@ class LazyTableReference: If specified, the ``Table`` subclass is imported from this path. For example, ``'my_app.tables'``. + .. hint:: + If the table is in the same file, you can pass in ``__name__``. + """ table_class_name: str diff --git a/tests/columns/test_reference.py b/tests/columns/test_reference.py index 21daa2f58..23e3f5ddd 100644 --- a/tests/columns/test_reference.py +++ b/tests/columns/test_reference.py @@ -5,10 +5,41 @@ from unittest import TestCase +from piccolo.columns import ForeignKey, Varchar from piccolo.columns.reference import LazyTableReference +from piccolo.table import Table +from tests.base import TableTest -class TestLazyTableReference(TestCase): +class Band(Table): + manager: ForeignKey["Manager"] = ForeignKey( + LazyTableReference("Manager", module_path=__name__) + ) + name = Varchar() + + +class Manager(Table): + name = Varchar() + + +class TestQueries(TableTest): + tables = [Band, Manager] + + def setUp(self): + super().setUp() + manager = Manager({Manager.name: "Guido"}) + manager.save().run_sync() + band = Band({Band.name: "Pythonistas", Band.manager: manager}) + band.save().run_sync() + + def test_select(self): + self.assertListEqual( + Band.select(Band.name, Band.manager._.name).run_sync(), + [{"name": "Pythonistas", "manager.name": "Guido"}], + ) + + +class TestInit(TestCase): def test_init(self): """ A ``LazyTableReference`` must be passed either an ``app_name`` or @@ -34,6 +65,8 @@ def test_init(self): module_path="tests.example_apps.music.tables", ) + +class TestStr(TestCase): def test_str(self): self.assertEqual( LazyTableReference( From ce250611adf6b0a1df7d03ca4bd8fecf698e83c1 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 27 Jun 2024 13:01:49 +0100 Subject: [PATCH 596/727] bumped version --- CHANGES.rst | 7 +++++++ piccolo/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 0adb3341f..29fc9f95e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,13 @@ Changes ======= +1.13.0 +------ + +Improved ``LazyTableReference``, to help prevent circular import errors. + +------------------------------------------------------------------------------- + 1.12.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 6a66f5591..d430f22d0 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.12.0" +__VERSION__ = "1.13.0" From 76ce9769daaebef45b9b4a43c0c7c041725fbfce Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 4 Jul 2024 14:03:51 +0100 Subject: [PATCH 597/727] 1042 Re-export aggregate functions from `select.py` (#1043) * re-export aggregate functions from `select.py` * fix pydantic linter warning --- piccolo/query/methods/select.py | 8 +++++++- piccolo/utils/pydantic.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 05302455f..4c6390fae 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -36,7 +36,13 @@ from piccolo.table import Table # noqa # Here to avoid breaking changes - will be removed in the future. -from piccolo.query.functions.aggregate import Count # noqa: F401 +from piccolo.query.functions.aggregate import ( # noqa: F401 + Avg, + Count, + Max, + Min, + Sum, +) class SelectRaw(Selectable): diff --git a/piccolo/utils/pydantic.py b/piccolo/utils/pydantic.py index b556ace85..3c88c8764 100644 --- a/piccolo/utils/pydantic.py +++ b/piccolo/utils/pydantic.py @@ -206,7 +206,7 @@ def create_pydantic_model( ########################################################################### columns: t.Dict[str, t.Any] = {} - validators: t.Dict[str, classmethod] = {} + validators: t.Dict[str, t.Callable] = {} piccolo_columns = tuple( table._meta.columns From bd759a6dd33bf02caf1d99908f22ae726897bd75 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 4 Jul 2024 14:07:44 +0100 Subject: [PATCH 598/727] bumped version --- CHANGES.rst | 9 +++++++++ piccolo/__init__.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 29fc9f95e..05eb98c4c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,15 @@ Changes ======= +1.13.1 +------ + +In Piccolo ``1.6.0`` we moved some aggregate functions to a new file. We now +re-export them from their original location to keep backwards compatibility. +Thanks to @sarvesh-deserve for reporting this issue. + +------------------------------------------------------------------------------- + 1.13.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index d430f22d0..d4e8d75c1 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.13.0" +__VERSION__ = "1.13.1" From e7b13fbe35d2ed4723c23e049dd5f5b12500513e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 12 Jul 2024 23:50:17 +0100 Subject: [PATCH 599/727] fix docs for `update_password` (#1050) --- docs/src/piccolo/authentication/baseuser.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/piccolo/authentication/baseuser.rst b/docs/src/piccolo/authentication/baseuser.rst index 1919f5cec..e2e55a98f 100644 --- a/docs/src/piccolo/authentication/baseuser.rst +++ b/docs/src/piccolo/authentication/baseuser.rst @@ -129,10 +129,10 @@ To change a user's password: .. code-block:: python # From within a coroutine: - await BaseUser.update_password(username="bob", password="abc123") + await BaseUser.update_password(user="bob", password="abc123") # When not in an event loop: - BaseUser.update_password_sync(username="bob", password="abc123") + BaseUser.update_password_sync(user="bob", password="abc123") .. warning:: Don't use bulk updates for passwords - use ``update_password`` / ``update_password_sync``, and they'll correctly hash the password. From 4ed6938c8906b03a788f4614779c67c3d3a6dd10 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 21 Jul 2024 08:52:43 +0100 Subject: [PATCH 600/727] Refactor `_process_results` to support new DB engines (#1054) --- piccolo/columns/base.py | 6 +++--- piccolo/engine/base.py | 7 +++++++ piccolo/engine/postgres.py | 7 +++++++ piccolo/query/base.py | 19 +++++-------------- piccolo/query/methods/select.py | 2 +- piccolo/querystring.py | 2 +- 6 files changed, 24 insertions(+), 19 deletions(-) diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 9d3e2b1cc..4725b78ad 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -250,11 +250,11 @@ def get_default_alias(self): if self.call_chain: column_name = ( - "$".join( + ".".join( t.cast(str, i._meta.db_column_name) for i in self.call_chain ) - + f"${column_name}" + + f".{column_name}" ) return column_name @@ -291,7 +291,7 @@ def get_full_name( 'band$manager.name' >>> Band.manager.name._meta.get_full_name(with_alias=True) - 'band$manager.name AS "manager$name"' + 'band$manager.name AS "manager.name"' :param include_quotes: If you're using the name in a SQL query, each component needs to be diff --git a/piccolo/engine/base.py b/piccolo/engine/base.py index bf59426ad..0181a29b5 100644 --- a/piccolo/engine/base.py +++ b/piccolo/engine/base.py @@ -132,6 +132,13 @@ async def run_querystring( ): pass + def transform_response_to_dicts(self, results) -> t.List[t.Dict]: + """ + If the database adapter returns something other than a list of + dictionaries, it should perform the transformation here. + """ + return results + @abstractmethod async def run_ddl(self, ddl: str, in_pool: bool = True): pass diff --git a/piccolo/engine/postgres.py b/piccolo/engine/postgres.py index 970623535..117bf9a53 100644 --- a/piccolo/engine/postgres.py +++ b/piccolo/engine/postgres.py @@ -579,6 +579,13 @@ async def run_ddl(self, ddl: str, in_pool: bool = True): return response + def transform_response_to_dicts(self, results) -> t.List[t.Dict]: + """ + asyncpg returns a special Record object, so we need to convert it to + a dict. + """ + return [dict(i) for i in results] + def atomic(self) -> Atomic: return Atomic(engine=self) diff --git a/piccolo/query/base.py b/piccolo/query/base.py index c169fde0e..444736f03 100644 --- a/piccolo/query/base.py +++ b/piccolo/query/base.py @@ -45,20 +45,11 @@ def engine_type(self) -> str: raise ValueError("Engine isn't defined.") async def _process_results(self, results) -> QueryResponseType: - if results: - keys = results[0].keys() - keys = [i.replace("$", ".") for i in keys] - if self.engine_type in ("postgres", "cockroach"): - # asyncpg returns a special Record object. We can pass it - # directly into zip without calling `values` on it. This can - # save us hundreds of microseconds, depending on the number of - # results. - raw = [dict(zip(keys, i)) for i in results] - else: - # SQLite returns a list of dictionaries. - raw = [dict(zip(keys, i.values())) for i in results] - else: - raw = [] + raw = ( + self.table._meta.db.transform_response_to_dicts(results) + if results + else [] + ) if hasattr(self, "_raw_response_callback"): self._raw_response_callback(raw) diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 4c6390fae..55267b703 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -574,7 +574,7 @@ def default_querystrings(self) -> t.Sequence[QueryString]: query += "{}" args.append(distinct.querystring) - columns_str = ", ".join("{}" for i in select_strings) + columns_str = ", ".join("{}" for _ in select_strings) query += f" {columns_str} FROM {self.table._meta.get_formatted_tablename()}" # noqa: E501 args.extend(select_strings) diff --git a/piccolo/querystring.py b/piccolo/querystring.py index 1f7282a7d..22f5f215a 100644 --- a/piccolo/querystring.py +++ b/piccolo/querystring.py @@ -246,7 +246,7 @@ def get_select_string( self, engine_type: str, with_alias: bool = True ) -> QueryString: if with_alias and self._alias: - return QueryString("{} AS " + self._alias, self) + return QueryString("{} AS " + f'"{self._alias}"', self) else: return self From d2ae6b725398880c89bb41fe608ebb715fe54ad1 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 21 Jul 2024 08:57:38 +0100 Subject: [PATCH 601/727] bumped version --- CHANGES.rst | 8 ++++++++ piccolo/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 05eb98c4c..033781fb7 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,14 @@ Changes ======= +1.14.0 +------ + +Laying the foundations for alterative Postgres database drivers (e.g. +``psqlpy``). Thanks to @insani7y and @chandr-andr for their help with this. + +------------------------------------------------------------------------------- + 1.13.1 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index d4e8d75c1..d27c8855a 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.13.1" +__VERSION__ = "1.14.0" From 748843e0ec1ea0997130ddac5606ef0e57d2c1d0 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 21 Jul 2024 09:37:25 +0100 Subject: [PATCH 602/727] add warnings to changelog --- CHANGES.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 033781fb7..cb258767c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -7,6 +7,13 @@ Changes Laying the foundations for alterative Postgres database drivers (e.g. ``psqlpy``). Thanks to @insani7y and @chandr-andr for their help with this. +.. warning:: + The SQL generated by Piccolo changed slightly in this release. Aliases used + to be like ``"manager$name"`` but now they are like ``"manager.name"`` + (note ``$`` changed to ``.``). If you are using ``SelectRaw`` in your queries + to refer to these columns, then they will need updating. Please let us know + if you encounter any other issues. + ------------------------------------------------------------------------------- 1.13.1 From 7f2cdb0757d35f746ffaca88bac495c554f57daa Mon Sep 17 00:00:00 2001 From: havk Date: Sat, 27 Jul 2024 22:23:31 +0200 Subject: [PATCH 603/727] Better password error messages (#1057) * Improve password error mesages Inform user about minimum and maximum password length constraints. * Add info about password minimum to doc * Fix formatting * fix minor linter issue --------- Co-authored-by: napcode Co-authored-by: Daniel Townsend --- docs/src/piccolo/authentication/baseuser.rst | 4 +++- piccolo/apps/user/tables.py | 8 ++++++-- tests/apps/user/test_tables.py | 16 ++++++++++++---- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/docs/src/piccolo/authentication/baseuser.rst b/docs/src/piccolo/authentication/baseuser.rst index e2e55a98f..c3dfcbbff 100644 --- a/docs/src/piccolo/authentication/baseuser.rst +++ b/docs/src/piccolo/authentication/baseuser.rst @@ -143,7 +143,9 @@ Limits ------ The maximum password length allowed is 128 characters. This should be -sufficiently long for most use cases. +sufficiently long for most use cases. + +The minimum password length allowed is 6 characters. ------------------------------------------------------------------------------- diff --git a/piccolo/apps/user/tables.py b/piccolo/apps/user/tables.py index a9a389101..1d61dfc2b 100644 --- a/piccolo/apps/user/tables.py +++ b/piccolo/apps/user/tables.py @@ -91,10 +91,14 @@ def _validate_password(cls, password: str): raise ValueError("A password must be provided.") if len(password) < cls._min_password_length: - raise ValueError("The password is too short.") + raise ValueError( + f"The password is too short. (min {cls._min_password_length})" + ) if len(password) > cls._max_password_length: - raise ValueError("The password is too long.") + raise ValueError( + f"The password is too long. (max {cls._max_password_length})" + ) if password.startswith("pbkdf2_sha256"): logger.warning( diff --git a/tests/apps/user/test_tables.py b/tests/apps/user/test_tables.py index 59d274905..123bb5a66 100644 --- a/tests/apps/user/test_tables.py +++ b/tests/apps/user/test_tables.py @@ -109,7 +109,7 @@ def test_update_password(self): BaseUser.update_password_sync(username, malicious_password) self.assertEqual( manager.exception.__str__(), - "The password is too long.", + f"The password is too long. (max {BaseUser._max_password_length})", ) # Test short passwords @@ -118,7 +118,10 @@ def test_update_password(self): BaseUser.update_password_sync(username, short_password) self.assertEqual( manager.exception.__str__(), - "The password is too short.", + ( + "The password is too short. (min " + f"{BaseUser._min_password_length})" + ), ) # Test no password @@ -205,7 +208,11 @@ def test_short_password_error(self): BaseUser.create_user_sync(username="bob", password="abc") self.assertEqual( - manager.exception.__str__(), "The password is too short." + manager.exception.__str__(), + ( + "The password is too short. (min " + f"{BaseUser._min_password_length})" + ), ) def test_long_password_error(self): @@ -216,7 +223,8 @@ def test_long_password_error(self): ) self.assertEqual( - manager.exception.__str__(), "The password is too long." + manager.exception.__str__(), + f"The password is too long. (max {BaseUser._max_password_length})", ) def test_no_username_error(self): From 31a76ebb6b2228e79f85a9a3fc2bfe76d917d6f9 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 28 Jul 2024 13:21:30 +0100 Subject: [PATCH 604/727] 1056 make refresh work with prefetch (#1058) * make `refresh` work with `prefetch` * update comment * make refresh work multiple levels deep * add tests, and raise an exception if trying to refresh a single column on a child object --- docs/src/piccolo/query_types/objects.rst | 14 +++ piccolo/query/methods/refresh.py | 83 ++++++++++++-- tests/table/test_refresh.py | 132 +++++++++++++++++++++-- 3 files changed, 214 insertions(+), 15 deletions(-) diff --git a/docs/src/piccolo/query_types/objects.rst b/docs/src/piccolo/query_types/objects.rst index 123f35756..0cb3c1568 100644 --- a/docs/src/piccolo/query_types/objects.rst +++ b/docs/src/piccolo/query_types/objects.rst @@ -282,6 +282,20 @@ has the latest data from the database, you can use the # Or just refresh certain columns: await band.refresh([Band.name]) +It works with ``prefetch`` too: + +.. code-block:: python + + # If we have an instance with a child object: + band = await Band.objects(Band.manager).first() + + # And it has gotten stale, we can refresh it: + await band.refresh() + + # The nested object will also be updated if it was stale: + >>> band.manager.name + "New value" + ------------------------------------------------------------------------------- Query clauses diff --git a/piccolo/query/methods/refresh.py b/piccolo/query/methods/refresh.py index 0d0258a6a..fe2355996 100644 --- a/piccolo/query/methods/refresh.py +++ b/piccolo/query/methods/refresh.py @@ -1,7 +1,6 @@ from __future__ import annotations import typing as t -from dataclasses import dataclass from piccolo.utils.sync import run_sync @@ -10,7 +9,6 @@ from piccolo.table import Table -@dataclass class Refresh: """ Used to refresh :class:`Table ` instances with the @@ -25,8 +23,25 @@ class Refresh: """ - instance: Table - columns: t.Optional[t.Sequence[Column]] = None + def __init__( + self, + instance: Table, + columns: t.Optional[t.Sequence[Column]] = None, + ): + self.instance = instance + + if columns: + for column in columns: + if len(column._meta.call_chain) > 0: + raise ValueError( + "We can't currently selectively refresh certain " + "columns on child objects (e.g. Concert.band_1.name). " + "Please just specify top level columns (e.g. " + "Concert.band_1), and the entire child object will be " + "refreshed." + ) + + self.columns = columns @property def _columns(self) -> t.Sequence[Column]: @@ -40,6 +55,47 @@ def _columns(self) -> t.Sequence[Column]: i for i in self.instance._meta.columns if not i._meta.primary_key ] + def _get_columns(self, instance: Table, columns: t.Sequence[Column]): + """ + If `prefetch` was used on the object, for example:: + + >>> await Band.objects(Band.manager) + + We should also update the prefetched object. + + It works multiple level deep. If we refresh this:: + + >>> await Album.objects(Album.band.manager).first() + + It will update the nested `band` object, and also the `manager` + object. + + """ + from piccolo.columns.column_types import ForeignKey + from piccolo.table import Table + + select_columns = [] + + for column in columns: + if isinstance(column, ForeignKey) and isinstance( + (child_instance := getattr(instance, column._meta.name)), + Table, + ): + select_columns.extend( + self._get_columns( + child_instance, + column.all_columns( + exclude=[ + child_instance.__class__._meta.primary_key + ] + ), + ) + ) + else: + select_columns.append(column) + + return select_columns + async def run( self, in_pool: bool = True, node: t.Optional[str] = None ) -> Table: @@ -54,7 +110,6 @@ async def run( Modifies the instance in place, but also returns it as a convenience. """ - instance = self.instance if not instance._exists_in_db: @@ -71,8 +126,12 @@ async def run( if not columns: raise ValueError("No columns to fetch.") + select_columns = self._get_columns( + instance=self.instance, columns=columns + ) + updated_values = ( - await instance.__class__.select(*columns) + await instance.__class__.select(*select_columns) .where(pk_column == primary_key_value) .first() .run(node=node, in_pool=in_pool) @@ -84,7 +143,17 @@ async def run( ) for key, value in updated_values.items(): - setattr(instance, key, value) + # For prefetched objects, make sure we update them correctly + object_to_update = instance + column_name = key + + if "." in key: + path = key.split(".") + column_name = path.pop() + for i in path: + object_to_update = getattr(object_to_update, i) + + setattr(object_to_update, column_name, value) return instance diff --git a/tests/table/test_refresh.py b/tests/table/test_refresh.py index ffb78ddb8..f1ff93074 100644 --- a/tests/table/test_refresh.py +++ b/tests/table/test_refresh.py @@ -1,5 +1,5 @@ -from tests.base import DBTestCase -from tests.example_apps.music.tables import Band +from tests.base import DBTestCase, TableTest +from tests.example_apps.music.tables import Band, Concert, Manager, Venue class TestRefresh(DBTestCase): @@ -24,9 +24,55 @@ def test_refresh(self) -> None: # Refresh `band`, and make sure it has the correct data. band.refresh().run_sync() - self.assertTrue(band.name == "Pythonistas!!!") - self.assertTrue(band.popularity == 8000) - self.assertTrue(band.id == initial_data["id"]) + self.assertEqual(band.name, "Pythonistas!!!") + self.assertEqual(band.popularity, 8000) + self.assertEqual(band.id, initial_data["id"]) + + def test_refresh_with_prefetch(self) -> None: + """ + Make sure ``refresh`` works, when the object used prefetch to get + nested objets (the nested objects should be updated too). + """ + band = ( + Band.objects(Band.manager) + .where(Band.name == "Pythonistas") + .first() + .run_sync() + ) + assert band is not None + + # Modify the data in the database. + Manager.update({Manager.name: "Guido!!!"}).where( + Manager.name == "Guido" + ).run_sync() + + # Refresh `band`, and make sure it has the correct data. + band.refresh().run_sync() + + self.assertEqual(band.manager.name, "Guido!!!") + + def test_refresh_with_prefetch_multiple_layers_deep(self) -> None: + """ + Make sure ``refresh`` works, when the object used prefetch to get + nested objets (the nested objects should be updated too). + """ + band = ( + Band.objects(Band.manager) + .where(Band.name == "Pythonistas") + .first() + .run_sync() + ) + assert band is not None + + # Modify the data in the database. + Manager.update({Manager.name: "Guido!!!"}).where( + Manager.name == "Guido" + ).run_sync() + + # Refresh `band`, and make sure it has the correct data. + band.refresh().run_sync() + + self.assertEqual(band.manager.name, "Guido!!!") def test_columns(self) -> None: """ @@ -50,9 +96,9 @@ def test_columns(self) -> None: ) query.run_sync() - self.assertTrue(band.name == "Pythonistas!!!") - self.assertTrue(band.popularity == initial_data["popularity"]) - self.assertTrue(band.id == initial_data["id"]) + self.assertEqual(band.name, "Pythonistas!!!") + self.assertEqual(band.popularity, initial_data["popularity"]) + self.assertEqual(band.id, initial_data["id"]) def test_error_when_not_in_db(self) -> None: """ @@ -85,3 +131,73 @@ def test_error_when_pk_in_none(self) -> None: "The instance's primary key value isn't defined.", str(manager.exception), ) + + +class TestRefreshWithPrefetch(TableTest): + + tables = [Manager, Band, Concert, Venue] + + def setUp(self): + super().setUp() + + self.manager = Manager({Manager.name: "Guido"}) + self.manager.save().run_sync() + + self.band = Band( + {Band.name: "Pythonistas", Band.manager: self.manager} + ) + self.band.save().run_sync() + + self.concert = Concert({Concert.band_1: self.band}) + self.concert.save().run_sync() + + def test_single_layer(self) -> None: + """ + Make sure ``refresh`` works, when the object used prefetch to get + nested objects (the nested objects should be updated too). + """ + band = ( + Band.objects(Band.manager) + .where(Band.name == "Pythonistas") + .first() + .run_sync() + ) + assert band is not None + + # Modify the data in the database. + self.manager.name = "Guido!!!" + self.manager.save().run_sync() + + # Refresh `band`, and make sure it has the correct data. + band.refresh().run_sync() + self.assertEqual(self.band.manager.name, "Guido!!!") + + def test_multiple_layers(self) -> None: + """ + Make sure ``refresh`` works when ``prefetch`` was used to fetch objects + multiple layers deep. + """ + concert = ( + Concert.objects(Concert.band_1._.manager) + .where(Concert.band_1._.name == "Pythonistas") + .first() + ) + assert concert is not None + + # Modify the data in the database. + self.manager.name = "Guido!!!" + self.manager.save().run_sync() + + self.concert.refresh().run_sync() + self.assertEqual(self.concert.band_1.manager.name, "Guido!!!") + + def test_exception(self) -> None: + """ + We don't currently let the user refresh specific fields from nested + objects - an exception should be raised. + """ + with self.assertRaises(ValueError): + self.concert.refresh(columns=[Concert.band_1._.manager]).run_sync() + + # Shouldn't raise an exception: + self.concert.refresh(columns=[Concert.band_1]).run_sync() From 9a4a91940e80726c453f48a29e7a314d273bd1c8 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 29 Jul 2024 20:46:24 +0100 Subject: [PATCH 605/727] fix `refresh` if foreign key value has changed (#1059) --- piccolo/query/methods/refresh.py | 8 +++----- tests/table/test_refresh.py | 30 +++++++++++++++++++++++++++--- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/piccolo/query/methods/refresh.py b/piccolo/query/methods/refresh.py index fe2355996..2e71abf94 100644 --- a/piccolo/query/methods/refresh.py +++ b/piccolo/query/methods/refresh.py @@ -84,11 +84,9 @@ def _get_columns(self, instance: Table, columns: t.Sequence[Column]): select_columns.extend( self._get_columns( child_instance, - column.all_columns( - exclude=[ - child_instance.__class__._meta.primary_key - ] - ), + # Fetch all columns (even the primary key, just in + # case the foreign key now references a different row). + column.all_columns(), ) ) else: diff --git a/tests/table/test_refresh.py b/tests/table/test_refresh.py index f1ff93074..2c8b0ca7c 100644 --- a/tests/table/test_refresh.py +++ b/tests/table/test_refresh.py @@ -170,7 +170,7 @@ def test_single_layer(self) -> None: # Refresh `band`, and make sure it has the correct data. band.refresh().run_sync() - self.assertEqual(self.band.manager.name, "Guido!!!") + self.assertEqual(band.manager.name, "Guido!!!") def test_multiple_layers(self) -> None: """ @@ -181,6 +181,7 @@ def test_multiple_layers(self) -> None: Concert.objects(Concert.band_1._.manager) .where(Concert.band_1._.name == "Pythonistas") .first() + .run_sync() ) assert concert is not None @@ -188,8 +189,31 @@ def test_multiple_layers(self) -> None: self.manager.name = "Guido!!!" self.manager.save().run_sync() - self.concert.refresh().run_sync() - self.assertEqual(self.concert.band_1.manager.name, "Guido!!!") + concert.refresh().run_sync() + self.assertEqual(concert.band_1.manager.name, "Guido!!!") + + def test_updated_foreign_key(self) -> None: + """ + If a foreign key now references a different row, make sure this + is refreshed correctly. + """ + band = ( + Band.objects(Band.manager) + .where(Band.name == "Pythonistas") + .first() + .run_sync() + ) + assert band is not None + + # Assign a different manager to the band + new_manager = Manager({Manager.name: "New Manager"}) + new_manager.save().run_sync() + Band.update({Band.manager: new_manager.id}, force=True).run_sync() + + # Refresh `band`, and make sure it references the new manager. + band.refresh().run_sync() + self.assertEqual(band.manager.id, new_manager.id) + self.assertEqual(band.manager.name, "New Manager") def test_exception(self) -> None: """ From 8c1563e4e69e4cf0b849c02b4d93282703cbe6f9 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 30 Jul 2024 10:06:43 +0100 Subject: [PATCH 606/727] 1056 Make `refresh` work when foreign key is set to null, and with `load_json` (#1060) * make `refresh` work when fk is set to null, and with `load_json` * add missing `return` * fix tests * mention in the docs how `refresh` is useful in unit tests * update docs * add `TestRefreshWithLoadJSON` * improve refresh docs * fix test --- docs/src/piccolo/query_types/objects.rst | 14 +++++ piccolo/query/base.py | 4 +- piccolo/query/methods/refresh.py | 42 +++++++++----- piccolo/query/methods/select.py | 3 + piccolo/table.py | 10 +++- piccolo/utils/encoding.py | 43 +++++++++++++- tests/columns/m2m/test_m2m.py | 7 +++ tests/table/test_refresh.py | 72 +++++++++++++++++++++++- 8 files changed, 174 insertions(+), 21 deletions(-) diff --git a/docs/src/piccolo/query_types/objects.rst b/docs/src/piccolo/query_types/objects.rst index 0cb3c1568..968aefcab 100644 --- a/docs/src/piccolo/query_types/objects.rst +++ b/docs/src/piccolo/query_types/objects.rst @@ -296,6 +296,20 @@ It works with ``prefetch`` too: >>> band.manager.name "New value" +``refresh`` is very useful in unit tests: + +.. code-block:: python + + # If we have an instance: + band = await Band.objects().where(Band.name == "Pythonistas").first() + + # Call an API endpoint which updates the object (e.g. with httpx): + await client.patch(f"/band/{band.id}/", json={"popularity": 5000}) + + # Make sure the instance was updated: + await band.refresh() + assert band.popularity == 5000 + ------------------------------------------------------------------------------- Query clauses diff --git a/piccolo/query/base.py b/piccolo/query/base.py index 444736f03..ff49c0e3b 100644 --- a/piccolo/query/base.py +++ b/piccolo/query/base.py @@ -79,9 +79,7 @@ async def _process_results(self, results) -> QueryResponseType: if column._alias is not None: json_column_names.append(column._alias) elif len(column._meta.call_chain) > 0: - json_column_names.append( - column._meta.get_default_alias().replace("$", ".") - ) + json_column_names.append(column._meta.get_default_alias()) else: json_column_names.append(column._meta.name) diff --git a/piccolo/query/methods/refresh.py b/piccolo/query/methods/refresh.py index 2e71abf94..5807285be 100644 --- a/piccolo/query/methods/refresh.py +++ b/piccolo/query/methods/refresh.py @@ -2,6 +2,7 @@ import typing as t +from piccolo.utils.encoding import JSONDict from piccolo.utils.sync import run_sync if t.TYPE_CHECKING: # pragma: no cover @@ -20,6 +21,9 @@ class Refresh: :param columns: Which columns to refresh - it not specified, then all columns are refreshed. + :param load_json: + Whether to load ``JSON`` / ``JSONB`` columns as objects, instead of + just a string. """ @@ -27,6 +31,7 @@ def __init__( self, instance: Table, columns: t.Optional[t.Sequence[Column]] = None, + load_json: bool = False, ): self.instance = instance @@ -42,6 +47,7 @@ def __init__( ) self.columns = columns + self.load_json = load_json @property def _columns(self) -> t.Sequence[Column]: @@ -94,6 +100,24 @@ def _get_columns(self, instance: Table, columns: t.Sequence[Column]): return select_columns + def _update_instance(self, instance: Table, data_dict: t.Dict): + """ + Update the table instance. It is called recursively, if the instance + has child instances. + """ + for key, value in data_dict.items(): + if isinstance(value, dict) and not isinstance(value, JSONDict): + # If the value is a dict, then it's a child instance. + if all(i is None for i in value.values()): + # If all values in the nested object are None, then we can + # safely assume that the object itself is null, as the + # primary key value must be null. + setattr(instance, key, None) + else: + self._update_instance(getattr(instance, key), value) + else: + setattr(instance, key, value) + async def run( self, in_pool: bool = True, node: t.Optional[str] = None ) -> Table: @@ -128,30 +152,20 @@ async def run( instance=self.instance, columns=columns ) - updated_values = ( + data_dict = ( await instance.__class__.select(*select_columns) .where(pk_column == primary_key_value) + .output(nested=True, load_json=self.load_json) .first() .run(node=node, in_pool=in_pool) ) - if updated_values is None: + if data_dict is None: raise ValueError( "The object doesn't exist in the database any more." ) - for key, value in updated_values.items(): - # For prefetched objects, make sure we update them correctly - object_to_update = instance - column_name = key - - if "." in key: - path = key.split(".") - column_name = path.pop() - for i in path: - object_to_update = getattr(object_to_update, i) - - setattr(object_to_update, column_name, value) + self._update_instance(instance=instance, data_dict=data_dict) return instance diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 55267b703..0c590918b 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -406,6 +406,9 @@ def output(self: Self, *, load_json: bool) -> Self: ... def output(self: Self, *, load_json: bool, as_list: bool) -> SelectJSON: # type: ignore # noqa: E501 ... + @t.overload + def output(self: Self, *, load_json: bool, nested: bool) -> Self: ... + @t.overload def output(self: Self, *, nested: bool) -> Self: ... diff --git a/piccolo/table.py b/piccolo/table.py index 3b3ff4853..af0fed2fe 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -541,7 +541,9 @@ def remove(self) -> Delete: ) def refresh( - self, columns: t.Optional[t.Sequence[Column]] = None + self, + columns: t.Optional[t.Sequence[Column]] = None, + load_json: bool = False, ) -> Refresh: """ Used to fetch the latest data for this instance from the database. @@ -551,6 +553,10 @@ def refresh( If you only want to refresh certain columns, specify them here. Otherwise all columns are refreshed. + :param load_json: + Whether to load ``JSON`` / ``JSONB`` columns as objects, instead of + just a string. + Example usage:: # Get an instance from the database. @@ -564,7 +570,7 @@ def refresh( instance.refresh().run_sync() """ - return Refresh(instance=self, columns=columns) + return Refresh(instance=self, columns=columns, load_json=load_json) @t.overload def get_related( diff --git a/piccolo/utils/encoding.py b/piccolo/utils/encoding.py index 97fde4683..029ae37a3 100644 --- a/piccolo/utils/encoding.py +++ b/piccolo/utils/encoding.py @@ -29,5 +29,46 @@ def dump_json(data: t.Any, pretty: bool = False) -> str: return json.dumps(data, **params) # type: ignore +class JSONDict(dict): + """ + Once we have parsed a JSON string into a dictionary, we can't distinguish + it from other dictionaries. + + Sometimes we might want to - for example:: + + >>> await Album.select( + ... Album.all_columns(), + ... Album.recording_studio.all_columns() + ... ).output( + ... nested=True, + ... load_json=True + ... ) + + [{ + 'id': 1, + 'band': 1, + 'name': 'Awesome album 1', + 'recorded_at': { + 'id': 1, + 'facilities': {'restaurant': True, 'mixing_desk': True}, + 'name': 'Abbey Road' + }, + 'release_date': datetime.date(2021, 1, 1) + }] + + Facilities could be mistaken for a table. + + """ + + ... + + def load_json(data: str) -> t.Any: - return orjson.loads(data) if ORJSON else json.loads(data) # type: ignore + response = ( + orjson.loads(data) if ORJSON else json.loads(data) # type: ignore + ) + + if isinstance(response, dict): + return JSONDict(**response) + + return response diff --git a/tests/columns/m2m/test_m2m.py b/tests/columns/m2m/test_m2m.py index 85731a715..c2b9d1f42 100644 --- a/tests/columns/m2m/test_m2m.py +++ b/tests/columns/m2m/test_m2m.py @@ -4,6 +4,7 @@ import uuid from unittest import TestCase +from piccolo.utils.encoding import JSONDict from tests.base import engines_skip try: @@ -376,6 +377,9 @@ def test_select_single(self): if isinstance(column, UUID): self.assertIn(type(returned_value), (uuid.UUID, asyncpgUUID)) + elif isinstance(column, (JSON, JSONB)): + self.assertEqual(type(returned_value), JSONDict) + self.assertEqual(original_value, returned_value) else: self.assertEqual( type(original_value), @@ -401,6 +405,9 @@ def test_select_single(self): if isinstance(column, UUID): self.assertIn(type(returned_value), (uuid.UUID, asyncpgUUID)) self.assertEqual(str(original_value), str(returned_value)) + elif isinstance(column, (JSON, JSONB)): + self.assertEqual(type(returned_value), JSONDict) + self.assertEqual(original_value, returned_value) else: self.assertEqual( type(original_value), diff --git a/tests/table/test_refresh.py b/tests/table/test_refresh.py index 2c8b0ca7c..2bb9124ec 100644 --- a/tests/table/test_refresh.py +++ b/tests/table/test_refresh.py @@ -1,5 +1,13 @@ +import typing as t + from tests.base import DBTestCase, TableTest -from tests.example_apps.music.tables import Band, Concert, Manager, Venue +from tests.example_apps.music.tables import ( + Band, + Concert, + Manager, + RecordingStudio, + Venue, +) class TestRefresh(DBTestCase): @@ -215,6 +223,27 @@ def test_updated_foreign_key(self) -> None: self.assertEqual(band.manager.id, new_manager.id) self.assertEqual(band.manager.name, "New Manager") + def test_foreign_key_set_to_null(self): + """ + Make sure that if the foreign key was set to null, that ``refresh`` + sets the nested object to ``None``. + """ + band = ( + Band.objects(Band.manager) + .where(Band.name == "Pythonistas") + .first() + .run_sync() + ) + assert band is not None + + # Remove the manager from band + Band.update({Band.manager: None}, force=True).run_sync() + + # Refresh `band`, and make sure the foreign key value is now `None`, + # instead of a nested object. + band.refresh().run_sync() + self.assertIsNone(band.manager) + def test_exception(self) -> None: """ We don't currently let the user refresh specific fields from nested @@ -225,3 +254,44 @@ def test_exception(self) -> None: # Shouldn't raise an exception: self.concert.refresh(columns=[Concert.band_1]).run_sync() + + +class TestRefreshWithLoadJSON(TableTest): + + tables = [RecordingStudio] + + def setUp(self): + super().setUp() + + self.recording_studio = RecordingStudio( + {RecordingStudio.facilities: {"piano": True}} + ) + self.recording_studio.save().run_sync() + + def test_load_json(self): + """ + Make sure we can refresh an object, and load the JSON as a Python + object. + """ + RecordingStudio.update( + {RecordingStudio.facilities: {"electric piano": True}}, + force=True, + ).run_sync() + + # Refresh without load_json: + self.recording_studio.refresh().run_sync() + + self.assertEqual( + # Remove the white space, because some versions of Python add + # whitespace around JSON, and some don't. + self.recording_studio.facilities.replace(" ", ""), + '{"electricpiano":true}', + ) + + # Refresh with load_json: + self.recording_studio.refresh(load_json=True).run_sync() + + self.assertDictEqual( + t.cast(dict, self.recording_studio.facilities), + {"electric piano": True}, + ) From bb37027d2982ebb7ff69e37ca83c5a3fd1708707 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 30 Jul 2024 10:11:29 +0100 Subject: [PATCH 607/727] bumped version --- CHANGES.rst | 22 ++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index cb258767c..6e8e20940 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,28 @@ Changes ======= +1.15.0 +------ + +Improved ``refresh`` - it now works with prefetched objects. For example: + +.. code-block:: python + + >>> band = await Band.objects(Band.manager).first() + >>> band.manager.name + "Guido" + + # If the manager has changed in the database, when we refresh the band, the + # manager object will also be updated: + >>> await band.refresh() + >>> band.manager.name + "New name" + +Also, improved the error messages when creating a ``BaseUser`` - thanks to +@haaavk for this. + +------------------------------------------------------------------------------- + 1.14.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index d27c8855a..b618c4cf5 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.14.0" +__VERSION__ = "1.15.0" From 872d58f81ce943a76eb9df19cb94d32984b83614 Mon Sep 17 00:00:00 2001 From: havk Date: Wed, 31 Jul 2024 21:13:47 +0200 Subject: [PATCH 608/727] Add db drivers installation step to doc (#1062) Co-authored-by: napcode --- docs/src/piccolo/contributing/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/src/piccolo/contributing/index.rst b/docs/src/piccolo/contributing/index.rst index 52b37d72f..7a53e1dd0 100644 --- a/docs/src/piccolo/contributing/index.rst +++ b/docs/src/piccolo/contributing/index.rst @@ -36,6 +36,7 @@ Get the tests running * Install default dependencies: ``pip install -r requirements/requirements.txt`` * Install development dependencies: ``pip install -r requirements/dev-requirements.txt`` * Install test dependencies: ``pip install -r requirements/test-requirements.txt`` +* Install database drivers: ``pip install -r requirements/extras/postgres.txt -r requirements/extras/sqlite.txt`` * Setup Postgres, and make sure a database called ``piccolo`` exists (see ``tests/postgres_conf.py``). * Run the automated code linting/formatting tools: ``./scripts/lint.sh`` * Run the test suite with Postgres: ``./scripts/test-postgres.sh`` From 785a72b1060f21cb6b50532ebc8ecdd65fc13498 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 8 Aug 2024 13:48:37 +0100 Subject: [PATCH 609/727] EN-1063 Add Piccolo `TestCase` subclasses (#1064) * add test cases * remove accidentally changes * make sure transaction test gets rolled back * don't run tests on older Python versions * update docs * fix typo * fix more typos * one more typo in the docs --- docs/src/piccolo/testing/index.rst | 93 +++++++++++---- piccolo/testing/test_case.py | 120 ++++++++++++++++++++ tests/columns/foreign_key/test_reverse.py | 2 +- tests/columns/test_array.py | 3 +- tests/columns/test_bigint.py | 3 +- tests/columns/test_boolean.py | 2 +- tests/columns/test_bytea.py | 2 +- tests/columns/test_choices.py | 3 +- tests/columns/test_date.py | 2 +- tests/columns/test_double_precision.py | 2 +- tests/columns/test_interval.py | 2 +- tests/columns/test_json.py | 2 +- tests/columns/test_jsonb.py | 3 +- tests/columns/test_numeric.py | 2 +- tests/columns/test_primary_key.py | 2 +- tests/columns/test_readable.py | 2 +- tests/columns/test_real.py | 2 +- tests/columns/test_reference.py | 2 +- tests/columns/test_reserved_column_names.py | 2 +- tests/columns/test_smallint.py | 4 +- tests/columns/test_time.py | 3 +- tests/columns/test_timestamp.py | 2 +- tests/columns/test_timestamptz.py | 2 +- tests/columns/test_uuid.py | 2 +- tests/columns/test_varchar.py | 4 +- tests/query/functions/base.py | 2 +- tests/query/functions/test_datetime.py | 3 +- tests/query/functions/test_math.py | 2 +- tests/table/test_refresh.py | 3 +- tests/testing/test_test_case.py | 65 +++++++++++ 30 files changed, 291 insertions(+), 52 deletions(-) create mode 100644 piccolo/testing/test_case.py create mode 100644 tests/testing/test_test_case.py diff --git a/docs/src/piccolo/testing/index.rst b/docs/src/piccolo/testing/index.rst index 39791ab0d..84eabff83 100644 --- a/docs/src/piccolo/testing/index.rst +++ b/docs/src/piccolo/testing/index.rst @@ -100,52 +100,67 @@ Creating the test schema When running your unit tests, you usually start with a blank test database, create the tables, and then install test data. -To create the tables, there are a few different approaches you can take. Here -we use :func:`create_db_tables_sync ` and -:func:`drop_db_tables_sync `. +To create the tables, there are a few different approaches you can take. + +``create_db_tables`` / ``drop_db_tables`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Here we use :func:`create_db_tables ` and +:func:`drop_db_tables ` to create and drop the +tables. .. note:: - The async equivalents are :func:`create_db_tables ` - and :func:`drop_db_tables `. + The sync equivalents are :func:`create_db_tables_sync ` + and :func:`drop_db_tables_sync `, if + you need your tests to be synchronous for some reason. .. code-block:: python - from unittest import TestCase + from unittest import IsolatedAsyncioTestCase - from piccolo.table import create_db_tables_sync, drop_db_tables_sync + from piccolo.table import create_db_tables, drop_db_tables from piccolo.conf.apps import Finder + TABLES = Finder().get_table_classes() - class TestApp(TestCase): - def setUp(self): - create_db_tables_sync(*TABLES) - def tearDown(self): - drop_db_tables_sync(*TABLES) + class TestApp(IsolatedAsyncioTestCase): + async def setUp(self): + await create_db_tables(*TABLES) + + async def tearDown(self): + await drop_db_tables(*TABLES) - def test_app(self): + async def test_app(self): # Do some testing ... pass +You can remove this boiler plate by using +:class:`AsyncTransactionTest `, +which does this for you. + +Run migrations +~~~~~~~~~~~~~~ + Alternatively, you can run the migrations to setup the schema if you prefer: .. code-block:: python - from unittest import TestCase + from unittest import IsolatedAsyncioTestCase from piccolo.apps.migrations.commands.backwards import run_backwards from piccolo.apps.migrations.commands.forwards import run_forwards - from piccolo.utils.sync import run_sync - class TestApp(TestCase): - def setUp(self): - run_sync(run_forwards("all")) - def tearDown(self): - run_sync(run_backwards("all", auto_agree=True)) + class TestApp(IsolatedAsyncioTestCase): + async def setUp(self): + await run_forwards("all") - def test_app(self): + async def tearDown(self): + await run_backwards("all", auto_agree=True) + + async def test_app(self): # Do some testing ... pass @@ -156,7 +171,10 @@ Testing async code There are a few options for testing async code using pytest. -You can either call any async code using Piccolo's ``run_sync`` utility: +``run_sync`` +~~~~~~~~~~~~ + +You can call any async code using Piccolo's ``run_sync`` utility: .. code-block:: python @@ -169,7 +187,10 @@ You can either call any async code using Piccolo's ``run_sync`` utility: rows = run_sync(get_data()) assert len(rows) == 1 -Alternatively, you can make your tests natively async. +It's preferable to make your tests natively async though. + +``pytest-asyncio`` +~~~~~~~~~~~~~~~~~~ If you prefer using pytest's function based tests, then take a look at `pytest-asyncio `_. Simply @@ -182,6 +203,9 @@ like this: rows = await MyTable.select() assert len(rows) == 1 +``IsolatedAsyncioTestCase`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + If you prefer class based tests, and are using Python 3.8 or above, then have a look at :class:`IsolatedAsyncioTestCase ` from Python's standard library. You can then write tests like this: @@ -194,3 +218,26 @@ from Python's standard library. You can then write tests like this: async def test_select(self): rows = await MyTable.select() assert len(rows) == 1 + +Also look at the ``IsolatedAsyncioTestCase`` subclasses which Piccolo provides +(see :class:`AsyncTransactionTest ` +and :class:`AsyncTableTest ` below). + +------------------------------------------------------------------------------- + +``TestCase`` subclasses +----------------------- + +Piccolo ships with some ``unittest.TestCase`` subclasses which remove +boilerplate code from tests. + +.. currentmodule:: piccolo.testing.test_case + +.. autoclass:: AsyncTransactionTest + :class-doc-from: class + +.. autoclass:: AsyncTableTest + :class-doc-from: class + +.. autoclass:: TableTest + :class-doc-from: class diff --git a/piccolo/testing/test_case.py b/piccolo/testing/test_case.py new file mode 100644 index 000000000..085f1e14c --- /dev/null +++ b/piccolo/testing/test_case.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +import typing as t +from unittest import IsolatedAsyncioTestCase, TestCase + +from piccolo.engine import Engine, engine_finder +from piccolo.table import ( + Table, + create_db_tables, + create_db_tables_sync, + drop_db_tables, + drop_db_tables_sync, +) + + +class TableTest(TestCase): + """ + Identical to :class:`AsyncTableTest `, + except it only work for sync tests. Only use this if you can't make your + tests async (perhaps you're on Python 3.7 where ``IsolatedAsyncioTestCase`` + isn't available). + + For example:: + + class TestBand(TableTest): + tables = [Band] + + def test_example(self): + ... + + """ # noqa: E501 + + tables: t.List[t.Type[Table]] + + def setUp(self) -> None: + create_db_tables_sync(*self.tables) + + def tearDown(self) -> None: + drop_db_tables_sync(*self.tables) + + +class AsyncTableTest(IsolatedAsyncioTestCase): + """ + Used for tests where we need to create Piccolo tables - they will + automatically be created and dropped. + + For example:: + + class TestBand(AsyncTableTest): + tables = [Band] + + async def test_example(self): + ... + + """ + + tables: t.List[t.Type[Table]] + + async def asyncSetUp(self) -> None: + await create_db_tables(*self.tables) + + async def asyncTearDown(self) -> None: + await drop_db_tables(*self.tables) + + +class AsyncTransactionTest(IsolatedAsyncioTestCase): + """ + Wraps each test in a transaction, which is automatically rolled back when + the test finishes. + + .. warning:: + Python 3.11 and above only. + + If your test suite just contains ``AsyncTransactionTest`` tests, then you + can setup your database tables once before your test suite runs. Any + changes made to your tables by the tests will be rolled back automatically. + + Here's an example:: + + from piccolo.testing.test_case import AsyncTransactionTest + + + class TestBandEndpoint(AsyncTransactionTest): + + async def test_band_response(self): + \"\"\" + Make sure the endpoint returns a 200. + \"\"\" + band = Band({Band.name: "Pythonistas"}) + await band.save() + + # Using an API testing client, like httpx: + response = await client.get(f"/bands/{band.id}/") + self.assertEqual(response.status_code, 200) + + We add a ``Band`` to the database, but any subsequent tests won't see it, + as the changes are rolled back automatically. + + """ + + # We use `engine_finder` to find the current `Engine`, but you can + # explicity set it here if you prefer: + # + # class MyTest(AsyncTransactionTest): + # db = DB + # + # ... + # + db: t.Optional[Engine] = None + + async def asyncSetUp(self) -> None: + db = self.db or engine_finder() + assert db is not None + self.transaction = db.transaction() + # This is only available in Python 3.11 and above: + await self.enterAsyncContext(cm=self.transaction) # type: ignore + + async def asyncTearDown(self): + await super().asyncTearDown() + await self.transaction.rollback() diff --git a/tests/columns/foreign_key/test_reverse.py b/tests/columns/foreign_key/test_reverse.py index 2a90ac5ba..5bf490c09 100644 --- a/tests/columns/foreign_key/test_reverse.py +++ b/tests/columns/foreign_key/test_reverse.py @@ -1,6 +1,6 @@ from piccolo.columns import ForeignKey, Text, Varchar from piccolo.table import Table -from tests.base import TableTest +from piccolo.testing.test_case import TableTest class Band(Table): diff --git a/tests/columns/test_array.py b/tests/columns/test_array.py index b28204231..085331671 100644 --- a/tests/columns/test_array.py +++ b/tests/columns/test_array.py @@ -14,7 +14,8 @@ ) from piccolo.querystring import QueryString from piccolo.table import Table -from tests.base import TableTest, engines_only, engines_skip, sqlite_only +from piccolo.testing.test_case import TableTest +from tests.base import engines_only, engines_skip, sqlite_only class MyTable(Table): diff --git a/tests/columns/test_bigint.py b/tests/columns/test_bigint.py index 7f418cced..9cb1b8ae4 100644 --- a/tests/columns/test_bigint.py +++ b/tests/columns/test_bigint.py @@ -2,7 +2,8 @@ from piccolo.columns.column_types import BigInt from piccolo.table import Table -from tests.base import TableTest, engines_only +from piccolo.testing.test_case import TableTest +from tests.base import engines_only class MyTable(Table): diff --git a/tests/columns/test_boolean.py b/tests/columns/test_boolean.py index 57268b24a..08f2a504b 100644 --- a/tests/columns/test_boolean.py +++ b/tests/columns/test_boolean.py @@ -2,7 +2,7 @@ from piccolo.columns.column_types import Boolean from piccolo.table import Table -from tests.base import TableTest +from piccolo.testing.test_case import TableTest class MyTable(Table): diff --git a/tests/columns/test_bytea.py b/tests/columns/test_bytea.py index 4b520083f..8114e9325 100644 --- a/tests/columns/test_bytea.py +++ b/tests/columns/test_bytea.py @@ -1,6 +1,6 @@ from piccolo.columns.column_types import Bytea from piccolo.table import Table -from tests.base import TableTest +from piccolo.testing.test_case import TableTest class MyTable(Table): diff --git a/tests/columns/test_choices.py b/tests/columns/test_choices.py index 05502fe0a..d3e1822e5 100644 --- a/tests/columns/test_choices.py +++ b/tests/columns/test_choices.py @@ -2,7 +2,8 @@ from piccolo.columns.column_types import Array, Varchar from piccolo.table import Table -from tests.base import TableTest, engines_only +from piccolo.testing.test_case import TableTest +from tests.base import engines_only from tests.example_apps.music.tables import Shirt diff --git a/tests/columns/test_date.py b/tests/columns/test_date.py index 7cdfb89d9..1628c4758 100644 --- a/tests/columns/test_date.py +++ b/tests/columns/test_date.py @@ -3,7 +3,7 @@ from piccolo.columns.column_types import Date from piccolo.columns.defaults.date import DateNow from piccolo.table import Table -from tests.base import TableTest +from piccolo.testing.test_case import TableTest class MyTable(Table): diff --git a/tests/columns/test_double_precision.py b/tests/columns/test_double_precision.py index bdc8ff387..e29a0e134 100644 --- a/tests/columns/test_double_precision.py +++ b/tests/columns/test_double_precision.py @@ -1,6 +1,6 @@ from piccolo.columns.column_types import DoublePrecision from piccolo.table import Table -from tests.base import TableTest +from piccolo.testing.test_case import TableTest class MyTable(Table): diff --git a/tests/columns/test_interval.py b/tests/columns/test_interval.py index 2e56e6a84..484038003 100644 --- a/tests/columns/test_interval.py +++ b/tests/columns/test_interval.py @@ -3,7 +3,7 @@ from piccolo.columns.column_types import Interval from piccolo.columns.defaults.interval import IntervalCustom from piccolo.table import Table -from tests.base import TableTest +from piccolo.testing.test_case import TableTest class MyTable(Table): diff --git a/tests/columns/test_json.py b/tests/columns/test_json.py index 8b4d8e6cb..19669c61b 100644 --- a/tests/columns/test_json.py +++ b/tests/columns/test_json.py @@ -1,6 +1,6 @@ from piccolo.columns.column_types import JSON from piccolo.table import Table -from tests.base import TableTest +from piccolo.testing.test_case import TableTest class MyTable(Table): diff --git a/tests/columns/test_jsonb.py b/tests/columns/test_jsonb.py index 1e03aa10d..fe90e769b 100644 --- a/tests/columns/test_jsonb.py +++ b/tests/columns/test_jsonb.py @@ -1,6 +1,7 @@ from piccolo.columns.column_types import JSONB, ForeignKey, Varchar from piccolo.table import Table -from tests.base import TableTest, engines_only, engines_skip +from piccolo.testing.test_case import TableTest +from tests.base import engines_only, engines_skip class RecordingStudio(Table): diff --git a/tests/columns/test_numeric.py b/tests/columns/test_numeric.py index cc8c8a604..22c650c70 100644 --- a/tests/columns/test_numeric.py +++ b/tests/columns/test_numeric.py @@ -2,7 +2,7 @@ from piccolo.columns.column_types import Numeric from piccolo.table import Table -from tests.base import TableTest +from piccolo.testing.test_case import TableTest class MyTable(Table): diff --git a/tests/columns/test_primary_key.py b/tests/columns/test_primary_key.py index df041111e..55bff8ee5 100644 --- a/tests/columns/test_primary_key.py +++ b/tests/columns/test_primary_key.py @@ -8,7 +8,7 @@ Varchar, ) from piccolo.table import Table -from tests.base import TableTest +from piccolo.testing.test_case import TableTest class MyTableDefaultPrimaryKey(Table): diff --git a/tests/columns/test_readable.py b/tests/columns/test_readable.py index 3d433b24c..d04036147 100644 --- a/tests/columns/test_readable.py +++ b/tests/columns/test_readable.py @@ -1,7 +1,7 @@ from piccolo import columns from piccolo.columns.readable import Readable from piccolo.table import Table -from tests.base import TableTest +from piccolo.testing.test_case import TableTest class MyTable(Table): diff --git a/tests/columns/test_real.py b/tests/columns/test_real.py index 63ab6a4fe..a2cef5a75 100644 --- a/tests/columns/test_real.py +++ b/tests/columns/test_real.py @@ -1,6 +1,6 @@ from piccolo.columns.column_types import Real from piccolo.table import Table -from tests.base import TableTest +from piccolo.testing.test_case import TableTest class MyTable(Table): diff --git a/tests/columns/test_reference.py b/tests/columns/test_reference.py index 23e3f5ddd..8a10b6207 100644 --- a/tests/columns/test_reference.py +++ b/tests/columns/test_reference.py @@ -8,7 +8,7 @@ from piccolo.columns import ForeignKey, Varchar from piccolo.columns.reference import LazyTableReference from piccolo.table import Table -from tests.base import TableTest +from piccolo.testing.test_case import TableTest class Band(Table): diff --git a/tests/columns/test_reserved_column_names.py b/tests/columns/test_reserved_column_names.py index b87c8b755..1fc4a464e 100644 --- a/tests/columns/test_reserved_column_names.py +++ b/tests/columns/test_reserved_column_names.py @@ -1,6 +1,6 @@ from piccolo.columns.column_types import Integer, Varchar from piccolo.table import Table -from tests.base import TableTest +from piccolo.testing.test_case import TableTest class Concert(Table): diff --git a/tests/columns/test_smallint.py b/tests/columns/test_smallint.py index 75beb4275..808562322 100644 --- a/tests/columns/test_smallint.py +++ b/tests/columns/test_smallint.py @@ -2,8 +2,8 @@ from piccolo.columns.column_types import SmallInt from piccolo.table import Table - -from ..base import TableTest, engines_only +from piccolo.testing.test_case import TableTest +from tests.base import engines_only class MyTable(Table): diff --git a/tests/columns/test_time.py b/tests/columns/test_time.py index f6ab5e7ef..9fc48aaad 100644 --- a/tests/columns/test_time.py +++ b/tests/columns/test_time.py @@ -4,7 +4,8 @@ from piccolo.columns.column_types import Time from piccolo.columns.defaults.time import TimeNow from piccolo.table import Table -from tests.base import TableTest, engines_skip +from piccolo.testing.test_case import TableTest +from tests.base import engines_skip class MyTable(Table): diff --git a/tests/columns/test_timestamp.py b/tests/columns/test_timestamp.py index 0b82c6e42..084da7c6c 100644 --- a/tests/columns/test_timestamp.py +++ b/tests/columns/test_timestamp.py @@ -3,7 +3,7 @@ from piccolo.columns.column_types import Timestamp from piccolo.columns.defaults.timestamp import TimestampNow from piccolo.table import Table -from tests.base import TableTest +from piccolo.testing.test_case import TableTest class MyTable(Table): diff --git a/tests/columns/test_timestamptz.py b/tests/columns/test_timestamptz.py index 62d83a52b..cf3528b9a 100644 --- a/tests/columns/test_timestamptz.py +++ b/tests/columns/test_timestamptz.py @@ -9,7 +9,7 @@ TimestamptzOffset, ) from piccolo.table import Table -from tests.base import TableTest +from piccolo.testing.test_case import TableTest class MyTable(Table): diff --git a/tests/columns/test_uuid.py b/tests/columns/test_uuid.py index 28be64c8a..3dcce88a1 100644 --- a/tests/columns/test_uuid.py +++ b/tests/columns/test_uuid.py @@ -2,7 +2,7 @@ from piccolo.columns.column_types import UUID from piccolo.table import Table -from tests.base import TableTest +from piccolo.testing.test_case import TableTest class MyTable(Table): diff --git a/tests/columns/test_varchar.py b/tests/columns/test_varchar.py index f433cb4c7..c62a3a0fd 100644 --- a/tests/columns/test_varchar.py +++ b/tests/columns/test_varchar.py @@ -1,7 +1,7 @@ from piccolo.columns.column_types import Varchar from piccolo.table import Table - -from ..base import TableTest, engines_only +from piccolo.testing.test_case import TableTest +from tests.base import engines_only class MyTable(Table): diff --git a/tests/query/functions/base.py b/tests/query/functions/base.py index 623bc1a5a..1549709a6 100644 --- a/tests/query/functions/base.py +++ b/tests/query/functions/base.py @@ -1,4 +1,4 @@ -from tests.base import TableTest +from piccolo.testing.test_case import TableTest from tests.example_apps.music.tables import Band, Manager diff --git a/tests/query/functions/test_datetime.py b/tests/query/functions/test_datetime.py index 360833dc4..382f688ab 100644 --- a/tests/query/functions/test_datetime.py +++ b/tests/query/functions/test_datetime.py @@ -12,7 +12,8 @@ Year, ) from piccolo.table import Table -from tests.base import TableTest, engines_only, sqlite_only +from piccolo.testing.test_case import TableTest +from tests.base import engines_only, sqlite_only class Concert(Table): diff --git a/tests/query/functions/test_math.py b/tests/query/functions/test_math.py index 7029e7857..330645c36 100644 --- a/tests/query/functions/test_math.py +++ b/tests/query/functions/test_math.py @@ -3,7 +3,7 @@ from piccolo.columns import Numeric from piccolo.query.functions.math import Abs, Ceil, Floor, Round from piccolo.table import Table -from tests.base import TableTest +from piccolo.testing.test_case import TableTest class Ticket(Table): diff --git a/tests/table/test_refresh.py b/tests/table/test_refresh.py index 2bb9124ec..dd2109c7b 100644 --- a/tests/table/test_refresh.py +++ b/tests/table/test_refresh.py @@ -1,6 +1,7 @@ import typing as t -from tests.base import DBTestCase, TableTest +from piccolo.testing.test_case import TableTest +from tests.base import DBTestCase from tests.example_apps.music.tables import ( Band, Concert, diff --git a/tests/testing/test_test_case.py b/tests/testing/test_test_case.py new file mode 100644 index 000000000..963a3c371 --- /dev/null +++ b/tests/testing/test_test_case.py @@ -0,0 +1,65 @@ +import sys + +import pytest + +from piccolo.engine import engine_finder +from piccolo.testing.test_case import ( + AsyncTableTest, + AsyncTransactionTest, + TableTest, +) +from tests.example_apps.music.tables import Band, Manager + + +class TestTableTest(TableTest): + """ + Make sure the tables are created automatically. + """ + + tables = [Band, Manager] + + async def test_tables_created(self): + self.assertTrue(Band.table_exists().run_sync()) + self.assertTrue(Manager.table_exists().run_sync()) + + +class TestAsyncTableTest(AsyncTableTest): + """ + Make sure the tables are created automatically in async tests. + """ + + tables = [Band, Manager] + + async def test_tables_created(self): + self.assertTrue(await Band.table_exists()) + self.assertTrue(await Manager.table_exists()) + + +@pytest.mark.skipif(sys.version_info <= (3, 11), reason="Python 3.11 required") +class TestAsyncTransaction(AsyncTransactionTest): + """ + Make sure that the test exists within a transaction. + """ + + async def test_transaction_exists(self): + db = engine_finder() + assert db is not None + self.assertTrue(db.transaction_exists()) + + +@pytest.mark.skipif(sys.version_info <= (3, 11), reason="Python 3.11 required") +class TestAsyncTransactionRolledBack(AsyncTransactionTest): + """ + Make sure that the changes get rolled back automatically. + """ + + async def asyncTearDown(self): + await super().asyncTearDown() + + assert Manager.table_exists().run_sync() is False + + async def test_insert_data(self): + await Manager.create_table() + + manager = Manager({Manager.name: "Guido"}) + await manager.save() From 25897295f9e85c92986e9a8b7ad8939075a912e6 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 8 Aug 2024 14:00:16 +0100 Subject: [PATCH 610/727] bumped version --- CHANGES.rst | 39 ++++++++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- piccolo/testing/test_case.py | 4 ++-- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 6e8e20940..392d84d34 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,45 @@ Changes ======= +1.16.0 +------ + +Added custom async ``TestCase`` subclasses, to help with testing. + +For example ``AsyncTransactionTest``, which wraps each test in a transaction +automatically: + +.. code-block:: python + + class TestBandEndpoint(AsyncTransactionTest): + + async def test_band_response(self): + """ + Make sure the endpoint returns a 200. + """ + # This data automatically gets removed from the database when the + # test finishes: + band = Band({Band.name: "Pythonistas"}) + await band.save() + + # Using an API testing client, like httpx: + response = await client.get(f"/bands/{band.id}/") + self.assertEqual(response.status_code, 200) + +And ``AsyncTableTest``, which automatically creates and drops tables: + +.. code-block:: python + + class TestBand(AsyncTableTest): + + # These tables automatically get created and dropped: + tables = [Band] + + async def test_band(self): + ... + +------------------------------------------------------------------------------- + 1.15.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index b618c4cf5..497db2c05 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.15.0" +__VERSION__ = "1.16.0" diff --git a/piccolo/testing/test_case.py b/piccolo/testing/test_case.py index 085f1e14c..ba01a9cdc 100644 --- a/piccolo/testing/test_case.py +++ b/piccolo/testing/test_case.py @@ -25,7 +25,7 @@ class TableTest(TestCase): class TestBand(TableTest): tables = [Band] - def test_example(self): + def test_band(self): ... """ # noqa: E501 @@ -49,7 +49,7 @@ class AsyncTableTest(IsolatedAsyncioTestCase): class TestBand(AsyncTableTest): tables = [Band] - async def test_example(self): + async def test_band(self): ... """ From 4bf4fbc08da1b4a92139b8457358613420d2eb38 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Fri, 16 Aug 2024 11:16:35 +0200 Subject: [PATCH 611/727] granian support in asgi templates (#1067) --- piccolo/apps/asgi/commands/new.py | 4 ++-- piccolo/apps/asgi/commands/templates/app/main.py.jinja | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index 83d343c40..746c5db9a 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -9,10 +9,10 @@ from jinja2 import Environment, FileSystemLoader TEMPLATE_DIR = os.path.join(os.path.dirname(__file__), "templates/app/") -SERVERS = ["uvicorn", "Hypercorn"] +SERVERS = ["uvicorn", "Hypercorn", "granian"] ROUTER_DEPENDENCIES = { "starlette": ["starlette"], - "fastapi": ["fastapi>=0.100.0"], + "fastapi": ["fastapi>=0.112.1"], "blacksheep": ["blacksheep"], "litestar": ["litestar"], "esmerald": ["esmerald"], diff --git a/piccolo/apps/asgi/commands/templates/app/main.py.jinja b/piccolo/apps/asgi/commands/templates/app/main.py.jinja index 8fb934fbb..f3a0ce3ef 100644 --- a/piccolo/apps/asgi/commands/templates/app/main.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/main.py.jinja @@ -18,4 +18,8 @@ if __name__ == "__main__": asyncio.run(serve(app, CustomConfig())) serve(app) + {% elif server == 'granian' %} + import granian + + granian.Granian("app:app", interface="asgi").serve() {% endif %} From e1d8e10a449821de03bb7df170c53840066fa6e9 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 20 Aug 2024 18:37:23 +0100 Subject: [PATCH 612/727] add `wrap_in_transaction` option (#1069) --- .../apps/migrations/auto/migration_manager.py | 23 +++++++++++- .../migrations/auto/test_migration_manager.py | 37 ++++++++++++++++++- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/piccolo/apps/migrations/auto/migration_manager.py b/piccolo/apps/migrations/auto/migration_manager.py index ededa3a25..31ad5c120 100644 --- a/piccolo/apps/migrations/auto/migration_manager.py +++ b/piccolo/apps/migrations/auto/migration_manager.py @@ -130,11 +130,27 @@ def table_class_names(self) -> t.List[str]: AsyncFunction = t.Callable[[], t.Coroutine] +class SkippedTransaction: + async def __aenter__(self): + print("Automatic transaction disabled") + + async def __aexit__(self, *args, **kwargs): + pass + + @dataclass class MigrationManager: """ Each auto generated migration returns a MigrationManager. It contains all of the schema changes that migration wants to make. + + :param wrap_in_transaction: + By default, the migration is wrapped in a transaction, so if anything + fails, the whole migration will get rolled back. You can disable this + behaviour if you want - for example, in a manual migration you might + want to create the transaction yourself (perhaps you're using + savepoints), or you may want multiple transactions. + """ migration_id: str = "" @@ -166,6 +182,7 @@ class MigrationManager: default_factory=list ) fake: bool = False + wrap_in_transaction: bool = True def add_table( self, @@ -935,7 +952,11 @@ async def run(self, backwards: bool = False): if not engine: raise Exception("Can't find engine") - async with engine.transaction(): + async with ( + engine.transaction() + if self.wrap_in_transaction + else SkippedTransaction() + ): if not self.preview: if direction == "backwards": raw_list = self.raw_backwards diff --git a/tests/apps/migrations/auto/test_migration_manager.py b/tests/apps/migrations/auto/test_migration_manager.py index 872b9b936..6e71846e0 100644 --- a/tests/apps/migrations/auto/test_migration_manager.py +++ b/tests/apps/migrations/auto/test_migration_manager.py @@ -2,7 +2,7 @@ import random import typing as t from io import StringIO -from unittest import TestCase +from unittest import IsolatedAsyncioTestCase, TestCase from unittest.mock import MagicMock, patch from piccolo.apps.migrations.auto.migration_manager import MigrationManager @@ -11,6 +11,7 @@ from piccolo.columns.base import OnDelete, OnUpdate from piccolo.columns.column_types import ForeignKey from piccolo.conf.apps import AppConfig +from piccolo.engine import engine_finder from piccolo.table import Table, sort_table_classes from piccolo.utils.lazy_loader import LazyLoader from tests.base import AsyncMock, DBTestCase, engine_is, engines_only @@ -1052,3 +1053,37 @@ def test_change_table_schema(self): output, ' - 1 [preview forwards]... CREATE SCHEMA IF NOT EXISTS "schema_1"\nALTER TABLE "manager" SET SCHEMA "schema_1"\n', # noqa: E501 ) + + +class TestWrapInTransaction(IsolatedAsyncioTestCase): + + async def test_enabled(self): + """ + Make sure we can wrap the migration in a transaction if we want to. + """ + + async def run(): + db = engine_finder() + assert db + assert db.transaction_exists() is True + + manager = MigrationManager(wrap_in_transaction=True) + manager.add_raw(run) + + await manager.run() + + async def test_disabled(self): + """ + Make sure we can stop the migration being wrapped in a transaction if + we want to. + """ + + async def run(): + db = engine_finder() + assert db + assert db.transaction_exists() is False + + manager = MigrationManager(wrap_in_transaction=False) + manager.add_raw(run) + + await manager.run() From e14b8a567b44028db9959bd742871fbb7fda4ee9 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 20 Aug 2024 18:46:26 +0100 Subject: [PATCH 613/727] bumped version --- CHANGES.rst | 21 +++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 392d84d34..0ff79b564 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,27 @@ Changes ======= +1.17.0 +------ + +Each migration is automatically wrapped in a transaction - this can now be +disabled using the ``wrap_in_transaction`` argument: + +.. code-block:: python + + manager = MigrationManager( + wrap_in_transaction=False, + ... + ) + +This is useful when writing a manual migration, and you want to manage all of +the transaction logic yourself (or want multiple transactions). + +``granian`` is now a supported server in the ASGI templates. Thanks to +@sinisaos for this. + +------------------------------------------------------------------------------- + 1.16.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 497db2c05..ab7457d02 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.16.0" +__VERSION__ = "1.17.0" From 716b7f05e48433f69b60e1cc5539f0713011751e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 11 Sep 2024 00:47:00 +0100 Subject: [PATCH 614/727] add banner text for Piccolo Admin MFA release --- docs/src/conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/src/conf.py b/docs/src/conf.py index bae741f09..fd5f0d2c0 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -69,6 +69,8 @@ globaltoc_maxdepth = 3 html_theme_options = { "source_url": "https://github.com/piccolo-orm/piccolo/", + "banner_text": 'Piccolo Admin now supports Multi-factor Authentication!', # noqa : E501 + "banner_hiding": "permanent", } # -- Options for HTMLHelp output --------------------------------------------- From 47e530520e85d9c01c02a5ce62771ebc62adb00a Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 14 Sep 2024 07:48:31 +0100 Subject: [PATCH 615/727] add docs for conditional where clauses (#1074) --- docs/src/piccolo/query_clauses/where.rst | 43 +++++++++++++++++------- 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/docs/src/piccolo/query_clauses/where.rst b/docs/src/piccolo/query_clauses/where.rst index 472c9d796..edf4aa070 100644 --- a/docs/src/piccolo/query_clauses/where.rst +++ b/docs/src/piccolo/query_clauses/where.rst @@ -53,8 +53,8 @@ You can use the ``<, >, <=, >=`` operators, which work as you expect. ------------------------------------------------------------------------------- -like / ilike -------------- +``like`` / ``ilike`` +-------------------- The percentage operator is required to designate where the match should occur. @@ -80,8 +80,8 @@ The percentage operator is required to designate where the match should occur. ------------------------------------------------------------------------------- -not_like --------- +``not_like`` +------------ Usage is the same as ``like`` excepts it excludes matching rows. @@ -93,8 +93,8 @@ Usage is the same as ``like`` excepts it excludes matching rows. ------------------------------------------------------------------------------- -is_in / not_in --------------- +``is_in`` / ``not_in`` +---------------------- You can get all rows with a value contained in the list: @@ -114,8 +114,8 @@ And all rows with a value not contained in the list: ------------------------------------------------------------------------------- -is_null / is_not_null ---------------------- +``is_null`` / ``is_not_null`` +----------------------------- These queries work, but some linters will complain about doing a comparison with ``None``: @@ -202,8 +202,8 @@ Also, multiple arguments inside ``where`` clause is equivalent to an AND. Band.popularity >= 100, Band.popularity < 1000 ) -Using And / Or directly -~~~~~~~~~~~~~~~~~~~~~~~ +Using ``And`` / ``Or`` directly +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Rather than using the ``|`` and ``&`` characters, you can use the ``And`` and ``Or`` classes, which are what's used under the hood. @@ -221,8 +221,8 @@ Rather than using the ``|`` and ``&`` characters, you can use the ``And`` and ------------------------------------------------------------------------------- -WhereRaw --------- +``WhereRaw`` +------------ In certain situations you may want to have raw SQL in your where clause. @@ -268,3 +268,22 @@ The ``where`` clause has full support for joins. For example: >>> await Band.select(Band.name).where(Band.manager.name == 'Guido') [{'name': 'Pythonistas'}] + +------------------------------------------------------------------------------- + +Conditional ``where`` clauses +----------------------------- + +You can add ``where`` clauses conditionally (e.g. based on user input): + +.. code-block:: python + + async def get_band_names(only_popular_bands: bool) -> list[str]: + query = Band.select(Band.name).output(as_list=True) + + if only_popular_bands: + query = query.where(Band.popularity >= 1000) + + return await query + +.. hint:: This works with all clauses, not just ``where`` clauses. From 2f8e84cad3a337a7738b726cf6bacf8bc11dfff1 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 14 Sep 2024 17:56:36 +0100 Subject: [PATCH 616/727] 1071 Fix `compare_dicts` for numeric values (#1077) * fix `compare_dicts` for numeric values * version pin esmerald for now --- piccolo/apps/asgi/commands/new.py | 2 +- .../apps/migrations/auto/diffable_table.py | 13 ++++++++++--- .../auto/integration/test_migrations.py | 19 +++++++++++++++++++ .../migrations/auto/test_diffable_table.py | 13 +++++++++++++ 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index 746c5db9a..5c83eca0c 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -15,7 +15,7 @@ "fastapi": ["fastapi>=0.112.1"], "blacksheep": ["blacksheep"], "litestar": ["litestar"], - "esmerald": ["esmerald"], + "esmerald": ["esmerald==3.3.0"], "lilya": ["lilya"], } ROUTERS = list(ROUTER_DEPENDENCIES.keys()) diff --git a/piccolo/apps/migrations/auto/diffable_table.py b/piccolo/apps/migrations/auto/diffable_table.py index aa609f041..522f4f001 100644 --- a/piccolo/apps/migrations/auto/diffable_table.py +++ b/piccolo/apps/migrations/auto/diffable_table.py @@ -40,10 +40,17 @@ def compare_dicts( for key, value in dict_1.items(): dict_2_value = dict_2.get(key, ...) + if ( - dict_2_value is not ... - and dict_2_value != value - or dict_2_value is ... + # If the value is `...` then it means no value was found. + (dict_2_value is ...) + # We have to compare the types, because if we just use equality + # then 1.0 == 1 is True. + # See this issue: + # https://github.com/piccolo-orm/piccolo/issues/1071 + or (type(value) is not type(dict_2_value)) + # Finally compare the actual values. + or (dict_2_value != value) ): output[key] = value diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index 84b194ea8..0ba2901fd 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -888,6 +888,25 @@ def test_column_type_conversion_float_decimal(self): ] ) + def test_column_type_conversion_integer_float(self): + """ + Make sure conversion between ``Integer`` and ``Real`` works - related + to this bug: + + https://github.com/piccolo-orm/piccolo/issues/1071 + + """ + self._test_migrations( + table_snapshots=[ + [self.table(column)] + for column in [ + Real(default=1.0), + Integer(default=1), + Real(default=1.0), + ] + ] + ) + def test_column_type_conversion_json(self): self._test_migrations( table_snapshots=[ diff --git a/tests/apps/migrations/auto/test_diffable_table.py b/tests/apps/migrations/auto/test_diffable_table.py index 7c0cda9f3..cacd6d612 100644 --- a/tests/apps/migrations/auto/test_diffable_table.py +++ b/tests/apps/migrations/auto/test_diffable_table.py @@ -73,6 +73,19 @@ def test_enum_values(self): response = compare_dicts(dict_1, dict_2) self.assertEqual(response, {"a": OnDelete.set_default}) + def test_numeric_values(self): + """ + Make sure that if we have two numbers which are equal, but different + types, then they are identified as being different. + + https://github.com/piccolo-orm/piccolo/issues/1071 + + """ + dict_1 = {"a": 1} + dict_2 = {"a": 1.0} + response = compare_dicts(dict_1, dict_2) + self.assertEqual(response, {"a": 1}) + class TestDiffableTable(TestCase): def test_subtract(self): From 89f1adbe8147607ef0546534f668c8f58800ba4d Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 14 Sep 2024 21:39:45 +0100 Subject: [PATCH 617/727] bumped version --- CHANGES.rst | 8 ++++++++ piccolo/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 0ff79b564..0b57e42d8 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,14 @@ Changes ======= +1.17.1 +------ + +Fixed a bug with migrations, where altering a column type from ``Integer`` to +``Float`` could fail. Thanks to @kurtportelli for reporting this issue. + +------------------------------------------------------------------------------- + 1.17.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index ab7457d02..922883d70 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.17.0" +__VERSION__ = "1.17.1" From df86c2e189849356ac766f8c89c366b97a468aca Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 16 Sep 2024 16:33:30 +0100 Subject: [PATCH 618/727] 1079 Add `batch` to `raw` queries (#1080) * add `batch` to `raw` queries * link to `batch` clause from `Raw` docs page --- docs/src/piccolo/query_clauses/batch.rst | 1 + docs/src/piccolo/query_types/raw.rst | 15 +++++++-- piccolo/query/methods/raw.py | 13 ++++++++ tests/table/test_batch.py | 41 ++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 2 deletions(-) diff --git a/docs/src/piccolo/query_clauses/batch.rst b/docs/src/piccolo/query_clauses/batch.rst index 139a360b9..0447ebca8 100644 --- a/docs/src/piccolo/query_clauses/batch.rst +++ b/docs/src/piccolo/query_clauses/batch.rst @@ -6,6 +6,7 @@ batch You can use ``batch`` clauses with the following queries: * :ref:`Objects` +* :ref:`Raw` * :ref:`Select` Example diff --git a/docs/src/piccolo/query_types/raw.rst b/docs/src/piccolo/query_types/raw.rst index 11ed86781..4e1e0aaf4 100644 --- a/docs/src/piccolo/query_types/raw.rst +++ b/docs/src/piccolo/query_types/raw.rst @@ -7,7 +7,7 @@ Should you need to, you can execute raw SQL. .. code-block:: python - >>> await Band.raw('select name from band') + >>> await Band.raw('SELECT name FROM band') [{'name': 'Pythonistas'}] It's recommended that you parameterise any values. Use curly braces ``{}`` as @@ -15,7 +15,18 @@ placeholders: .. code-block:: python - >>> await Band.raw('select * from band where name = {}', 'Pythonistas') + >>> await Band.raw('SELECT * FROM band WHERE name = {}', 'Pythonistas') [{'name': 'Pythonistas', 'manager': 1, 'popularity': 1000, 'id': 1}] .. warning:: Be careful to avoid SQL injection attacks. Don't add any user submitted data into your SQL strings, unless it's parameterised. + + +------------------------------------------------------------------------------- + +Query clauses +------------- + +batch +~~~~~ + +See :ref:`batch`. diff --git a/piccolo/query/methods/raw.py b/piccolo/query/methods/raw.py index 260448485..4a4ea215f 100644 --- a/piccolo/query/methods/raw.py +++ b/piccolo/query/methods/raw.py @@ -2,6 +2,7 @@ import typing as t +from piccolo.engine.base import BaseBatch from piccolo.query.base import Query from piccolo.querystring import QueryString @@ -21,6 +22,18 @@ def __init__( super().__init__(table, **kwargs) self.querystring = querystring + async def batch( + self, + batch_size: t.Optional[int] = None, + node: t.Optional[str] = None, + **kwargs, + ) -> BaseBatch: + if batch_size: + kwargs.update(batch_size=batch_size) + if node: + kwargs.update(node=node) + return await self.table._meta.db.batch(self, **kwargs) + @property def default_querystrings(self) -> t.Sequence[QueryString]: return [self.querystring] diff --git a/tests/table/test_batch.py b/tests/table/test_batch.py index 2de9e0c7d..762701f53 100644 --- a/tests/table/test_batch.py +++ b/tests/table/test_batch.py @@ -95,6 +95,47 @@ def test_batch(self): self.assertEqual(iterations, _iterations) +class TestBatchRaw(DBTestCase): + def _check_results(self, batch): + """ + Make sure the data is returned in the correct format. + """ + self.assertEqual(type(batch), list) + if len(batch) > 0: + row = batch[0] + self.assertIsInstance(row, Manager) + + async def run_batch(self, batch_size): + row_count = 0 + iterations = 0 + + async with await Manager.raw("SELECT * FROM manager").batch( + batch_size=batch_size + ) as batch: + async for _batch in batch: + self._check_results(_batch) + _row_count = len(_batch) + row_count += _row_count + iterations += 1 + + return row_count, iterations + + async def test_batch(self): + row_count = 1000 + self.insert_many_rows(row_count) + + batch_size = 10 + + _row_count, iterations = asyncio.run( + self.run_batch(batch_size=batch_size), debug=True + ) + + _iterations = math.ceil(row_count / batch_size) + + self.assertEqual(_row_count, row_count) + self.assertEqual(iterations, _iterations) + + @engines_only("postgres", "cockroach") class TestBatchNodeArg(TestCase): def test_batch_extra_node(self): From e36a9ed9f361a1b8c8702f13704d748f5ea5a487 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 20 Sep 2024 20:10:00 -0400 Subject: [PATCH 619/727] 1073 Add `update_self` method (#1081) * prototype for `UpdateSelf` * fleshed out implementation * add tests * update docstring - use `Band` table as an example * improve docs * finish docs --- docs/src/piccolo/query_types/objects.rst | 49 +++++++++++++++++---- piccolo/query/methods/objects.py | 56 ++++++++++++++++++++++++ piccolo/table.py | 39 ++++++++++++++++- tests/table/test_update_self.py | 27 ++++++++++++ 4 files changed, 162 insertions(+), 9 deletions(-) create mode 100644 tests/table/test_update_self.py diff --git a/docs/src/piccolo/query_types/objects.rst b/docs/src/piccolo/query_types/objects.rst index 968aefcab..c8d65d923 100644 --- a/docs/src/piccolo/query_types/objects.rst +++ b/docs/src/piccolo/query_types/objects.rst @@ -78,6 +78,9 @@ We also have this shortcut which combines the above into a single line: Updating objects ---------------- +``save`` +~~~~~~~~ + Objects have a :meth:`save ` method, which is convenient for updating values: @@ -95,6 +98,36 @@ convenient for updating values: # Or specify specific columns to save: await band.save([Band.popularity]) +``update_self`` +~~~~~~~~~~~~~~~ + +The :meth:`save ` method is fine in the majority of +cases, but there are some situations where the :meth:`update_self ` +method is preferable. + +For example, if we want to increment the ``popularity`` value, we can do this: + +.. code-block:: python + + await band.update_self({ + Band.popularity: Band.popularity + 1 + }) + +Which does the following: + +* Increments the popularity in the database +* Assigns the new value to the object + +This is safer than: + +.. code-block:: python + + band.popularity += 1 + await band.save() + +Because ``update_self`` increments the current ``popularity`` value in the +database, not the one on the object, which might be out of date. + ------------------------------------------------------------------------------- Deleting objects @@ -115,8 +148,8 @@ Similarly, we can delete objects, using the ``remove`` method. Fetching related objects ------------------------ -get_related -~~~~~~~~~~~ +``get_related`` +~~~~~~~~~~~~~~~ If you have an object from a table with a :class:`ForeignKey ` column, and you want to fetch the related row as an object, you can do so @@ -195,8 +228,8 @@ prefer. ------------------------------------------------------------------------------- -get_or_create -------------- +``get_or_create`` +----------------- With ``get_or_create`` you can get an existing record matching the criteria, or create a new one with the ``defaults`` arguments: @@ -239,8 +272,8 @@ Complex where clauses are supported, but only within reason. For example: ------------------------------------------------------------------------------- -to_dict -------- +``to_dict`` +----------- If you need to convert an object into a dictionary, you can do so using the ``to_dict`` method. @@ -264,8 +297,8 @@ the columns: ------------------------------------------------------------------------------- -refresh -------- +``refresh`` +----------- If you have an object which has gotten stale, and want to refresh it, so it has the latest data from the database, you can use the diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index 7f2b5aaed..878116099 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -27,6 +27,7 @@ if t.TYPE_CHECKING: # pragma: no cover from piccolo.columns import Column + from piccolo.table import Table ############################################################################### @@ -173,6 +174,61 @@ def run_sync(self, *args, **kwargs) -> TableInstance: return run_sync(self.run(*args, **kwargs)) +class UpdateSelf: + + def __init__( + self, + row: Table, + values: t.Dict[t.Union[Column, str], t.Any], + ): + self.row = row + self.values = values + + async def run( + self, + node: t.Optional[str] = None, + in_pool: bool = True, + ) -> None: + if not self.row._exists_in_db: + raise ValueError("This row doesn't exist in the database.") + + TableClass = self.row.__class__ + + primary_key = TableClass._meta.primary_key + primary_key_value = getattr(self.row, primary_key._meta.name) + + if primary_key_value is None: + raise ValueError("The primary key is None") + + columns = [ + TableClass._meta.get_column_by_name(i) if isinstance(i, str) else i + for i in self.values.keys() + ] + + response = ( + await TableClass.update(self.values) + .where(primary_key == primary_key_value) + .returning(*columns) + .run( + node=node, + in_pool=in_pool, + ) + ) + + for key, value in response[0].items(): + setattr(self.row, key, value) + + def __await__(self) -> t.Generator[None, None, None]: + """ + If the user doesn't explicity call .run(), proxy to it as a + convenience. + """ + return self.run().__await__() + + def run_sync(self, *args, **kwargs) -> None: + return run_sync(self.run(*args, **kwargs)) + + ############################################################################### diff --git a/piccolo/table.py b/piccolo/table.py index af0fed2fe..b50855f95 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -46,7 +46,7 @@ ) from piccolo.query.methods.create_index import CreateIndex from piccolo.query.methods.indexes import Indexes -from piccolo.query.methods.objects import First +from piccolo.query.methods.objects import First, UpdateSelf from piccolo.query.methods.refresh import Refresh from piccolo.querystring import QueryString from piccolo.utils import _camel_to_snake @@ -525,6 +525,43 @@ def save( == getattr(self, self._meta.primary_key._meta.name) ) + def update_self( + self, values: t.Dict[t.Union[Column, str], t.Any] + ) -> UpdateSelf: + """ + This allows the user to update a single object - useful when the values + are derived from the database in some way. + + For example, if we have the following table:: + + class Band(Table): + name = Varchar() + popularity = Integer() + + And we fetch an object:: + + >>> band = await Band.objects().get(name="Pythonistas") + + We could use the typical syntax for updating the object:: + + >>> band.popularity += 1 + >>> await band.save() + + The problem with this, is what if another object has already + incremented ``popularity``? It would overide the value. + + Instead we can do this: + + >>> await band.update_self({ + ... Band.popularity: Band.popularity + 1 + ... }) + + This updates ``popularity`` in the database, and also sets the new + value for ``popularity`` on the object. + + """ + return UpdateSelf(row=self, values=values) + def remove(self) -> Delete: """ A proxy to a delete query. diff --git a/tests/table/test_update_self.py b/tests/table/test_update_self.py new file mode 100644 index 000000000..c06afe708 --- /dev/null +++ b/tests/table/test_update_self.py @@ -0,0 +1,27 @@ +from piccolo.testing.test_case import AsyncTableTest +from tests.example_apps.music.tables import Band, Manager + + +class TestUpdateSelf(AsyncTableTest): + + tables = [Band, Manager] + + async def test_update_self(self): + band = Band({Band.name: "Pythonistas", Band.popularity: 1000}) + + # Make sure we get a ValueError if it's not in the database yet. + with self.assertRaises(ValueError): + await band.update_self({Band.popularity: Band.popularity + 1}) + + # Save it, so it's in the database + await band.save() + + # Make sure we can successfully update the object + await band.update_self({Band.popularity: Band.popularity + 1}) + + # Make sure the value was updated on the object + assert band.popularity == 1001 + + # Make sure the value was updated in the database + await band.refresh() + assert band.popularity == 1001 From 0f4716fde2ad6c23e3bcc45fa84c2d8f6dd7b299 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 21 Sep 2024 22:23:16 +0100 Subject: [PATCH 620/727] bumped version --- CHANGES.rst | 50 +++++++++++++++++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 0b57e42d8..ec4d888df 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,56 @@ Changes ======= +1.18.0 +------ + +``update_self`` +~~~~~~~~~~~~~~~ + +Added the ``update_self`` method, which is an alternative to the ``save`` +method. Here's an example where it's useful: + +.. code-block:: python + + # If we have a band object: + >>> band = Band.objects().get(name="Pythonistas") + >>> band.popularity + 1000 + + # We can increment the popularity, based on the current value in the + # database: + >>> await band.update_self({ + ... Band.popularity: Band.popularity + 1 + ... }) + + # The new value is set on the object: + >>> band.popularity + 1001 + + # It's safer than using the `save` method, because the popularity value on + # the object might be out of date with what's in the database: + band.popularity += 1 + await band.save() + +Thanks to @trondhindenes for suggesting this feature. + +Batch raw queries +~~~~~~~~~~~~~~~~~ + +The ``batch`` method can now be used with ``raw`` queries. For example: + +.. code-block:: python + + async with await MyTable.raw("SELECT * FROM my_table").batch() as batch: + async for _batch in batch: + print(_batch) + +This is useful when you expect a raw query to return a lot of data. + +Thanks to @devsarvesh92 for suggesting this feature. + +------------------------------------------------------------------------------- + 1.17.1 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 922883d70..901d36e4f 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.17.1" +__VERSION__ = "1.18.0" From ab63b32a12f64fc31367f4871eb74c974656ee89 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 21 Sep 2024 18:11:34 -0400 Subject: [PATCH 621/727] improve docs for fetching objects (#1084) --- docs/src/piccolo/query_types/objects.rst | 27 ++++++++++++++++-------- docs/src/piccolo/query_types/select.rst | 5 ++++- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/docs/src/piccolo/query_types/objects.rst b/docs/src/piccolo/query_types/objects.rst index c8d65d923..be7147586 100644 --- a/docs/src/piccolo/query_types/objects.rst +++ b/docs/src/piccolo/query_types/objects.rst @@ -16,36 +16,45 @@ examples. Fetching objects ---------------- -To get all objects: +To get all rows: .. code-block:: python >>> await Band.objects() - [, ] + [, , ] -To get certain rows: +To limit the number of rows returned, use the :ref:`order_by` and :ref:`limit` +clauses: + +.. code-block:: python + + >>> await Band.objects().order_by(Band.popularity, ascending=False).limit(2) + [, ] + +To filter the rows we use the :ref:`where` clause: .. code-block:: python >>> await Band.objects().where(Band.name == 'Pythonistas') [] -To get a single row (or ``None`` if it doesn't exist): +To get a single row (or ``None`` if it doesn't exist) use the :ref:`first` +clause: .. code-block:: python - >>> await Band.objects().get(Band.name == 'Pythonistas') + >>> await Band.objects().where(Band.name == 'Pythonistas').first() -To get the first row: +Alternatively, you can use this abbreviated syntax: .. code-block:: python - >>> await Band.objects().first() + >>> await Band.objects().get(Band.name == 'Pythonistas') -You'll notice that the API is similar to :ref:`Select` - except it returns all -columns. +You'll notice that the API is similar to :ref:`Select` (expect with ``select`` +you can specify which columns are returned). ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/query_types/select.rst b/docs/src/piccolo/query_types/select.rst index 092291e4c..ab1d0a629 100644 --- a/docs/src/piccolo/query_types/select.rst +++ b/docs/src/piccolo/query_types/select.rst @@ -5,7 +5,10 @@ Select .. hint:: Follow along by installing Piccolo and running ``piccolo playground run`` - see :ref:`Playground`. -To get all rows: +Columns +------- + +To get all columns: .. code-block:: python From b17e335b8bbbde816b355279c761b5505adbade6 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 23 Sep 2024 16:02:57 +0100 Subject: [PATCH 622/727] fix typo in changelog - was missing `await` --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index ec4d888df..d9f1deabd 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -13,7 +13,7 @@ method. Here's an example where it's useful: .. code-block:: python # If we have a band object: - >>> band = Band.objects().get(name="Pythonistas") + >>> band = await Band.objects().get(name="Pythonistas") >>> band.popularity 1000 From d3d0d3066ea83818935189fcf5975e442d1dce8a Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 24 Sep 2024 00:44:48 +0100 Subject: [PATCH 623/727] 1065 Select for update (#1085) * Add for_update clause * Skip testing for_update for SQLite * Fix typos in exception messages * Rename method to 'lock_for' * Format 'test_select' * wip doc changes * rename `lock_for` to `lock_rows` --------- Co-authored-by: Denis Kopitsa --- docs/src/piccolo/query_clauses/index.rst | 1 + docs/src/piccolo/query_clauses/lock_rows.rst | 93 ++++++++++++++++++++ docs/src/piccolo/query_types/objects.rst | 5 ++ docs/src/piccolo/query_types/select.rst | 6 ++ piccolo/query/methods/objects.py | 25 ++++++ piccolo/query/methods/select.py | 34 +++++++ piccolo/query/mixins.py | 88 ++++++++++++++++++ tests/table/test_select.py | 34 +++++++ 8 files changed, 286 insertions(+) create mode 100644 docs/src/piccolo/query_clauses/lock_rows.rst diff --git a/docs/src/piccolo/query_clauses/index.rst b/docs/src/piccolo/query_clauses/index.rst index f9167ff63..feac07b0f 100644 --- a/docs/src/piccolo/query_clauses/index.rst +++ b/docs/src/piccolo/query_clauses/index.rst @@ -25,6 +25,7 @@ by modifying the return values. ./distinct ./freeze ./group_by + ./lock_rows ./offset ./on_conflict ./output diff --git a/docs/src/piccolo/query_clauses/lock_rows.rst b/docs/src/piccolo/query_clauses/lock_rows.rst new file mode 100644 index 000000000..54d435326 --- /dev/null +++ b/docs/src/piccolo/query_clauses/lock_rows.rst @@ -0,0 +1,93 @@ +.. _lock_rows: + +lock_rows +========= + +You can use the ``lock_rows`` clause with the following queries: + +* :ref:`Objects` +* :ref:`Select` + +It returns a query that locks rows until the end of the transaction, generating a ``SELECT ... FOR UPDATE`` SQL statement or similar with other lock strengths. + +.. note:: Postgres and CockroachDB only. + +------------------------------------------------------------------------------- + +Basic Usage +----------- + +Basic usage without parameters: + +.. code-block:: python + + await Band.select(Band.name == 'Pythonistas').lock_rows() + +Equivalent to: + +.. code-block:: sql + + SELECT ... FOR UPDATE + + +lock_strength +------------- + +The parameter ``lock_strength`` controls the strength of the row lock when performing an operation in PostgreSQL. +The value can be a predefined constant from the ``LockStrength`` enum or one of the following strings (case-insensitive): + +* ``UPDATE`` (default): Acquires an exclusive lock on the selected rows, preventing other transactions from modifying or locking them until the current transaction is complete. +* ``NO KEY UPDATE`` (Postgres only): Similar to ``UPDATE``, but allows other transactions to insert or delete rows that do not affect the primary key or unique constraints. +* ``KEY SHARE`` (Postgres only): Permits other transactions to acquire key-share or share locks, allowing non-key modifications while preventing updates or deletes. +* ``SHARE``: Acquires a shared lock, allowing other transactions to read the rows but not modify or lock them. + +You can specify a different lock strength: + +.. code-block:: python + + await Band.select(Band.name == 'Pythonistas').lock_rows('SHARE') + +Which is equivalent to: + +.. code-block:: sql + + SELECT ... FOR SHARE + + +nowait +------ + +If another transaction has already acquired a lock on one or more selected rows, an exception will be raised instead of +waiting for the other transaction to release the lock. + +.. code-block:: python + + await Band.select(Band.name == 'Pythonistas').lock_rows('UPDATE', nowait=True) + + +skip_locked +----------- + +Ignore locked rows. + +.. code-block:: python + + await Band.select(Band.name == 'Pythonistas').lock_rows('UPDATE', skip_locked=True) + + +of +-- + +By default, if there are many tables in a query (e.g. when joining), all tables will be locked. +Using ``of``, you can specify which tables should be locked. + +.. code-block:: python + + await Band.select().where(Band.manager.name == 'Guido').lock_rows('UPDATE', of=(Band, )) + + +Learn more +---------- + +* `Postgres docs `_ +* `CockroachDB docs `_ diff --git a/docs/src/piccolo/query_types/objects.rst b/docs/src/piccolo/query_types/objects.rst index be7147586..892569a02 100644 --- a/docs/src/piccolo/query_types/objects.rst +++ b/docs/src/piccolo/query_types/objects.rst @@ -377,6 +377,11 @@ limit See :ref:`limit`. +lock_rows +~~~~~~~~ + +See :ref:`lock_rows`. + offset ~~~~~~ diff --git a/docs/src/piccolo/query_types/select.rst b/docs/src/piccolo/query_types/select.rst index ab1d0a629..4fff7eb6e 100644 --- a/docs/src/piccolo/query_types/select.rst +++ b/docs/src/piccolo/query_types/select.rst @@ -379,6 +379,12 @@ limit See :ref:`limit`. + +lock_rows +~~~~~~~~ + +See :ref:`lock_rows`. + offset ~~~~~~ diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index 878116099..5e4dda50b 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -13,6 +13,8 @@ CallbackDelegate, CallbackType, LimitDelegate, + LockRowsDelegate, + LockStrength, OffsetDelegate, OrderByDelegate, OrderByRaw, @@ -250,6 +252,7 @@ class Objects( "callback_delegate", "prefetch_delegate", "where_delegate", + "lock_rows_delegate", ) def __init__( @@ -269,6 +272,7 @@ def __init__( self.prefetch_delegate = PrefetchDelegate() self.prefetch(*prefetch) self.where_delegate = WhereDelegate() + self.lock_rows_delegate = LockRowsDelegate() def output(self: Self, load_json: bool = False) -> Self: self.output_delegate.output( @@ -328,6 +332,26 @@ def first(self) -> First[TableInstance]: self.limit_delegate.limit(1) return First[TableInstance](query=self) + def lock_rows( + self: Self, + lock_strength: t.Union[ + LockStrength, + t.Literal[ + "UPDATE", + "NO KEY UPDATE", + "KEY SHARE", + "SHARE", + ], + ] = LockStrength.update, + nowait: bool = False, + skip_locked: bool = False, + of: t.Tuple[type[Table], ...] = (), + ) -> Self: + self.lock_rows_delegate.lock_rows( + lock_strength, nowait, skip_locked, of + ) + return self + def get(self, where: Combinable) -> Get[TableInstance]: self.where_delegate.where(where) self.limit_delegate.limit(1) @@ -378,6 +402,7 @@ def default_querystrings(self) -> t.Sequence[QueryString]: "offset_delegate", "output_delegate", "order_by_delegate", + "lock_rows_delegate", ): setattr(select, attr, getattr(self, attr)) diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 0c590918b..5d7856c5a 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -19,6 +19,8 @@ DistinctDelegate, GroupByDelegate, LimitDelegate, + LockRowsDelegate, + LockStrength, OffsetDelegate, OrderByDelegate, OrderByRaw, @@ -150,6 +152,7 @@ class Select(Query[TableInstance, t.List[t.Dict[str, t.Any]]]): "output_delegate", "callback_delegate", "where_delegate", + "lock_rows_delegate", ) def __init__( @@ -174,6 +177,7 @@ def __init__( self.output_delegate = OutputDelegate() self.callback_delegate = CallbackDelegate() self.where_delegate = WhereDelegate() + self.lock_rows_delegate = LockRowsDelegate() self.columns(*columns_list) @@ -219,6 +223,26 @@ def offset(self: Self, number: int) -> Self: self.offset_delegate.offset(number) return self + def lock_rows( + self: Self, + lock_strength: t.Union[ + LockStrength, + t.Literal[ + "UPDATE", + "NO KEY UPDATE", + "KEY SHARE", + "SHARE", + ], + ] = LockStrength.update, + nowait: bool = False, + skip_locked: bool = False, + of: t.Tuple[type[Table], ...] = (), + ) -> Self: + self.lock_rows_delegate.lock_rows( + lock_strength, nowait, skip_locked, of + ) + return self + async def _splice_m2m_rows( self, response: t.List[t.Dict[str, t.Any]], @@ -618,6 +642,16 @@ def default_querystrings(self) -> t.Sequence[QueryString]: query += "{}" args.append(self.offset_delegate._offset.querystring) + if self.lock_rows_delegate._lock_rows: + if engine_type == "sqlite": + raise NotImplementedError( + "SQLite doesn't support row locking e.g. SELECT ... FOR " + "UPDATE" + ) + + query += "{}" + args.append(self.lock_rows_delegate._lock_rows.querystring) + querystring = QueryString(query, *args) return [querystring] diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index d9d5f84ca..b1b726cf3 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -784,3 +784,91 @@ def on_conflict( target=target, action=action_, values=values, where=where ) ) + + +class LockStrength(str, Enum): + """ + Specify lock strength + + https://www.postgresql.org/docs/current/sql-select.html#SQL-FOR-UPDATE-SHARE + """ + + update = "UPDATE" + no_key_update = "NO KEY UPDATE" + share = "SHARE" + key_share = "KEY SHARE" + + +@dataclass +class LockRows: + __slots__ = ("lock_strength", "nowait", "skip_locked", "of") + + lock_strength: LockStrength + nowait: bool + skip_locked: bool + of: t.Tuple[t.Type[Table], ...] + + def __post_init__(self): + if not isinstance(self.lock_strength, LockStrength): + raise TypeError("lock_strength must be a LockStrength") + if not isinstance(self.nowait, bool): + raise TypeError("nowait must be a bool") + if not isinstance(self.skip_locked, bool): + raise TypeError("skip_locked must be a bool") + if not isinstance(self.of, tuple) or not all( + hasattr(x, "_meta") for x in self.of + ): + raise TypeError("of must be a tuple of Table") + if self.nowait and self.skip_locked: + raise TypeError( + "The nowait option cannot be used with skip_locked" + ) + + @property + def querystring(self) -> QueryString: + sql = f" FOR {self.lock_strength.value}" + if self.of: + tables = ", ".join( + i._meta.get_formatted_tablename() for i in self.of + ) + sql += " OF " + tables + if self.nowait: + sql += " NOWAIT" + if self.skip_locked: + sql += " SKIP LOCKED" + + return QueryString(sql) + + def __str__(self) -> str: + return self.querystring.__str__() + + +@dataclass +class LockRowsDelegate: + + _lock_rows: t.Optional[LockRows] = None + + def lock_rows( + self, + lock_strength: t.Union[ + LockStrength, + t.Literal[ + "UPDATE", + "NO KEY UPDATE", + "KEY SHARE", + "SHARE", + ], + ] = LockStrength.update, + nowait=False, + skip_locked=False, + of: t.Tuple[type[Table], ...] = (), + ): + lock_strength_: LockStrength + if isinstance(lock_strength, LockStrength): + lock_strength_ = lock_strength + elif isinstance(lock_strength, str): + lock_strength_ = LockStrength(lock_strength.upper()) + else: + raise ValueError("Unrecognised `lock_strength` value.") + + self._lock_rows = LockRows(lock_strength_, nowait, skip_locked, of) diff --git a/tests/table/test_select.py b/tests/table/test_select.py index ebf2c3ff8..74b876d5f 100644 --- a/tests/table/test_select.py +++ b/tests/table/test_select.py @@ -1028,6 +1028,40 @@ def test_select_raw(self): response, [{"name": "Pythonistas", "popularity_log": 3.0}] ) + @pytest.mark.skipif( + is_running_sqlite(), + reason="SQLite doesn't support SELECT ... FOR UPDATE.", + ) + def test_lock_rows(self): + """ + Make sure the for_update clause works. + """ + self.insert_rows() + + query = Band.select() + self.assertNotIn("FOR UPDATE", query.__str__()) + + query = query.lock_rows() + self.assertTrue(query.__str__().endswith("FOR UPDATE")) + + query = query.lock_rows(lock_strength="KEY SHARE") + self.assertTrue(query.__str__().endswith("FOR KEY SHARE")) + + query = query.lock_rows(skip_locked=True) + self.assertTrue(query.__str__().endswith("FOR UPDATE SKIP LOCKED")) + + query = query.lock_rows(nowait=True) + self.assertTrue(query.__str__().endswith("FOR UPDATE NOWAIT")) + + query = query.lock_rows(of=(Band,)) + self.assertTrue(query.__str__().endswith('FOR UPDATE OF "band"')) + + with self.assertRaises(TypeError): + query = query.lock_rows(skip_locked=True, nowait=True) + + response = query.run_sync() + assert response is not None + class TestSelectSecret(TestCase): def setUp(self): From 9467248d4d9347b3f62c0c5cf72fe67272bc1436 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 24 Sep 2024 01:22:26 +0100 Subject: [PATCH 624/727] add a full example for `lock_rows` --- docs/src/piccolo/query_clauses/lock_rows.rst | 35 ++++++++++++++++++++ docs/src/piccolo/query_types/objects.rst | 2 +- docs/src/piccolo/query_types/select.rst | 2 +- 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/docs/src/piccolo/query_clauses/lock_rows.rst b/docs/src/piccolo/query_clauses/lock_rows.rst index 54d435326..0a791f6e8 100644 --- a/docs/src/piccolo/query_clauses/lock_rows.rst +++ b/docs/src/piccolo/query_clauses/lock_rows.rst @@ -85,6 +85,41 @@ Using ``of``, you can specify which tables should be locked. await Band.select().where(Band.manager.name == 'Guido').lock_rows('UPDATE', of=(Band, )) +------------------------------------------------------------------------------- + +Full example +------------ + +If we have this table: + +.. code-block:: python + + class Concert(Table): + name = Varchar() + tickets_available = Integer() + +And we want to make sure that ``tickets_available`` never goes below 0, we can +do the following: + +.. code-block:: python + + async def book_tickets(ticket_count: int): + async with Concert._meta.db.transaction(): + concert = await Concert.objects().where( + Concert.name == "Awesome Concert" + ).first().lock_rows() + + if concert.tickets_available >= ticket_count: + await concert.update_self({ + Concert.tickets_available: Concert.tickets_available - ticket_count + }) + else: + raise ValueError("Not enough tickets are available!") + +This means that when multiple transactions are running at the same time, it +isn't possible to book more tickets than are available. + +------------------------------------------------------------------------------- Learn more ---------- diff --git a/docs/src/piccolo/query_types/objects.rst b/docs/src/piccolo/query_types/objects.rst index 892569a02..e4a2c126e 100644 --- a/docs/src/piccolo/query_types/objects.rst +++ b/docs/src/piccolo/query_types/objects.rst @@ -378,7 +378,7 @@ limit See :ref:`limit`. lock_rows -~~~~~~~~ +~~~~~~~~~ See :ref:`lock_rows`. diff --git a/docs/src/piccolo/query_types/select.rst b/docs/src/piccolo/query_types/select.rst index 4fff7eb6e..2025f35f4 100644 --- a/docs/src/piccolo/query_types/select.rst +++ b/docs/src/piccolo/query_types/select.rst @@ -381,7 +381,7 @@ See :ref:`limit`. lock_rows -~~~~~~~~ +~~~~~~~~~ See :ref:`lock_rows`. From c93a370dc4f8a827612dc5fc00a46bf06b220caa Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 24 Sep 2024 01:24:28 +0100 Subject: [PATCH 625/727] bumped version --- CHANGES.rst | 36 ++++++++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index d9f1deabd..818ed9d21 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,42 @@ Changes ======= +1.19.0 +------ + +Added support for row locking (i.e. ``SELECT ... FOR UPDATE``). + +For example, if we have this table: + +.. code-block:: python + + class Concert(Table): + name = Varchar() + tickets_available = Integer() + +And we want to make sure that ``tickets_available`` never goes below 0, we can +do the following: + +.. code-block:: python + + async def book_tickets(ticket_count: int): + async with Concert._meta.db.transaction(): + concert = await Concert.objects().where( + Concert.name == "Awesome Concert" + ).first().lock_rows() + + if concert.tickets_available >= ticket_count: + await concert.update_self({ + Concert.tickets_available: Concert.tickets_available - ticket_count + }) + else: + raise ValueError("Not enough tickets are available!") + +This means that when multiple transactions are running at the same time, it +isn't possible to book more tickets than are available. + +------------------------------------------------------------------------------- + 1.18.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 901d36e4f..d6e05896d 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.18.0" +__VERSION__ = "1.19.0" From f044c21172a3defdb4b97f768508b35f06660da7 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 24 Sep 2024 01:25:49 +0100 Subject: [PATCH 626/727] Update CHANGES.rst --- CHANGES.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 818ed9d21..4d589b67b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -35,6 +35,8 @@ do the following: This means that when multiple transactions are running at the same time, it isn't possible to book more tickets than are available. +Thanks to @dkopitsa for adding this feature. + ------------------------------------------------------------------------------- 1.18.0 From d7ead8261444ff0020a9202c0576bf5f7ef01c63 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 27 Sep 2024 13:19:28 +0100 Subject: [PATCH 627/727] add video link to `lock_rows` docs --- docs/src/piccolo/query_clauses/lock_rows.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/src/piccolo/query_clauses/lock_rows.rst b/docs/src/piccolo/query_clauses/lock_rows.rst index 0a791f6e8..17ac134d3 100644 --- a/docs/src/piccolo/query_clauses/lock_rows.rst +++ b/docs/src/piccolo/query_clauses/lock_rows.rst @@ -119,6 +119,10 @@ do the following: This means that when multiple transactions are running at the same time, it isn't possible to book more tickets than are available. +.. note:: + + There is a `video tutorial on YouTube `__. + ------------------------------------------------------------------------------- Learn more From 6aa998cb43b4c00a27f0d4b59387d84f1f0f56ac Mon Sep 17 00:00:00 2001 From: sinisaos Date: Mon, 30 Sep 2024 13:41:05 +0200 Subject: [PATCH 628/727] Update asgi templates (#1088) * update Litestar template * format other asgi templates --- .../templates/app/_blacksheep_app.py.jinja | 22 ++++------ .../templates/app/_esmerald_app.py.jinja | 38 ++++++---------- .../templates/app/_lilya_app.py.jinja | 13 +++--- .../templates/app/_litestar_app.py.jinja | 43 ++++++++++++++----- 4 files changed, 61 insertions(+), 55 deletions(-) diff --git a/piccolo/apps/asgi/commands/templates/app/_blacksheep_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_blacksheep_app.py.jinja index 7c20f05b4..60d06f33b 100644 --- a/piccolo/apps/asgi/commands/templates/app/_blacksheep_app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/_blacksheep_app.py.jinja @@ -1,20 +1,18 @@ import typing as t -from piccolo_admin.endpoints import create_admin -from piccolo_api.crud.serializers import create_pydantic_model -from piccolo.engine import engine_finder - from blacksheep.server import Application from blacksheep.server.bindings import FromJSON -from blacksheep.server.responses import json from blacksheep.server.openapi.v3 import OpenAPIHandler +from blacksheep.server.responses import json from openapidocs.v3 import Info +from piccolo.engine import engine_finder +from piccolo_admin.endpoints import create_admin +from piccolo_api.crud.serializers import create_pydantic_model from home.endpoints import home from home.piccolo_app import APP_CONFIG from home.tables import Task - app = Application() app.mount( @@ -47,7 +45,7 @@ TaskModelPartial: t.Any = create_pydantic_model( @app.router.get("/tasks/") async def tasks() -> t.List[TaskModelOut]: - return await Task.select().order_by(Task.id) + return await Task.select().order_by(Task._meta.primary_key, ascending=False) @app.router.post("/tasks/") @@ -58,10 +56,8 @@ async def create_task(task_model: FromJSON[TaskModelIn]) -> TaskModelOut: @app.router.put("/tasks/{task_id}/") -async def put_task( - task_id: int, task_model: FromJSON[TaskModelIn] -) -> TaskModelOut: - task = await Task.objects().get(Task.id == task_id) +async def put_task(task_id: int, task_model: FromJSON[TaskModelIn]) -> TaskModelOut: + task = await Task.objects().get(Task._meta.primary_key == task_id) if not task: return json({}, status=404) @@ -77,7 +73,7 @@ async def put_task( async def patch_task( task_id: int, task_model: FromJSON[TaskModelPartial] ) -> TaskModelOut: - task = await Task.objects().get(Task.id == task_id) + task = await Task.objects().get(Task._meta.primary_key == task_id) if not task: return json({}, status=404) @@ -92,7 +88,7 @@ async def patch_task( @app.router.delete("/tasks/{task_id}/") async def delete_task(task_id: int): - task = await Task.objects().get(Task.id == task_id) + task = await Task.objects().get(Task._meta.primary_key == task_id) if not task: return json({}, status=404) diff --git a/piccolo/apps/asgi/commands/templates/app/_esmerald_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_esmerald_app.py.jinja index 60560b980..978300bf6 100644 --- a/piccolo/apps/asgi/commands/templates/app/_esmerald_app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/_esmerald_app.py.jinja @@ -1,23 +1,21 @@ import typing as t - from pathlib import Path -from piccolo.utils.pydantic import create_pydantic_model -from piccolo.engine import engine_finder -from piccolo_admin.endpoints import create_admin - from esmerald import ( + APIView, Esmerald, - Include, Gateway, + Include, JSONResponse, - APIView, + delete, get, post, put, - delete ) from esmerald.config import StaticFilesConfig +from piccolo.engine import engine_finder +from piccolo.utils.pydantic import create_pydantic_model +from piccolo_admin.endpoints import create_admin from home.endpoints import home from home.piccolo_app import APP_CONFIG @@ -40,14 +38,9 @@ async def close_database_connection_pool(): print("Unable to connect to the database") -TaskModelIn: t.Any = create_pydantic_model( - table=Task, - model_name='TaskModelIn' -) +TaskModelIn: t.Any = create_pydantic_model(table=Task, model_name="TaskModelIn") TaskModelOut: t.Any = create_pydantic_model( - table=Task, - include_default_columns=True, - model_name='TaskModelOut' + table=Task, include_default_columns=True, model_name="TaskModelOut" ) @@ -57,19 +50,17 @@ class TaskAPIView(APIView): @get("/") async def tasks(self) -> t.List[TaskModelOut]: - return await Task.select().order_by(Task.id) - + return await Task.select().order_by(Task._meta.primary_key, ascending=False) - @post('/') + @post("/") async def create_task(self, payload: TaskModelIn) -> TaskModelOut: task = Task(**payload.dict()) await task.save() return task.to_dict() - - @put('/{task_id}') + @put("/{task_id}") async def update_task(self, payload: TaskModelIn, task_id: int) -> TaskModelOut: - task = await Task.objects().get(Task.id == task_id) + task = await Task.objects().get(Task._meta.primary_key == task_id) if not task: return JSONResponse({}, status_code=404) @@ -80,10 +71,9 @@ class TaskAPIView(APIView): return task.to_dict() - - @delete('/{task_id}') + @delete("/{task_id}") async def delete_task(self, task_id: int) -> None: - task = await Task.objects().get(Task.id == task_id) + task = await Task.objects().get(Task._meta.primary_key == task_id) if not task: return JSONResponse({}, status_code=404) diff --git a/piccolo/apps/asgi/commands/templates/app/_lilya_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_lilya_app.py.jinja index f65dd7f06..30ce3639e 100644 --- a/piccolo/apps/asgi/commands/templates/app/_lilya_app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/_lilya_app.py.jinja @@ -1,15 +1,14 @@ -from piccolo_admin.endpoints import create_admin -from piccolo_api.crud.endpoints import PiccoloCRUD -from piccolo.engine import engine_finder -from lilya.routing import Path, Include from lilya.apps import Lilya +from lilya.routing import Include, Path from lilya.staticfiles import StaticFiles +from piccolo.engine import engine_finder +from piccolo_admin.endpoints import create_admin +from piccolo_api.crud.endpoints import PiccoloCRUD from home.endpoints import HomeController from home.piccolo_app import APP_CONFIG from home.tables import Task - app = Lilya( routes=[ Path("/", HomeController), @@ -19,10 +18,10 @@ app = Lilya( tables=APP_CONFIG.table_classes, # Required when running under HTTPS: # allowed_hosts=['my_site.com'] - ) + ), ), Include("/static/", StaticFiles(directory="static")), - Include("/tasks/", PiccoloCRUD(table=Task)) + Include("/tasks/", PiccoloCRUD(table=Task)), ], ) diff --git a/piccolo/apps/asgi/commands/templates/app/_litestar_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_litestar_app.py.jinja index 6302972bb..a8df30169 100644 --- a/piccolo/apps/asgi/commands/templates/app/_litestar_app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/_litestar_app.py.jinja @@ -1,8 +1,5 @@ import typing as t -from home.endpoints import home -from home.piccolo_app import APP_CONFIG -from home.tables import Task from litestar import Litestar, asgi, delete, get, patch, post from litestar.contrib.jinja import JinjaTemplateEngine from litestar.exceptions import NotFoundException @@ -10,8 +7,19 @@ from litestar.static_files import StaticFilesConfig from litestar.template import TemplateConfig from litestar.types import Receive, Scope, Send from piccolo.engine import engine_finder -from piccolo.utils.pydantic import create_pydantic_model from piccolo_admin.endpoints import create_admin +from pydantic import BaseModel + +from home.endpoints import home +from home.piccolo_app import APP_CONFIG +from home.tables import Task + +""" +NOTE: `create_pydantic_model` is not compatible with Litestar +version higher than 2.11.0. If you are using Litestar<=2.11.0, +you can use `create_pydantic_model` as in other asgi templates + +from piccolo.utils.pydantic import create_pydantic_model TaskModelIn: t.Any = create_pydantic_model( table=Task, @@ -22,6 +30,18 @@ TaskModelOut: t.Any = create_pydantic_model( include_default_columns=True, model_name="TaskModelOut", ) +""" + + +class TaskModelIn(BaseModel): + name: str + completed: bool = False + + +class TaskModelOut(BaseModel): + id: int + name: str + completed: bool = False # mounting Piccolo Admin @@ -32,31 +52,32 @@ async def admin(scope: "Scope", receive: "Receive", send: "Send") -> None: @get("/tasks", tags=["Task"]) async def tasks() -> t.List[TaskModelOut]: - return await Task.select().order_by(Task.id, ascending=False) + tasks = await Task.select().order_by(Task._meta.primary_key, ascending=False) + return [TaskModelOut(**task) for task in tasks] @post("/tasks", tags=["Task"]) async def create_task(data: TaskModelIn) -> TaskModelOut: - task = Task(**data.dict()) + task = Task(**data.model_dump()) await task.save() - return task.to_dict() + return TaskModelOut(**task.to_dict()) @patch("/tasks/{task_id:int}", tags=["Task"]) async def update_task(task_id: int, data: TaskModelIn) -> TaskModelOut: - task = await Task.objects().get(Task.id == task_id) + task = await Task.objects().get(Task._meta.primary_key == task_id) if not task: raise NotFoundException("Task does not exist") - for key, value in data.dict().items(): + for key, value in data.model_dump().items(): setattr(task, key, value) await task.save() - return task.to_dict() + return TaskModelOut(**task.to_dict()) @delete("/tasks/{task_id:int}", tags=["Task"]) async def delete_task(task_id: int) -> None: - task = await Task.objects().get(Task.id == task_id) + task = await Task.objects().get(Task._meta.primary_key == task_id) if not task: raise NotFoundException("Task does not exist") await task.remove() From 36ccc28913245eba7e59e0e29ccb7eb6c4cc96bf Mon Sep 17 00:00:00 2001 From: sinisaos Date: Mon, 30 Sep 2024 16:58:47 +0200 Subject: [PATCH 629/727] integration test update for Esmerald (#1087) * integration test update for Esmerald * use @dantownsend code for dummy_server --- piccolo/apps/asgi/commands/new.py | 4 +- requirements/test-requirements.txt | 1 + .../apps/asgi/commands/files/dummy_server.py | 38 +++++-------------- 3 files changed, 13 insertions(+), 30 deletions(-) diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index 5c83eca0c..85ceee021 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -12,10 +12,10 @@ SERVERS = ["uvicorn", "Hypercorn", "granian"] ROUTER_DEPENDENCIES = { "starlette": ["starlette"], - "fastapi": ["fastapi>=0.112.1"], + "fastapi": ["fastapi"], "blacksheep": ["blacksheep"], "litestar": ["litestar"], - "esmerald": ["esmerald==3.3.0"], + "esmerald": ["esmerald"], "lilya": ["lilya"], } ROUTERS = list(ROUTER_DEPENDENCIES.keys()) diff --git a/requirements/test-requirements.txt b/requirements/test-requirements.txt index 2f350ed31..006d8f2d9 100644 --- a/requirements/test-requirements.txt +++ b/requirements/test-requirements.txt @@ -1,4 +1,5 @@ coveralls==3.3.1 +httpx==0.27.2 pytest-cov==3.0.0 pytest==6.2.5 python-dateutil==2.8.2 diff --git a/tests/apps/asgi/commands/files/dummy_server.py b/tests/apps/asgi/commands/files/dummy_server.py index 3c3250776..9b83470a3 100644 --- a/tests/apps/asgi/commands/files/dummy_server.py +++ b/tests/apps/asgi/commands/files/dummy_server.py @@ -3,8 +3,10 @@ import sys import typing as t +from httpx import AsyncClient -def dummy_server(app: t.Union[str, t.Callable] = "app:app"): + +async def dummy_server(app: t.Union[str, t.Callable] = "app:app"): """ A very simplistic ASGI server. It's used to run the generated ASGI applications in unit tests. @@ -20,33 +22,13 @@ def dummy_server(app: t.Union[str, t.Callable] = "app:app"): if isinstance(app, str): path, app_name = app.rsplit(":") module = importlib.import_module(path) - app = getattr(module, app_name) - - async def send(message): - if message["type"] == "http.response.start": - if message["status"] == 200: - print("Received 200") - else: - sys.exit("200 not received from app") - - async def receive(): - pass - - scope = { - "scheme": "http", - "type": "http", - "path": "/", - "raw_path": b"/", - "method": "GET", - "query_string": b"", - "headers": [], - } - if callable(app): - asyncio.run(app(scope, receive, send)) - print("Exiting dummy server ...") - else: - sys.exit("The app isn't callable!") + app = t.cast(t.Callable, getattr(module, app_name)) + + async with AsyncClient(app=app) as client: + response = await client.get("http://localhost:8000") + if response.status_code != 200: + sys.exit("The app isn't callable!") if __name__ == "__main__": - dummy_server() + asyncio.run(dummy_server()) From 5173a39e825ea22cb802c70c10c8159b773d5452 Mon Sep 17 00:00:00 2001 From: Mauricio Date: Tue, 1 Oct 2024 01:43:38 -0700 Subject: [PATCH 630/727] 1090 fix(m2m): Return empty list if there are no m2m entries (#1089) * fix(m2m): Return empty list if there are no m2m entries * add a test --------- Co-authored-by: Daniel Townsend --- piccolo/columns/m2m.py | 8 ++++++-- tests/columns/m2m/base.py | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/piccolo/columns/m2m.py b/piccolo/columns/m2m.py index 29bafe9b5..0a75f9eb7 100644 --- a/piccolo/columns/m2m.py +++ b/piccolo/columns/m2m.py @@ -384,8 +384,12 @@ async def run(self): .output(as_list=True) ) - results = await secondary_table.objects().where( - secondary_table._meta.primary_key.is_in(ids) + results = ( + await secondary_table.objects().where( + secondary_table._meta.primary_key.is_in(ids) + ) + if len(ids) > 0 + else [] ) return results diff --git a/tests/columns/m2m/base.py b/tests/columns/m2m/base.py index 6386ffcaf..5d1c16436 100644 --- a/tests/columns/m2m/base.py +++ b/tests/columns/m2m/base.py @@ -434,6 +434,22 @@ def test_get_m2m(self): self.assertEqual([i.name for i in genres], ["Rock", "Folk"]) + def test_get_m2m_no_rows(self): + """ + If there are no matching objects, then an empty list should be + returned. + + https://github.com/piccolo-orm/piccolo/issues/1090 + + """ + band = Band.objects().get(Band.name == "Pythonistas").run_sync() + assert band is not None + + Genre.delete(force=True).run_sync() + + genres = band.get_m2m(Band.genres).run_sync() + self.assertEqual(genres, []) + def test_remove_m2m(self): """ Make sure we can remove related items via the joining table. From 1b34405cd9ee211d9eff96fd30e91328fef0acc7 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 1 Oct 2024 09:49:35 +0100 Subject: [PATCH 631/727] bumped version --- CHANGES.rst | 12 ++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 4d589b67b..6a2d0974b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,18 @@ Changes ======= +1.19.1 +------ + +Fixed a bug with the ``get_m2m`` method, which would raise a ``ValueError`` +when no objects were found. It now handles this gracefully and returns an empty +list instead. Thanks to @nVitius for this fix. + +Improved the ASGI templates (including a fix for the latest LiteStar version). +Thanks to @sinisaos for this. + +------------------------------------------------------------------------------- + 1.19.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index d6e05896d..cca064282 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.19.0" +__VERSION__ = "1.19.1" From a8d401eb787112bbde7445e84bf2b5e3fdc626fd Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 1 Oct 2024 09:52:21 +0100 Subject: [PATCH 632/727] fix typo in changelog --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 6a2d0974b..00f53339e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,7 +8,7 @@ Fixed a bug with the ``get_m2m`` method, which would raise a ``ValueError`` when no objects were found. It now handles this gracefully and returns an empty list instead. Thanks to @nVitius for this fix. -Improved the ASGI templates (including a fix for the latest LiteStar version). +Improved the ASGI templates (including a fix for the latest Litestar version). Thanks to @sinisaos for this. ------------------------------------------------------------------------------- From 2cae26b647075638951982d29f73f94a523f3007 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 4 Oct 2024 23:01:21 +0100 Subject: [PATCH 633/727] 1091 Make `get_related` work multiple levels deep (#1092) * make `get_related` work multiple levels deep * fix linter errors --- docs/src/piccolo/query_types/objects.rst | 7 +++ piccolo/query/methods/objects.py | 50 +++++++++++++++- piccolo/table.py | 25 +++----- tests/table/instance/test_get_related.py | 72 +++++++++++++++--------- tests/type_checking.py | 6 ++ 5 files changed, 115 insertions(+), 45 deletions(-) diff --git a/docs/src/piccolo/query_types/objects.rst b/docs/src/piccolo/query_types/objects.rst index e4a2c126e..0d6f93d26 100644 --- a/docs/src/piccolo/query_types/objects.rst +++ b/docs/src/piccolo/query_types/objects.rst @@ -176,6 +176,13 @@ using ``get_related``. >>> manager.name 'Guido' +It works multiple levels deep - for example: + +.. code-block:: python + + concert = await Concert.objects().first() + manager = await concert.get_related(Concert.band_1.manager) + Prefetching related objects ~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index 5e4dda50b..eff040201 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -2,7 +2,7 @@ import typing as t -from piccolo.columns.column_types import ForeignKey +from piccolo.columns.column_types import ForeignKey, ReferencedTable from piccolo.columns.combination import And, Where from piccolo.custom_types import Combinable, TableInstance from piccolo.engine.base import BaseBatch @@ -231,6 +231,54 @@ def run_sync(self, *args, **kwargs) -> None: return run_sync(self.run(*args, **kwargs)) +class GetRelated(t.Generic[ReferencedTable]): + + def __init__(self, row: Table, foreign_key: ForeignKey[ReferencedTable]): + self.row = row + self.foreign_key = foreign_key + + async def run( + self, + node: t.Optional[str] = None, + in_pool: bool = True, + ) -> t.Optional[ReferencedTable]: + references = t.cast( + t.Type[ReferencedTable], + self.foreign_key._foreign_key_meta.resolved_references, + ) + + data = ( + await self.row.__class__.select( + *[ + i.as_alias(i._meta.name) + for i in self.foreign_key.all_columns() + ] + ) + .first() + .run(node=node, in_pool=in_pool) + ) + + # Make sure that some values were returned: + if data is None or not any(data.values()): + return None + + referenced_object = references(**data) + referenced_object._exists_in_db = True + return referenced_object + + def __await__( + self, + ) -> t.Generator[None, None, t.Optional[ReferencedTable]]: + """ + If the user doesn't explicity call .run(), proxy to it as a + convenience. + """ + return self.run().__await__() + + def run_sync(self, *args, **kwargs) -> t.Optional[ReferencedTable]: + return run_sync(self.run(*args, **kwargs)) + + ############################################################################### diff --git a/piccolo/table.py b/piccolo/table.py index b50855f95..bae9b8a47 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -46,7 +46,7 @@ ) from piccolo.query.methods.create_index import CreateIndex from piccolo.query.methods.indexes import Indexes -from piccolo.query.methods.objects import First, UpdateSelf +from piccolo.query.methods.objects import GetRelated, UpdateSelf from piccolo.query.methods.refresh import Refresh from piccolo.querystring import QueryString from piccolo.utils import _camel_to_snake @@ -612,14 +612,14 @@ def refresh( @t.overload def get_related( self, foreign_key: ForeignKey[ReferencedTable] - ) -> First[ReferencedTable]: ... + ) -> GetRelated[ReferencedTable]: ... @t.overload - def get_related(self, foreign_key: str) -> First[Table]: ... + def get_related(self, foreign_key: str) -> GetRelated[Table]: ... def get_related( self, foreign_key: t.Union[str, ForeignKey[ReferencedTable]] - ) -> t.Union[First[Table], First[ReferencedTable]]: + ) -> GetRelated[ReferencedTable]: """ Used to fetch a ``Table`` instance, for the target of a foreign key. @@ -630,8 +630,8 @@ def get_related( >>> print(manager.name) 'Guido' - It can only follow foreign keys one level currently. - i.e. ``Band.manager``, but not ``Band.manager.x.y.z``. + It can only follow foreign keys multiple levels deep. For example, + ``Concert.band_1.manager``. """ if isinstance(foreign_key, str): @@ -645,18 +645,7 @@ def get_related( "ForeignKey column." ) - column_name = foreign_key._meta.name - - references = foreign_key._foreign_key_meta.resolved_references - - return ( - references.objects() - .where( - foreign_key._foreign_key_meta.resolved_target_column - == getattr(self, column_name) - ) - .first() - ) + return GetRelated(foreign_key=foreign_key, row=self) def get_m2m(self, m2m: M2M) -> M2MGetRelated: """ diff --git a/tests/table/instance/test_get_related.py b/tests/table/instance/test_get_related.py index 28c572314..6cae3b9fc 100644 --- a/tests/table/instance/test_get_related.py +++ b/tests/table/instance/test_get_related.py @@ -1,42 +1,62 @@ import typing as t -from unittest import TestCase -from tests.example_apps.music.tables import Band, Manager +from piccolo.testing.test_case import AsyncTableTest +from tests.example_apps.music.tables import Band, Concert, Manager, Venue -TABLES = [Manager, Band] +class TestGetRelated(AsyncTableTest): + tables = [Manager, Band, Concert, Venue] -class TestGetRelated(TestCase): - def setUp(self): - for table in TABLES: - table.create_table().run_sync() + async def asyncSetUp(self): + await super().asyncSetUp() - def tearDown(self): - for table in reversed(TABLES): - table.alter().drop_table().run_sync() + self.manager = Manager(name="Guido") + await self.manager.save() - def test_get_related(self) -> None: + self.band = Band( + name="Pythonistas", manager=self.manager.id, popularity=100 + ) + await self.band.save() + + async def test_foreign_key(self) -> None: """ Make sure you can get a related object from another object instance. """ - manager = Manager(name="Guido") - manager.save().run_sync() + manager = await self.band.get_related(Band.manager) + assert manager is not None + self.assertTrue(manager.name == "Guido") - band = Band(name="Pythonistas", manager=manager.id, popularity=100) - band.save().run_sync() + async def test_non_foreign_key(self): + """ + Make sure that non-ForeignKey raise an exception. + """ + with self.assertRaises(ValueError): + self.band.get_related(Band.name) # type: ignore - _manager = band.get_related(Band.manager).run_sync() - assert _manager is not None - self.assertTrue(_manager.name == "Guido") + async def test_string(self): + """ + Make sure it also works using a string representation of a foreign key. + """ + manager = t.cast(Manager, await self.band.get_related("manager")) + self.assertTrue(manager.name == "Guido") - # Test non-ForeignKey + async def test_invalid_string(self): + """ + Make sure an exception is raised if the foreign key string is invalid. + """ with self.assertRaises(ValueError): - band.get_related(Band.name) # type: ignore + self.band.get_related("abc123") + + async def test_multiple_levels(self): + """ + Make sure ``get_related`` works multiple levels deep. + """ + concert = Concert(band_1=self.band) + await concert.save() - # Make sure it also works using a string - _manager_2 = t.cast(Manager, band.get_related("manager").run_sync()) - self.assertTrue(_manager_2.name == "Guido") + manager = await concert.get_related(Concert.band_1._.manager) + assert manager is not None + self.assertTrue(manager.name == "Guido") - # Test an invalid string - with self.assertRaises(ValueError): - band.get_related("abc123") + band_2_manager = await concert.get_related(Concert.band_2._.manager) + assert band_2_manager is None diff --git a/tests/type_checking.py b/tests/type_checking.py index d1e9d96ca..7288768e1 100644 --- a/tests/type_checking.py +++ b/tests/type_checking.py @@ -49,6 +49,12 @@ async def get_related() -> None: manager = await band.get_related(Band.manager) assert_type(manager, t.Optional[Manager]) + async def get_related_multiple_levels() -> None: + concert = await Concert.objects().first() + assert concert is not None + manager = await concert.get_related(Concert.band_1._.manager) + assert_type(manager, t.Optional[Manager]) + async def get_or_create() -> None: query = Band.objects().get_or_create(Band.name == "Pythonistas") assert_type(await query, Band) From 57673af39b46281244b8f3302d4889fbe6541072 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 4 Oct 2024 23:56:03 +0100 Subject: [PATCH 634/727] more tests (#1095) --- piccolo/query/methods/objects.py | 19 +++++++--- tests/table/instance/test_get_related.py | 48 +++++++++++++++++++++--- 2 files changed, 56 insertions(+), 11 deletions(-) diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index eff040201..f351f1319 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -242,18 +242,22 @@ async def run( node: t.Optional[str] = None, in_pool: bool = True, ) -> t.Optional[ReferencedTable]: - references = t.cast( - t.Type[ReferencedTable], - self.foreign_key._foreign_key_meta.resolved_references, - ) + if not self.row._exists_in_db: + raise ValueError("The object doesn't exist in the database.") + + root_table = self.row.__class__ data = ( - await self.row.__class__.select( + await root_table.select( *[ i.as_alias(i._meta.name) for i in self.foreign_key.all_columns() ] ) + .where( + root_table._meta.primary_key + == getattr(self.row, root_table._meta.primary_key._meta.name) + ) .first() .run(node=node, in_pool=in_pool) ) @@ -262,6 +266,11 @@ async def run( if data is None or not any(data.values()): return None + references = t.cast( + t.Type[ReferencedTable], + self.foreign_key._foreign_key_meta.resolved_references, + ) + referenced_object = references(**data) referenced_object._exists_in_db = True return referenced_object diff --git a/tests/table/instance/test_get_related.py b/tests/table/instance/test_get_related.py index 6cae3b9fc..a54938917 100644 --- a/tests/table/instance/test_get_related.py +++ b/tests/table/instance/test_get_related.py @@ -10,6 +10,9 @@ class TestGetRelated(AsyncTableTest): async def asyncSetUp(self): await super().asyncSetUp() + # Setup two pairs of manager/band, so we can make sure the correct + # objects are returned. + self.manager = Manager(name="Guido") await self.manager.save() @@ -18,13 +21,25 @@ async def asyncSetUp(self): ) await self.band.save() + self.manager_2 = Manager(name="Graydon") + await self.manager_2.save() + + self.band_2 = Band( + name="Rustaceans", manager=self.manager_2.id, popularity=100 + ) + await self.band_2.save() + async def test_foreign_key(self) -> None: """ Make sure you can get a related object from another object instance. """ manager = await self.band.get_related(Band.manager) assert manager is not None - self.assertTrue(manager.name == "Guido") + self.assertTrue(manager.id == self.manager.id) + + manager_2 = await self.band_2.get_related(Band.manager) + assert manager_2 is not None + self.assertTrue(manager_2.id == self.manager_2.id) async def test_non_foreign_key(self): """ @@ -38,7 +53,7 @@ async def test_string(self): Make sure it also works using a string representation of a foreign key. """ manager = t.cast(Manager, await self.band.get_related("manager")) - self.assertTrue(manager.name == "Guido") + self.assertTrue(manager.id == self.manager.id) async def test_invalid_string(self): """ @@ -51,12 +66,33 @@ async def test_multiple_levels(self): """ Make sure ``get_related`` works multiple levels deep. """ - concert = Concert(band_1=self.band) + concert = Concert(band_1=self.band, band_2=self.band_2) await concert.save() manager = await concert.get_related(Concert.band_1._.manager) assert manager is not None - self.assertTrue(manager.name == "Guido") + self.assertTrue(manager.id == self.manager.id) + + manager_2 = await concert.get_related(Concert.band_2._.manager) + assert manager_2 is not None + self.assertTrue(manager_2.id == self.manager_2.id) - band_2_manager = await concert.get_related(Concert.band_2._.manager) - assert band_2_manager is None + async def test_no_match(self): + """ + If not related object exists, make sure ``None`` is returned. + """ + concert = Concert(band_1=self.band, band_2=None) + await concert.save() + + manager_2 = await concert.get_related(Concert.band_2._.manager) + assert manager_2 is None + + async def test_not_in_db(self): + """ + If the object we're calling ``get_related`` on doesn't exist in the + database, then make sure an error is raised. + """ + concert = Concert(band_1=self.band, band_2=self.band_2) + + with self.assertRaises(ValueError): + await concert.get_related(Concert.band_1._.manager) From f4bf25a133dd6a0f28c95571ee97bf75be87befe Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 4 Oct 2024 23:37:05 +0100 Subject: [PATCH 635/727] bump version --- CHANGES.rst | 12 ++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 00f53339e..7639f4cff 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,18 @@ Changes ======= +1.20.0 +------ + +``get_related`` now works multiple layers deep: + +.. code-block:: python + + concert = await Concert.objects().first() + manager = await concert.get_related(Concert.band_1._.manager) + +------------------------------------------------------------------------------- + 1.19.1 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index cca064282..e1974bd81 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.19.1" +__VERSION__ = "1.20.0" From db69230fe364141c95be916f2c7e859b353b7124 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 18 Oct 2024 12:24:27 +0100 Subject: [PATCH 636/727] 1101 Fix joins with `ForeignKey` columns with `db_column_name` specified (#1102) * fix joins with `db_column_name` * update tests --- piccolo/query/methods/select.py | 2 +- tests/columns/test_db_column_name.py | 71 +++++++++++++++++++++++----- 2 files changed, 59 insertions(+), 14 deletions(-) diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 5d7856c5a..c45fdbd46 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -534,7 +534,7 @@ def _get_joins(self, columns: t.Sequence[Selectable]) -> t.List[str]: _joins.append( f'LEFT JOIN {right_tablename} "{table_alias}"' " ON " - f'({left_tablename}."{key._meta.name}" = "{table_alias}"."{pk_name}")' # noqa: E501 + f'({left_tablename}."{key._meta.db_column_name}" = "{table_alias}"."{pk_name}")' # noqa: E501 ) joins.extend(_joins) diff --git a/tests/columns/test_db_column_name.py b/tests/columns/test_db_column_name.py index 33beffed9..96a182149 100644 --- a/tests/columns/test_db_column_name.py +++ b/tests/columns/test_db_column_name.py @@ -1,12 +1,20 @@ -from piccolo.columns.column_types import Integer, Serial, Varchar -from piccolo.table import Table +import typing as t + +from piccolo.columns.column_types import ForeignKey, Integer, Serial, Varchar +from piccolo.table import Table, create_db_tables_sync, drop_db_tables_sync from tests.base import DBTestCase, engine_is, engines_only, engines_skip +class Manager(Table): + id: Serial + name = Varchar() + + class Band(Table): id: Serial name = Varchar(db_column_name="regrettable_column_name") popularity = Integer() + manager = ForeignKey(Manager, db_column_name="manager_fk") class TestDBColumnName(DBTestCase): @@ -22,10 +30,15 @@ class MyTable(Table): """ def setUp(self): - Band.create_table().run_sync() + create_db_tables_sync(Band, Manager) def tearDown(self): - Band.alter().drop_table().run_sync() + drop_db_tables_sync(Band, Manager) + + def insert_band(self, manager: t.Optional[Manager] = None) -> Band: + band = Band(name="Pythonistas", popularity=1000, manager=manager) + band.save().run_sync() + return band @engines_only("postgres", "cockroach") def test_column_name_correct(self): @@ -45,8 +58,7 @@ def test_save(self): """ Make sure save queries work correctly. """ - band = Band(name="Pythonistas", popularity=1000) - band.save().run_sync() + self.insert_band() band_from_db = Band.objects().first().run_sync() assert band_from_db is not None @@ -56,11 +68,7 @@ def test_create(self): """ Make sure create queries work correctly. """ - band = ( - Band.objects() - .create(name="Pythonistas", popularity=1000) - .run_sync() - ) + band = self.insert_band() self.assertEqual(band.name, "Pythonistas") band_from_db = Band.objects().first().run_sync() @@ -74,7 +82,7 @@ def test_select(self): name to it's alias, but it's hard to predict what behaviour the user wants. """ - Band.objects().create(name="Pythonistas", popularity=1000).run_sync() + self.insert_band() # Make sure we can select all columns bands = Band.select().run_sync() @@ -86,6 +94,7 @@ def test_select(self): "id": bands[0]["id"], "regrettable_column_name": "Pythonistas", "popularity": 1000, + "manager_fk": None, } ], ) @@ -97,6 +106,7 @@ def test_select(self): "id": 1, "regrettable_column_name": "Pythonistas", "popularity": 1000, + "manager_fk": None, } ], ) @@ -123,11 +133,36 @@ def test_select(self): ], ) + def test_join(self): + """ + Make sure that foreign keys with a ``db_column_name`` specified still + work for joins. + + https://github.com/piccolo-orm/piccolo/issues/1101 + + """ + manager = Manager.objects().create(name="Guido").run_sync() + band = self.insert_band(manager=manager) + + bands = Band.select().where(Band.manager.name == "Guido").run_sync() + + self.assertListEqual( + bands, + [ + { + "id": band.id, + "manager_fk": manager.id, + "popularity": 1000, + "regrettable_column_name": "Pythonistas", + } + ], + ) + def test_update(self): """ Make sure update queries work correctly. """ - Band.objects().create(name="Pythonistas", popularity=1000).run_sync() + self.insert_band() Band.update({Band.name: "Pythonistas 2"}, force=True).run_sync() @@ -140,6 +175,7 @@ def test_update(self): "id": bands[0]["id"], "regrettable_column_name": "Pythonistas 2", "popularity": 1000, + "manager_fk": None, } ], ) @@ -151,6 +187,7 @@ def test_update(self): "id": 1, "regrettable_column_name": "Pythonistas 2", "popularity": 1000, + "manager_fk": None, } ], ) @@ -166,6 +203,7 @@ def test_update(self): "id": bands[0]["id"], "regrettable_column_name": "Pythonistas 3", "popularity": 1000, + "manager_fk": None, } ], ) @@ -177,6 +215,7 @@ def test_update(self): "id": 1, "regrettable_column_name": "Pythonistas 3", "popularity": 1000, + "manager_fk": None, } ], ) @@ -199,11 +238,13 @@ def test_delete(self): "id": 1, "regrettable_column_name": "Pythonistas", "popularity": 1000, + "manager_fk": None, }, { "id": 2, "regrettable_column_name": "Rustaceans", "popularity": 500, + "manager_fk": None, }, ], ) @@ -218,6 +259,7 @@ def test_delete(self): "id": 1, "regrettable_column_name": "Pythonistas", "popularity": 1000, + "manager_fk": None, } ], ) @@ -244,11 +286,13 @@ def test_delete_alt(self): "id": result[0]["id"], "regrettable_column_name": "Pythonistas", "popularity": 1000, + "manager_fk": None, }, { "id": result[1]["id"], "regrettable_column_name": "Rustaceans", "popularity": 500, + "manager_fk": None, }, ], ) @@ -263,6 +307,7 @@ def test_delete_alt(self): "id": result[0]["id"], "regrettable_column_name": "Pythonistas", "popularity": 1000, + "manager_fk": None, } ], ) From d1b0bb13322affe5baf6fb216cd0a09cc4d80e0f Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 18 Oct 2024 13:27:43 +0100 Subject: [PATCH 637/727] add Postgres 17 (#1106) --- .github/workflows/tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 26690051f..9e56a9ace 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -41,7 +41,7 @@ jobs: # These tests are slow, so we only run on the latest Python # version. python-version: ["3.10"] - postgres-version: [14] + postgres-version: [17] services: postgres: image: postgres:${{ matrix.postgres-version }} @@ -86,7 +86,7 @@ jobs: strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - postgres-version: [11, 12, 13, 14, 15, 16] + postgres-version: [12, 13, 14, 15, 16, 17] # Service containers to run with `container-job` services: From 67be2d6b3b834c56effcfd2b88312f437d3f5128 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 18 Oct 2024 13:38:26 +0100 Subject: [PATCH 638/727] bumped version --- CHANGES.rst | 10 ++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 7639f4cff..11dba5beb 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,16 @@ Changes ======= +1.21.0 +------ + +Postgres 17 is now officially supported. + +Fixed a bug with joins, when a ``ForeignKey`` column had ``db_column_name`` +specified. Thanks to @jessemcl-flwls for reporting this issue. + +------------------------------------------------------------------------------- + 1.20.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index e1974bd81..fa25b57e7 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.20.0" +__VERSION__ = "1.21.0" From e2fa8f89f94bc995722ca42db856d9aad6471a46 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 18 Oct 2024 20:33:13 +0100 Subject: [PATCH 639/727] be able to pass an engine into `TableStorage` (#1109) --- piccolo/table_reflection.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/piccolo/table_reflection.py b/piccolo/table_reflection.py index 69c0bb7e2..3d0500990 100644 --- a/piccolo/table_reflection.py +++ b/piccolo/table_reflection.py @@ -8,6 +8,8 @@ from dataclasses import dataclass from piccolo.apps.schema.commands.generate import get_output_schema +from piccolo.engine import engine_finder +from piccolo.engine.base import Engine from piccolo.table import Table @@ -78,9 +80,16 @@ class TableStorage(metaclass=Singleton): works with Postgres. """ - def __init__(self): + def __init__(self, engine: t.Optional[Engine] = None): + """ + :param engine: + Which engine to use to make the database queries. If not specified, + we try importing an engine from ``piccolo_conf.py``. + + """ + self.engine = engine or engine_finder() self.tables = ImmutableDict() - self._schema_tables = {} + self._schema_tables: t.Dict[str, t.List[str]] = {} async def reflect( self, @@ -120,10 +129,13 @@ async def reflect( exclude_list = self._to_list(exclude) if keep_existing: - exclude += self._schema_tables.get(schema_name, []) + exclude_list += self._schema_tables.get(schema_name, []) output_schema = await get_output_schema( - schema_name=schema_name, include=include_list, exclude=exclude_list + schema_name=schema_name, + include=include_list, + exclude=exclude_list, + engine=self.engine, ) add_tables = [ self._add_table(schema_name=schema_name, table=table) @@ -177,7 +189,7 @@ async def _add_table(self, schema_name: str, table: t.Type[Table]) -> None: def _add_to_schema_tables(self, schema_name: str, table_name: str) -> None: """ - We keep record of schemas and their tables for easy use. This method + We keep a record of schemas and their tables for easy use. This method adds a table to its schema. """ From 265b0c2d82a87e4a07dcdf6fe4e938df6196749d Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 20 Oct 2024 07:26:18 +0100 Subject: [PATCH 640/727] 1094 Add Python 3.13 support (#1104) * add Python 3.13 to `setup.py` and CI * upgrade `asyncpg` * run integration tests on Python 3.12 for now --- .github/workflows/release.yaml | 2 +- .github/workflows/tests.yaml | 16 ++++++++-------- requirements/extras/postgres.txt | 2 +- setup.py | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 60f0b83b1..4d70f80a5 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -14,7 +14,7 @@ jobs: - uses: "actions/checkout@v3" - uses: "actions/setup-python@v1" with: - python-version: 3.12 + python-version: 3.13 - name: "Install dependencies" run: "pip install -r requirements/dev-requirements.txt" - name: "Publish to PyPI" diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 9e56a9ace..0d71446a3 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -16,7 +16,7 @@ jobs: timeout-minutes: 60 strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v3 @@ -40,7 +40,7 @@ jobs: matrix: # These tests are slow, so we only run on the latest Python # version. - python-version: ["3.10"] + python-version: ["3.12"] postgres-version: [17] services: postgres: @@ -85,7 +85,7 @@ jobs: timeout-minutes: 60 strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] postgres-version: [12, 13, 14, 15, 16, 17] # Service containers to run with `container-job` @@ -134,14 +134,14 @@ jobs: PG_PASSWORD: postgres - name: Upload coverage uses: codecov/codecov-action@v1 - if: matrix.python-version == '3.12' + if: matrix.python-version == '3.13' cockroach: runs-on: ubuntu-latest timeout-minutes: 60 strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] cockroachdb-version: ["v24.1.0"] steps: - uses: actions/checkout@v3 @@ -168,14 +168,14 @@ jobs: PG_DATABASE: piccolo - name: Upload coverage uses: codecov/codecov-action@v1 - if: matrix.python-version == '3.12' + if: matrix.python-version == '3.13' sqlite: runs-on: ubuntu-latest timeout-minutes: 60 strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v3 @@ -193,4 +193,4 @@ jobs: run: ./scripts/test-sqlite.sh - name: Upload coverage uses: codecov/codecov-action@v1 - if: matrix.python-version == '3.12' + if: matrix.python-version == '3.13' diff --git a/requirements/extras/postgres.txt b/requirements/extras/postgres.txt index 864747fa4..1b54800e6 100644 --- a/requirements/extras/postgres.txt +++ b/requirements/extras/postgres.txt @@ -1 +1 @@ -asyncpg>=0.21.0 +asyncpg>=0.30.0 diff --git a/setup.py b/setup.py index 0996a1297..315dc5e6e 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ def extras_require() -> t.Dict[str, t.List[str]]: long_description_content_type="text/markdown", author="Daniel Townsend", author_email="dan@dantownsend.co.uk", - python_requires=">=3.8.0", + python_requires=">=3.9.0", url="https://github.com/piccolo-orm/piccolo", packages=find_packages(exclude=("tests",)), package_data={ @@ -84,11 +84,11 @@ def extras_require() -> t.Dict[str, t.List[str]]: "License :: OSI Approved :: MIT License", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Framework :: AsyncIO", "Typing :: Typed", From 85ead5a055639aff4e9247baff173c6455145106 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 23 Oct 2024 21:40:19 +0100 Subject: [PATCH 641/727] 1103 Improve arrow function (#1112) * improve arrow function * fix tests * refactor, so we have an `Arrow` function * add test for nested arrow functions * skip sqlite * allow `arrow` to access keys multiple levels deep * improve the example data in the playground for JSON data * move arrow function to JSON, as it can be used by JSON or JSONB * add `arrow` function to Arrow, so it can be called recursively * change heading levels of JSON docs * move `Arrow` to operators folder * update docs * improve docstring * add `technicians` to example JSON * improve docstrings * allow `QueryString` as an arg type to `Arrow` * fix docstring error * make sure integers can be passed in * add `QueryString` as an arg type to `arrow` method * added `GetElementFromPath` * add docs for ``from_path`` * add `__getitem__` as a shortcut for the arrow method * update the docs to use the square bracket notation * explain why the method is called `arrow` * move arrow tests into separate class * add `test_multiple_levels_deep` * add tests for `for_path` * last documentation tweaks * add basic operator tests --- docs/src/piccolo/schema/column_types.rst | 123 ++++++++++++++++--- piccolo/apps/playground/commands/run.py | 9 ++ piccolo/columns/column_types.py | 117 ++++++++++++------- piccolo/query/base.py | 17 ++- piccolo/query/operators/__init__.py | 0 piccolo/query/operators/json.py | 111 ++++++++++++++++++ piccolo/querystring.py | 16 ++- tests/columns/test_jsonb.py | 143 ++++++++++++++++------- tests/query/operators/__init__.py | 0 tests/query/operators/test_json.py | 52 +++++++++ 10 files changed, 482 insertions(+), 106 deletions(-) create mode 100644 piccolo/query/operators/__init__.py create mode 100644 piccolo/query/operators/json.py create mode 100644 tests/query/operators/__init__.py create mode 100644 tests/query/operators/test_json.py diff --git a/docs/src/piccolo/schema/column_types.rst b/docs/src/piccolo/schema/column_types.rst index 37484978a..5c70dd482 100644 --- a/docs/src/piccolo/schema/column_types.rst +++ b/docs/src/piccolo/schema/column_types.rst @@ -189,18 +189,15 @@ Storing JSON can be useful in certain situations, for example - raw API responses, data from a Javascript app, and for storing data with an unknown or changing schema. -==== -JSON -==== +==================== +``JSON`` / ``JSONB`` +==================== .. autoclass:: JSON -===== -JSONB -===== - .. autoclass:: JSONB +=========== Serialising =========== @@ -224,6 +221,7 @@ You can also pass in a JSON string if you prefer: ) await studio.save() +============= Deserialising ============= @@ -257,29 +255,122 @@ With ``objects`` queries, we can modify the returned JSON, and then save it: studio['facilities']['restaurant'] = False await studio.save() -arrow -===== +================ +Getting elements +================ + +``JSON`` and ``JSONB`` columns have an ``arrow`` method (representing the +``->`` operator in Postgres), which is useful for retrieving a child element +from the JSON data. + +.. note:: Postgres and CockroachDB only. -``JSONB`` columns have an ``arrow`` function, which is useful for retrieving -a subset of the JSON data: +``select`` queries +================== + +If we have the following JSON stored in the ``RecordingStudio.facilities`` +column: + +.. code-block:: json + + { + "instruments": { + "drum_kits": 2, + "electric_guitars": 10 + }, + "restaurant": true, + "technicians": [ + { + "name": "Alice Jones" + }, + { + "name": "Bob Williams" + } + ] + } + +We can retrieve the ``restaurant`` value from the JSON object: .. code-block:: python >>> await RecordingStudio.select( - ... RecordingStudio.name, - ... RecordingStudio.facilities.arrow('mixing_desk').as_alias('mixing_desk') + ... RecordingStudio.facilities.arrow('restaurant') + ... .as_alias('restaurant') ... ).output(load_json=True) - [{'name': 'Abbey Road', 'mixing_desk': True}] + [{'restaurant': True}, ...] -It can also be used for filtering in a where clause: +As a convenience, you can use square brackets, instead of calling ``arrow`` +explicitly: + +.. code-block:: python + + >>> await RecordingStudio.select( + ... RecordingStudio.facilities['restaurant'] + ... .as_alias('restaurant') + ... ).output(load_json=True) + [{'restaurant': True}, ...] + +You can drill multiple levels deep by calling ``arrow`` multiple times (or +alternatively use the :ref:`from_path` method - see below). + +Here we fetch the number of drum kits that the recording studio has: + +.. code-block:: python + + >>> await RecordingStudio.select( + ... RecordingStudio.facilities["instruments"]["drum_kits"] + ... .as_alias("drum_kits") + ... ).output(load_json=True) + [{'drum_kits': 2}, ...] + +If you have a JSON object which consists of arrays and objects, then you can +navigate the array elements by passing in an integer to ``arrow``. + +Here we fetch the first technician from the array: + +.. code-block:: python + + >>> await RecordingStudio.select( + ... RecordingStudio.facilities["technicians"][0]["name"] + ... .as_alias("technician_name") + ... ).output(load_json=True) + + [{'technician_name': 'Alice Jones'}, ...] + +``where`` clauses +================= + +The ``arrow`` operator can also be used for filtering in a where clause: .. code-block:: python >>> await RecordingStudio.select(RecordingStudio.name).where( - ... RecordingStudio.facilities.arrow('mixing_desk') == True + ... RecordingStudio.facilities['mixing_desk'].eq(True) ... ) [{'name': 'Abbey Road'}] +.. _from_path: + +============= +``from_path`` +============= + +This works the same as ``arrow`` but is more optimised if you need to return +part of a highly nested JSON structure. + +.. code-block:: python + + >>> await RecordingStudio.select( + ... RecordingStudio.facilities.from_path([ + ... "technicians", + ... 0, + ... "name" + ... ]).as_alias("technician_name") + ... ).output(load_json=True) + + [{'technician_name': 'Alice Jones'}, ...] + +============= Handling null ============= diff --git a/piccolo/apps/playground/commands/run.py b/piccolo/apps/playground/commands/run.py index 3f435c44b..343c02e38 100644 --- a/piccolo/apps/playground/commands/run.py +++ b/piccolo/apps/playground/commands/run.py @@ -233,6 +233,11 @@ def populate(): RecordingStudio.facilities: { "restaurant": True, "mixing_desk": True, + "instruments": {"electric_guitars": 10, "drum_kits": 2}, + "technicians": [ + {"name": "Alice Jones"}, + {"name": "Bob Williams"}, + ], }, } ) @@ -244,6 +249,10 @@ def populate(): RecordingStudio.facilities: { "restaurant": False, "mixing_desk": True, + "instruments": {"electric_guitars": 6, "drum_kits": 3}, + "technicians": [ + {"name": "Frank Smith"}, + ], }, }, ) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index d16329b49..5d0b04acc 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -70,6 +70,10 @@ class Band(Table): if t.TYPE_CHECKING: # pragma: no cover from piccolo.columns.base import ColumnMeta + from piccolo.query.operators.json import ( + GetChildElement, + GetElementFromPath, + ) from piccolo.table import Table @@ -2319,6 +2323,76 @@ def column_type(self): else: return "JSON" + ########################################################################### + + def arrow(self, key: t.Union[str, int, QueryString]) -> GetChildElement: + """ + Allows a child element of the JSON structure to be returned - for + example:: + + >>> await RecordingStudio.select( + ... RecordingStudio.facilities.arrow("restaurant") + ... ) + + """ + from piccolo.query.operators.json import GetChildElement + + alias = self._alias or self._meta.get_default_alias() + return GetChildElement(identifier=self, key=key, alias=alias) + + def __getitem__( + self, value: t.Union[str, int, QueryString] + ) -> GetChildElement: + """ + A shortcut for the ``arrow`` method, used for retrieving a child + element. + + For example: + + .. code-block:: python + + >>> await RecordingStudio.select( + ... RecordingStudio.facilities["restaurant"] + ... ) + + """ + return self.arrow(key=value) + + def from_path( + self, + path: t.List[t.Union[str, int]], + ) -> GetElementFromPath: + """ + Allows an element of the JSON structure to be returned, which can be + arbitrarily deep. For example:: + + >>> await RecordingStudio.select( + ... RecordingStudio.facilities.from_path([ + ... "technician", + ... 0, + ... "first_name" + ... ]) + ... ) + + It's the same as calling ``arrow`` multiple times, but is more + efficient / convenient if extracting highly nested data:: + + >>> await RecordingStudio.select( + ... RecordingStudio.facilities.arrow( + ... "technician" + ... ).arrow( + ... 0 + ... ).arrow( + ... "first_name" + ... ) + ... ) + + """ + from piccolo.query.operators.json import GetElementFromPath + + alias = self._alias or self._meta.get_default_alias() + return GetElementFromPath(identifier=self, path=path, alias=alias) + ########################################################################### # Descriptors @@ -2337,10 +2411,10 @@ def __set__(self, obj, value: t.Union[str, t.Dict]): class JSONB(JSON): """ - Used for storing JSON strings - Postgres only. The data is stored in a - binary format, and can be queried. Insertion can be slower (as it needs to - be converted to the binary format). The benefits of JSONB generally - outweigh the downsides. + Used for storing JSON strings - Postgres / CochroachDB only. The data is + stored in a binary format, and can be queried more efficiently. Insertion + can be slower (as it needs to be converted to the binary format). The + benefits of JSONB generally outweigh the downsides. :param default: Either a JSON string can be provided, or a Python ``dict`` or ``list`` @@ -2352,41 +2426,6 @@ class JSONB(JSON): def column_type(self): return "JSONB" # Must be defined, we override column_type() in JSON() - def arrow(self, key: str) -> JSONB: - """ - Allows part of the JSON structure to be returned - for example, - for {"a": 1}, and a key value of "a", then 1 will be returned. - """ - instance = t.cast(JSONB, self.copy()) - instance.json_operator = f"-> '{key}'" - return instance - - def get_select_string( - self, engine_type: str, with_alias: bool = True - ) -> QueryString: - select_string = self._meta.get_full_name(with_alias=False) - - if self.json_operator is not None: - select_string += f" {self.json_operator}" - - if with_alias: - alias = self._alias or self._meta.get_default_alias() - select_string += f' AS "{alias}"' - - return QueryString(select_string) - - def eq(self, value) -> Where: - """ - See ``Boolean.eq`` for more details. - """ - return self.__eq__(value) - - def ne(self, value) -> Where: - """ - See ``Boolean.ne`` for more details. - """ - return self.__ne__(value) - ########################################################################### # Descriptors diff --git a/piccolo/query/base.py b/piccolo/query/base.py index ff49c0e3b..45049e1e1 100644 --- a/piccolo/query/base.py +++ b/piccolo/query/base.py @@ -6,6 +6,7 @@ from piccolo.columns.column_types import JSON, JSONB from piccolo.custom_types import QueryResponseType, TableInstance from piccolo.query.mixins import ColumnsDelegate +from piccolo.query.operators.json import JSONQueryString from piccolo.querystring import QueryString from piccolo.utils.encoding import load_json from piccolo.utils.objects import make_nested_object @@ -65,16 +66,20 @@ async def _process_results(self, results) -> QueryResponseType: self, "columns_delegate", None ) + json_column_names: t.List[str] = [] + if columns_delegate is not None: - json_columns = [ - i - for i in columns_delegate.selected_columns - if isinstance(i, (JSON, JSONB)) - ] + json_columns: t.List[t.Union[JSON, JSONB]] = [] + + for column in columns_delegate.selected_columns: + if isinstance(column, (JSON, JSONB)): + json_columns.append(column) + elif isinstance(column, JSONQueryString): + if alias := column._alias: + json_column_names.append(alias) else: json_columns = self.table._meta.json_columns - json_column_names = [] for column in json_columns: if column._alias is not None: json_column_names.append(column._alias) diff --git a/piccolo/query/operators/__init__.py b/piccolo/query/operators/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/piccolo/query/operators/json.py b/piccolo/query/operators/json.py new file mode 100644 index 000000000..ea6d05097 --- /dev/null +++ b/piccolo/query/operators/json.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import typing as t + +from piccolo.querystring import QueryString +from piccolo.utils.encoding import dump_json + +if t.TYPE_CHECKING: + from piccolo.columns.column_types import JSON + + +class JSONQueryString(QueryString): + + def clean_value(self, value: t.Any): + if not isinstance(value, (str, QueryString)): + value = dump_json(value) + return value + + def __eq__(self, value) -> QueryString: # type: ignore[override] + value = self.clean_value(value) + return QueryString("{} = {}", self, value) + + def __ne__(self, value) -> QueryString: # type: ignore[override] + value = self.clean_value(value) + return QueryString("{} != {}", self, value) + + def eq(self, value) -> QueryString: + return self.__eq__(value) + + def ne(self, value) -> QueryString: + return self.__ne__(value) + + +class GetChildElement(JSONQueryString): + """ + Allows you to get a child element from a JSON object. + + You can access this via the ``arrow`` function on ``JSON`` and ``JSONB`` + columns. + + """ + + def __init__( + self, + identifier: t.Union[JSON, QueryString], + key: t.Union[str, int, QueryString], + alias: t.Optional[str] = None, + ): + if isinstance(key, int): + # asyncpg only accepts integer keys if we explicitly mark it as an + # int. + key = QueryString("{}::int", key) + + super().__init__("{} -> {}", identifier, key, alias=alias) + + def arrow(self, key: t.Union[str, int, QueryString]) -> GetChildElement: + """ + This allows you to drill multiple levels deep into a JSON object if + needed. + + For example:: + + >>> await RecordingStudio.select( + ... RecordingStudio.name, + ... RecordingStudio.facilities.arrow( + ... "instruments" + ... ).arrow( + ... "drum_kits" + ... ).as_alias("drum_kits") + ... ).output(load_json=True) + [ + {'name': 'Abbey Road', 'drum_kits': 2}, + {'name': 'Electric Lady', 'drum_kits': 3} + ] + + """ + return GetChildElement(identifier=self, key=key, alias=self._alias) + + def __getitem__( + self, value: t.Union[str, int, QueryString] + ) -> GetChildElement: + return GetChildElement(identifier=self, key=value, alias=self._alias) + + +class GetElementFromPath(JSONQueryString): + """ + Allows you to retrieve an element from a JSON object by specifying a path. + It can be several levels deep. + + You can access this via the ``from_path`` function on ``JSON`` and + ``JSONB`` columns. + + """ + + def __init__( + self, + identifier: t.Union[JSON, QueryString], + path: t.List[t.Union[str, int]], + alias: t.Optional[str] = None, + ): + """ + :param path: + For example: ``["technician", 0, "name"]``. + + """ + super().__init__( + "{} #> {}", + identifier, + [str(i) if isinstance(i, int) else i for i in path], + alias=alias, + ) diff --git a/piccolo/querystring.py b/piccolo/querystring.py index 22f5f215a..7dec758a8 100644 --- a/piccolo/querystring.py +++ b/piccolo/querystring.py @@ -259,10 +259,22 @@ def get_where_string(self, engine_type: str) -> QueryString: # Basic logic def __eq__(self, value) -> QueryString: # type: ignore[override] - return QueryString("{} = {}", self, value) + if value is None: + return QueryString("{} IS NULL", self) + else: + return QueryString("{} = {}", self, value) def __ne__(self, value) -> QueryString: # type: ignore[override] - return QueryString("{} != {}", self, value) + if value is None: + return QueryString("{} IS NOT NULL", self, value) + else: + return QueryString("{} != {}", self, value) + + def eq(self, value) -> QueryString: + return self.__eq__(value) + + def ne(self, value) -> QueryString: + return self.__ne__(value) def __add__(self, value) -> QueryString: return QueryString("{} + {}", self, value) diff --git a/tests/columns/test_jsonb.py b/tests/columns/test_jsonb.py index fe90e769b..f38c0de05 100644 --- a/tests/columns/test_jsonb.py +++ b/tests/columns/test_jsonb.py @@ -1,6 +1,6 @@ from piccolo.columns.column_types import JSONB, ForeignKey, Varchar from piccolo.table import Table -from piccolo.testing.test_case import TableTest +from piccolo.testing.test_case import AsyncTableTest, TableTest from tests.base import engines_only, engines_skip @@ -137,93 +137,150 @@ def test_as_alias_join(self): [{"name": "Guitar", "studio_facilities": {"mixing_desk": True}}], ) - def test_arrow(self): + +@engines_only("postgres", "cockroach") +class TestArrow(AsyncTableTest): + tables = [RecordingStudio, Instrument] + + async def insert_row(self): + await RecordingStudio( + name="Abbey Road", facilities='{"mixing_desk": true}' + ).save() + + async def test_arrow(self): """ Test using the arrow function to retrieve a subset of the JSON. """ - RecordingStudio( - name="Abbey Road", facilities='{"mixing_desk": true}' - ).save().run_sync() + await self.insert_row() - row = ( - RecordingStudio.select( - RecordingStudio.facilities.arrow("mixing_desk") - ) - .first() - .run_sync() - ) + row = await RecordingStudio.select( + RecordingStudio.facilities.arrow("mixing_desk") + ).first() assert row is not None self.assertEqual(row["facilities"], "true") - row = ( + row = await ( RecordingStudio.select( RecordingStudio.facilities.arrow("mixing_desk") ) .output(load_json=True) .first() - .run_sync() ) assert row is not None self.assertEqual(row["facilities"], True) - def test_arrow_as_alias(self): + async def test_arrow_as_alias(self): """ Test using the arrow function to retrieve a subset of the JSON. """ - RecordingStudio( - name="Abbey Road", facilities='{"mixing_desk": true}' - ).save().run_sync() + await self.insert_row() - row = ( - RecordingStudio.select( - RecordingStudio.facilities.arrow("mixing_desk").as_alias( - "mixing_desk" - ) + row = await RecordingStudio.select( + RecordingStudio.facilities.arrow("mixing_desk").as_alias( + "mixing_desk" ) - .first() - .run_sync() - ) + ).first() + assert row is not None + self.assertEqual(row["mixing_desk"], "true") + + async def test_square_brackets(self): + """ + Make sure we can use square brackets instead of calling ``arrow`` + explicitly. + """ + await self.insert_row() + + row = await RecordingStudio.select( + RecordingStudio.facilities["mixing_desk"].as_alias("mixing_desk") + ).first() assert row is not None self.assertEqual(row["mixing_desk"], "true") - def test_arrow_where(self): + async def test_multiple_levels_deep(self): + """ + Make sure elements can be extracted multiple levels deep, and using + array indexes. + """ + await RecordingStudio( + name="Abbey Road", + facilities={ + "technicians": [ + {"name": "Alice Jones"}, + {"name": "Bob Williams"}, + ] + }, + ).save() + + response = await RecordingStudio.select( + RecordingStudio.facilities["technicians"][0]["name"].as_alias( + "technician_name" + ) + ).output(load_json=True) + assert response is not None + self.assertListEqual(response, [{"technician_name": "Alice Jones"}]) + + async def test_arrow_where(self): """ Make sure the arrow function can be used within a WHERE clause. """ - RecordingStudio( - name="Abbey Road", facilities='{"mixing_desk": true}' - ).save().run_sync() + await self.insert_row() self.assertEqual( - RecordingStudio.count() - .where(RecordingStudio.facilities.arrow("mixing_desk").eq(True)) - .run_sync(), + await RecordingStudio.count().where( + RecordingStudio.facilities.arrow("mixing_desk").eq(True) + ), 1, ) self.assertEqual( - RecordingStudio.count() - .where(RecordingStudio.facilities.arrow("mixing_desk").eq(False)) - .run_sync(), + await RecordingStudio.count().where( + RecordingStudio.facilities.arrow("mixing_desk").eq(False) + ), 0, ) - def test_arrow_first(self): + async def test_arrow_first(self): """ Make sure the arrow function can be used with the first clause. """ - RecordingStudio.insert( + await RecordingStudio.insert( RecordingStudio(facilities='{"mixing_desk": true}'), RecordingStudio(facilities='{"mixing_desk": false}'), - ).run_sync() + ) self.assertEqual( - RecordingStudio.select( + await RecordingStudio.select( RecordingStudio.facilities.arrow("mixing_desk").as_alias( "mixing_desk" ) - ) - .first() - .run_sync(), + ).first(), {"mixing_desk": "true"}, ) + + +@engines_only("postgres", "cockroach") +class TestFromPath(AsyncTableTest): + + tables = [RecordingStudio, Instrument] + + async def test_from_path(self): + """ + Make sure ``from_path`` can be used for complex nested data. + """ + await RecordingStudio( + name="Abbey Road", + facilities={ + "technicians": [ + {"name": "Alice Jones"}, + {"name": "Bob Williams"}, + ] + }, + ).save() + + response = await RecordingStudio.select( + RecordingStudio.facilities.from_path( + ["technicians", 0, "name"] + ).as_alias("technician_name") + ).output(load_json=True) + assert response is not None + self.assertListEqual(response, [{"technician_name": "Alice Jones"}]) diff --git a/tests/query/operators/__init__.py b/tests/query/operators/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/query/operators/test_json.py b/tests/query/operators/test_json.py new file mode 100644 index 000000000..d7840ef9b --- /dev/null +++ b/tests/query/operators/test_json.py @@ -0,0 +1,52 @@ +from unittest import TestCase + +from piccolo.columns import JSONB +from piccolo.query.operators.json import GetChildElement, GetElementFromPath +from piccolo.table import Table +from tests.base import engines_skip + + +class RecordingStudio(Table): + facilities = JSONB(null=True) + + +@engines_skip("sqlite") +class TestGetChildElement(TestCase): + + def test_query(self): + """ + Make sure the generated SQL looks correct. + """ + querystring = GetChildElement( + GetChildElement(RecordingStudio.facilities, "a"), "b" + ) + + sql, query_args = querystring.compile_string() + + self.assertEqual( + sql, + '"recording_studio"."facilities" -> $1 -> $2', + ) + + self.assertListEqual(query_args, ["a", "b"]) + + +@engines_skip("sqlite") +class TestGetElementFromPath(TestCase): + + def test_query(self): + """ + Make sure the generated SQL looks correct. + """ + querystring = GetElementFromPath( + RecordingStudio.facilities, ["a", "b"] + ) + + sql, query_args = querystring.compile_string() + + self.assertEqual( + sql, + '"recording_studio"."facilities" #> $1', + ) + + self.assertListEqual(query_args, [["a", "b"]]) From db57d6e4f952ddd6e50d9fb10728ded4f4cdd4b3 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 23 Oct 2024 22:41:18 +0100 Subject: [PATCH 642/727] bumped version --- CHANGES.rst | 39 +++++++++++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 11dba5beb..6b952a99c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,45 @@ Changes ======= +1.22.0 +------ + +Python 3.13 is now officially supported. + +``JSON`` / ``JSONB`` querying has been significantly improved. For example, if +we have this table: + +.. code-block:: python + + class RecordingStudio(Table): + facilities = JSONB() + +And the ``facilities`` column contains the following JSON data: + +.. code-block:: python + + { + "technicians": [ + {"name": "Alice Jones"}, + {"name": "Bob Williams"}, + ] + } + +We can get the first technician name as follows: + +.. code-block:: python + + >>> await RecordingStudio.select( + ... RecordingStudio.facilities["technicians"][0]["name"].as_alias("name") + ... ).output(load_json=True) + [{'name': 'Alice Jones'}, ...] + +``TableStorage`` (used for dynamically creating Piccolo ``Table`` classes from +an existing database) was improved, to support a Dockerised version of Piccolo +Admin, which is coming soon. + +------------------------------------------------------------------------------- + 1.21.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index fa25b57e7..43503fd5a 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.21.0" +__VERSION__ = "1.22.0" From 448a818e3e7420e6f052b6e79473bbeee0b3e76f Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 23 Oct 2024 23:07:42 +0100 Subject: [PATCH 643/727] upgrade `actions/setup-python` (#1116) --- .github/workflows/release.yaml | 2 +- .github/workflows/tests.yaml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 4d70f80a5..0f14f4210 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -12,7 +12,7 @@ jobs: steps: - uses: "actions/checkout@v3" - - uses: "actions/setup-python@v1" + - uses: "actions/setup-python@v5" with: python-version: 3.13 - name: "Install dependencies" diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 0d71446a3..aab37d093 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -21,7 +21,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -57,7 +57,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -109,7 +109,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -146,7 +146,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -180,7 +180,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies From 7373a8490c84e3c1577d987a34dbc376474c2c30 Mon Sep 17 00:00:00 2001 From: David Flores Date: Wed, 6 Nov 2024 23:04:51 -0600 Subject: [PATCH 644/727] Update insert.rst (#1125) A missing comma was added --- docs/src/piccolo/query_types/insert.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/piccolo/query_types/insert.rst b/docs/src/piccolo/query_types/insert.rst index f8d1de007..6648d6d0b 100644 --- a/docs/src/piccolo/query_types/insert.rst +++ b/docs/src/piccolo/query_types/insert.rst @@ -8,7 +8,7 @@ This is used to bulk insert rows into the table: .. code-block:: python await Band.insert( - Band(name="Pythonistas") + Band(name="Pythonistas"), Band(name="Darts"), Band(name="Gophers") ) From 965c5b3ef2311a0ecb39a09341d5a6ebabc25b5d Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 7 Jan 2025 20:02:15 +0000 Subject: [PATCH 645/727] 1127 Fix bug with large integers in SQLite (#1128) * fix large integers in SQLite * make test sqlite only * fix integration tests (`httpx` removed `app` argument) * use `Decimal` --- piccolo/engine/sqlite.py | 16 ++++++++-- requirements/test-requirements.txt | 2 +- .../apps/asgi/commands/files/dummy_server.py | 4 +-- tests/columns/test_integer.py | 32 +++++++++++++++++++ 4 files changed, 49 insertions(+), 5 deletions(-) create mode 100644 tests/columns/test_integer.py diff --git a/piccolo/engine/sqlite.py b/piccolo/engine/sqlite.py index 3f7649d76..c8183e336 100644 --- a/piccolo/engine/sqlite.py +++ b/piccolo/engine/sqlite.py @@ -173,9 +173,21 @@ def convert_numeric_out(value: str) -> Decimal: @decode_to_string def convert_int_out(value: str) -> int: """ - Make sure Integer values are actually of type int. + Make sure INTEGER values are actually of type ``int``. + + SQLite doesn't enforce that the values in INTEGER columns are actually + integers - they could be strings ('hello'), or floats (1.0). + + There's not much we can do if the value is something like 'hello' - a + ``ValueError`` is appropriate in this situation. + + For a value like ``1.0``, it seems reasonable to handle this, and return a + value of ``1``. + """ - return int(float(value)) + # We used to use int(float(value)), but it was incorrect, because float has + # limited precision for large numbers. + return int(Decimal(value)) @decode_to_string diff --git a/requirements/test-requirements.txt b/requirements/test-requirements.txt index 006d8f2d9..6b8be1348 100644 --- a/requirements/test-requirements.txt +++ b/requirements/test-requirements.txt @@ -1,5 +1,5 @@ coveralls==3.3.1 -httpx==0.27.2 +httpx==0.28.0 pytest-cov==3.0.0 pytest==6.2.5 python-dateutil==2.8.2 diff --git a/tests/apps/asgi/commands/files/dummy_server.py b/tests/apps/asgi/commands/files/dummy_server.py index 9b83470a3..a4807aa66 100644 --- a/tests/apps/asgi/commands/files/dummy_server.py +++ b/tests/apps/asgi/commands/files/dummy_server.py @@ -3,7 +3,7 @@ import sys import typing as t -from httpx import AsyncClient +from httpx import ASGITransport, AsyncClient async def dummy_server(app: t.Union[str, t.Callable] = "app:app"): @@ -24,7 +24,7 @@ async def dummy_server(app: t.Union[str, t.Callable] = "app:app"): module = importlib.import_module(path) app = t.cast(t.Callable, getattr(module, app_name)) - async with AsyncClient(app=app) as client: + async with AsyncClient(transport=ASGITransport(app=app)) as client: response = await client.get("http://localhost:8000") if response.status_code != 200: sys.exit("The app isn't callable!") diff --git a/tests/columns/test_integer.py b/tests/columns/test_integer.py new file mode 100644 index 000000000..fe42aaf18 --- /dev/null +++ b/tests/columns/test_integer.py @@ -0,0 +1,32 @@ +from piccolo.columns.column_types import Integer +from piccolo.table import Table +from piccolo.testing.test_case import AsyncTableTest +from tests.base import sqlite_only + + +class MyTable(Table): + integer = Integer() + + +@sqlite_only +class TestInteger(AsyncTableTest): + tables = [MyTable] + + async def test_large_integer(self): + """ + Make sure large integers can be inserted and retrieved correctly. + + There was a bug with this in SQLite: + + https://github.com/piccolo-orm/piccolo/issues/1127 + + """ + integer = 625757527765811240 + + row = MyTable(integer=integer) + await row.save() + + _row = MyTable.objects().first().run_sync() + assert _row is not None + + self.assertEqual(_row.integer, integer) From 02ef5f5c7ae9c1c4c3f20cd588395886dd74499c Mon Sep 17 00:00:00 2001 From: sinisaos Date: Fri, 17 Jan 2025 16:11:28 +0100 Subject: [PATCH 646/727] add Quart asgi template (#1138) --- README.md | 2 +- piccolo/apps/asgi/commands/new.py | 1 + .../templates/app/_quart_app.py.jinja | 119 ++++++++++++++++++ .../asgi/commands/templates/app/app.py.jinja | 2 + .../app/home/_quart_endpoints.py.jinja | 18 +++ .../templates/app/home/endpoints.py.jinja | 2 + .../app/home/templates/home.html.jinja_raw | 5 + 7 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 piccolo/apps/asgi/commands/templates/app/_quart_app.py.jinja create mode 100644 piccolo/apps/asgi/commands/templates/app/home/_quart_endpoints.py.jinja diff --git a/README.md b/README.md index a1804d46a..139c2dbb0 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ Let Piccolo scaffold you an ASGI web app, using Piccolo as the ORM: piccolo asgi new ``` -[Starlette](https://www.starlette.io/), [FastAPI](https://fastapi.tiangolo.com/), [BlackSheep](https://www.neoteroi.dev/blacksheep/), [Litestar](https://litestar.dev/), [Esmerald](https://esmerald.dev/) and [Lilya](https://lilya.dev) are currently supported. +[Starlette](https://www.starlette.io/), [FastAPI](https://fastapi.tiangolo.com/), [BlackSheep](https://www.neoteroi.dev/blacksheep/), [Litestar](https://litestar.dev/), [Esmerald](https://esmerald.dev/), [Lilya](https://lilya.dev) and [Quart](https://quart.palletsprojects.com/en/latest/) are currently supported. ## Are you a Django user? diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index 85ceee021..19c762359 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -17,6 +17,7 @@ "litestar": ["litestar"], "esmerald": ["esmerald"], "lilya": ["lilya"], + "quart": ["quart", "quart_schema"], } ROUTERS = list(ROUTER_DEPENDENCIES.keys()) diff --git a/piccolo/apps/asgi/commands/templates/app/_quart_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_quart_app.py.jinja new file mode 100644 index 000000000..8de0228e8 --- /dev/null +++ b/piccolo/apps/asgi/commands/templates/app/_quart_app.py.jinja @@ -0,0 +1,119 @@ +import typing as t +from http import HTTPStatus + +from hypercorn.middleware import DispatcherMiddleware +from piccolo.engine import engine_finder +from piccolo_admin.endpoints import create_admin +from piccolo_api.crud.serializers import create_pydantic_model +from quart import Quart +from quart_schema import ( + Info, + QuartSchema, + hide, + tag, + validate_request, + validate_response, +) + +from home.endpoints import index +from home.piccolo_app import APP_CONFIG +from home.tables import Task + + +app = Quart(__name__, static_folder="static") +QuartSchema(app, info=Info(title="Quart API", version="0.1.0")) + + +TaskModelIn: t.Any = create_pydantic_model( + table=Task, + model_name="TaskModelIn", +) +TaskModelOut: t.Any = create_pydantic_model( + table=Task, + include_default_columns=True, + model_name="TaskModelOut", +) + + +@app.get("/") +@hide +def home(): + return index() + + +@app.get("/tasks/") +@validate_response(t.List[TaskModelOut]) +@tag(["Task"]) +async def tasks(): + return await Task.select().order_by(Task._meta.primary_key, ascending=False) + + +@app.post("/tasks/") +@validate_request(TaskModelIn) +@validate_response(TaskModelOut) +@tag(["Task"]) +async def create_task(data: TaskModelIn): + task = Task(**data.model_dump()) + await task.save() + return task.to_dict(), HTTPStatus.CREATED + + +@app.put("/tasks//") +@validate_request(TaskModelIn) +@validate_response(TaskModelOut) +@tag(["Task"]) +async def update_task(task_id: int, data: TaskModelIn): + task = await Task.objects().get(Task._meta.primary_key == task_id) + if not task: + return {}, HTTPStatus.NOT_FOUND + + for key, value in data.model_dump().items(): + setattr(task, key, value) + + await task.save() + + return task.to_dict(), HTTPStatus.OK + + +@app.delete("/tasks//") +@validate_response(TaskModelOut) +@tag(["Task"]) +async def delete_task(task_id: int): + task = await Task.objects().get(Task._meta.primary_key == task_id) + if not task: + return {}, HTTPStatus.NOT_FOUND + + await task.remove() + + return {}, HTTPStatus.OK + + +@app.before_serving +async def open_database_connection_pool(): + try: + engine = engine_finder() + await engine.start_connection_pool() + except Exception: + print("Unable to connect to the database") + + +@app.after_serving +async def close_database_connection_pool(): + try: + engine = engine_finder() + await engine.close_connection_pool() + except Exception: + print("Unable to connect to the database") + + +# enable the admin application using DispatcherMiddleware +app = DispatcherMiddleware( # type: ignore + { + "/admin": create_admin( + tables=APP_CONFIG.table_classes, + # Required when running under HTTPS: + # allowed_hosts=['my_site.com'] + ), + "": app, + } +) diff --git a/piccolo/apps/asgi/commands/templates/app/app.py.jinja b/piccolo/apps/asgi/commands/templates/app/app.py.jinja index 165510466..d30c08e28 100644 --- a/piccolo/apps/asgi/commands/templates/app/app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/app.py.jinja @@ -10,4 +10,6 @@ {% include '_esmerald_app.py.jinja' %} {% elif router == 'lilya' %} {% include '_lilya_app.py.jinja' %} +{% elif router == 'quart' %} + {% include '_quart_app.py.jinja' %} {% endif %} diff --git a/piccolo/apps/asgi/commands/templates/app/home/_quart_endpoints.py.jinja b/piccolo/apps/asgi/commands/templates/app/home/_quart_endpoints.py.jinja new file mode 100644 index 000000000..977ebd211 --- /dev/null +++ b/piccolo/apps/asgi/commands/templates/app/home/_quart_endpoints.py.jinja @@ -0,0 +1,18 @@ +import os + +import jinja2 + +from quart import Response + +ENVIRONMENT = jinja2.Environment( + loader=jinja2.FileSystemLoader( + searchpath=os.path.join(os.path.dirname(__file__), "templates") + ) +) + + +def index(): + template = ENVIRONMENT.get_template("home.html.jinja") + content = template.render(title="Piccolo + ASGI") + return Response(content) + \ No newline at end of file diff --git a/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja b/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja index 31503063c..cca56fde0 100644 --- a/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja @@ -8,4 +8,6 @@ {% include '_esmerald_endpoints.py.jinja' %} {% elif router == 'lilya' %} {% include '_lilya_endpoints.py.jinja' %} +{% elif router == 'quart' %} + {% include '_quart_endpoints.py.jinja' %} {% endif %} diff --git a/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw b/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw index 5d48d767b..502224e92 100644 --- a/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw +++ b/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw @@ -66,6 +66,11 @@
  • Admin
  • JSON endpoint
    • +

    Quart

    + {% endblock content %} From 37b063e958b62c3f706281acbd62414ac0557ac8 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 24 Jan 2025 13:51:35 +0000 Subject: [PATCH 647/727] fix constraint typos (#1145) --- CHANGES.rst | 2 +- tests/table/test_insert.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 6b952a99c..2a88f740c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3737,7 +3737,7 @@ metaclass not being explicit enough when checking falsy values. which prevents circular import issues. * Faster column copying, which is important when specifying joins, e.g. ``await Band.select(Band.manager.name).run()``. -* Fixed a bug with migrations and foreign key contraints. +* Fixed a bug with migrations and foreign key constraints. ------------------------------------------------------------------------------- diff --git a/tests/table/test_insert.py b/tests/table/test_insert.py index b2c58e378..72a1de512 100644 --- a/tests/table/test_insert.py +++ b/tests/table/test_insert.py @@ -201,7 +201,7 @@ def test_target_tuple(self): Make sure that a composite unique constraint can be used as a target. We only run it on Postgres and Cockroach because we use ALTER TABLE - to add a contraint, which SQLite doesn't support. + to add a constraint, which SQLite doesn't support. """ Band = self.Band From fc7e44d91a7ee0de6657b8bd96dd809e1bec4786 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 26 Jan 2025 12:45:55 +0000 Subject: [PATCH 648/727] use python 3.13 in integration tests (#1147) --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index aab37d093..e23be030f 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -40,7 +40,7 @@ jobs: matrix: # These tests are slow, so we only run on the latest Python # version. - python-version: ["3.12"] + python-version: ["3.13"] postgres-version: [17] services: postgres: From 55c2dbcf4b7aaceaf818bc566d7feea7862187c5 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Sun, 26 Jan 2025 14:59:18 +0100 Subject: [PATCH 649/727] add Falcon asgi template (#1139) undefined --- README.md | 2 +- piccolo/apps/asgi/commands/new.py | 1 + .../templates/app/_falcon_app.py.jinja | 60 +++++++++++++++++++ .../asgi/commands/templates/app/app.py.jinja | 2 + .../app/home/_falcon_endpoints.py.jinja | 19 ++++++ .../templates/app/home/endpoints.py.jinja | 2 + .../app/home/templates/home.html.jinja_raw | 5 ++ 7 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 piccolo/apps/asgi/commands/templates/app/_falcon_app.py.jinja create mode 100644 piccolo/apps/asgi/commands/templates/app/home/_falcon_endpoints.py.jinja diff --git a/README.md b/README.md index 139c2dbb0..53d8c1038 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ Let Piccolo scaffold you an ASGI web app, using Piccolo as the ORM: piccolo asgi new ``` -[Starlette](https://www.starlette.io/), [FastAPI](https://fastapi.tiangolo.com/), [BlackSheep](https://www.neoteroi.dev/blacksheep/), [Litestar](https://litestar.dev/), [Esmerald](https://esmerald.dev/), [Lilya](https://lilya.dev) and [Quart](https://quart.palletsprojects.com/en/latest/) are currently supported. +[Starlette](https://www.starlette.io/), [FastAPI](https://fastapi.tiangolo.com/), [BlackSheep](https://www.neoteroi.dev/blacksheep/), [Litestar](https://litestar.dev/), [Esmerald](https://esmerald.dev/), [Lilya](https://lilya.dev), [Quart](https://quart.palletsprojects.com/en/latest/) and [Falcon](https://falconframework.org/) are currently supported. ## Are you a Django user? diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index 19c762359..1cdb75fba 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -18,6 +18,7 @@ "esmerald": ["esmerald"], "lilya": ["lilya"], "quart": ["quart", "quart_schema"], + "falcon": ["falcon"], } ROUTERS = list(ROUTER_DEPENDENCIES.keys()) diff --git a/piccolo/apps/asgi/commands/templates/app/_falcon_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_falcon_app.py.jinja new file mode 100644 index 000000000..0827debb4 --- /dev/null +++ b/piccolo/apps/asgi/commands/templates/app/_falcon_app.py.jinja @@ -0,0 +1,60 @@ +import os +import typing as t + +import falcon.asgi +from hypercorn.middleware import DispatcherMiddleware +from piccolo.engine import engine_finder +from piccolo_admin.endpoints import create_admin +from piccolo_api.crud.endpoints import PiccoloCRUD + +from home.endpoints import HomeEndpoint +from home.piccolo_app import APP_CONFIG +from home.tables import Task + + +async def open_database_connection_pool(): + try: + engine = engine_finder() + await engine.start_connection_pool() + except Exception: + print("Unable to connect to the database") + + +async def close_database_connection_pool(): + try: + engine = engine_finder() + await engine.close_connection_pool() + except Exception: + print("Unable to connect to the database") + + +class LifespanMiddleware: + async def process_startup( + self, scope: t.Dict[str, t.Any], event: t.Dict[str, t.Any] + ) -> None: + await open_database_connection_pool() + + async def process_shutdown( + self, scope: t.Dict[str, t.Any], event: t.Dict[str, t.Any] + ) -> None: + await close_database_connection_pool() + + +app: t.Any = falcon.asgi.App(middleware=LifespanMiddleware()) +app.add_static_route("/static", directory=os.path.abspath("static")) +app.add_route("/", HomeEndpoint()) + +PICCOLO_CRUD: t.Any = PiccoloCRUD(table=Task) + +# enable the Admin and PiccoloCrud app using DispatcherMiddleware +app = DispatcherMiddleware( # type: ignore + { + "/admin": create_admin( + tables=APP_CONFIG.table_classes, + # Required when running under HTTPS: + # allowed_hosts=['my_site.com'] + ), + "/tasks": PICCOLO_CRUD, + "": app, + } +) diff --git a/piccolo/apps/asgi/commands/templates/app/app.py.jinja b/piccolo/apps/asgi/commands/templates/app/app.py.jinja index d30c08e28..4b056d679 100644 --- a/piccolo/apps/asgi/commands/templates/app/app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/app.py.jinja @@ -12,4 +12,6 @@ {% include '_lilya_app.py.jinja' %} {% elif router == 'quart' %} {% include '_quart_app.py.jinja' %} +{% elif router == 'falcon' %} + {% include '_falcon_app.py.jinja' %} {% endif %} diff --git a/piccolo/apps/asgi/commands/templates/app/home/_falcon_endpoints.py.jinja b/piccolo/apps/asgi/commands/templates/app/home/_falcon_endpoints.py.jinja new file mode 100644 index 000000000..5fcbde41e --- /dev/null +++ b/piccolo/apps/asgi/commands/templates/app/home/_falcon_endpoints.py.jinja @@ -0,0 +1,19 @@ +import os + +import falcon +import jinja2 + +ENVIRONMENT = jinja2.Environment( + loader=jinja2.FileSystemLoader( + searchpath=os.path.join(os.path.dirname(__file__), "templates") + ) +) + + +class HomeEndpoint: + async def on_get(self, req, resp): + template = ENVIRONMENT.get_template("home.html.jinja") + content = template.render(title="Piccolo + ASGI",) + resp.status = falcon.HTTP_200 + resp.content_type = "text/html" + resp.text = content diff --git a/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja b/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja index cca56fde0..733808964 100644 --- a/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja @@ -10,4 +10,6 @@ {% include '_lilya_endpoints.py.jinja' %} {% elif router == 'quart' %} {% include '_quart_endpoints.py.jinja' %} +{% elif router == 'falcon' %} + {% include '_falcon_endpoints.py.jinja' %} {% endif %} diff --git a/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw b/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw index 502224e92..c1ae75276 100644 --- a/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw +++ b/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw @@ -71,6 +71,11 @@
  • Admin
  • Swagger API
    • +

    Falcon

    + {% endblock content %} From 2086772595f30d54f1cbef378c9b230eb14c393a Mon Sep 17 00:00:00 2001 From: Ethan <47520067+Skelmis@users.noreply.github.com> Date: Wed, 12 Feb 2025 09:11:43 +1300 Subject: [PATCH 650/727] feat: add Callable typing to Timestampz default types (#1149) --- piccolo/columns/defaults/timestamptz.py | 1 + 1 file changed, 1 insertion(+) diff --git a/piccolo/columns/defaults/timestamptz.py b/piccolo/columns/defaults/timestamptz.py index 5db6ebd54..82864b593 100644 --- a/piccolo/columns/defaults/timestamptz.py +++ b/piccolo/columns/defaults/timestamptz.py @@ -73,6 +73,7 @@ def from_datetime(cls, instance: datetime.datetime): # type: ignore Enum, None, datetime.datetime, + t.Callable[[], datetime.datetime], ] From 308ea73245dd66cec1fa564cc2ffe325f4076775 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Wed, 12 Feb 2025 00:34:42 +0100 Subject: [PATCH 651/727] add Sanic asgi template (#1140) * add Sanic asgi template * remove password from tests piccolo_conf * change dummy_server to use uvicorn only for the Sanic integration test * remove password from tests piccolo_conf * update pyproject.toml --- README.md | 2 +- piccolo/apps/asgi/commands/new.py | 1 + .../templates/app/_sanic_app.py.jinja | 121 ++++++++++++++++++ .../asgi/commands/templates/app/app.py.jinja | 4 +- .../app/home/_sanic_endpoints.py.jinja | 17 +++ .../templates/app/home/endpoints.py.jinja | 2 + .../app/home/templates/home.html.jinja_raw | 5 + pyproject.toml | 3 +- .../apps/asgi/commands/files/dummy_server.py | 17 ++- 9 files changed, 164 insertions(+), 8 deletions(-) create mode 100644 piccolo/apps/asgi/commands/templates/app/_sanic_app.py.jinja create mode 100644 piccolo/apps/asgi/commands/templates/app/home/_sanic_endpoints.py.jinja diff --git a/README.md b/README.md index 53d8c1038..9b8d1e173 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ Let Piccolo scaffold you an ASGI web app, using Piccolo as the ORM: piccolo asgi new ``` -[Starlette](https://www.starlette.io/), [FastAPI](https://fastapi.tiangolo.com/), [BlackSheep](https://www.neoteroi.dev/blacksheep/), [Litestar](https://litestar.dev/), [Esmerald](https://esmerald.dev/), [Lilya](https://lilya.dev), [Quart](https://quart.palletsprojects.com/en/latest/) and [Falcon](https://falconframework.org/) are currently supported. +[Starlette](https://www.starlette.io/), [FastAPI](https://fastapi.tiangolo.com/), [BlackSheep](https://www.neoteroi.dev/blacksheep/), [Litestar](https://litestar.dev/), [Esmerald](https://esmerald.dev/), [Lilya](https://lilya.dev), [Quart](https://quart.palletsprojects.com/en/latest/), [Falcon](https://falconframework.org/) and [Sanic](https://sanic.dev/en/) are currently supported. ## Are you a Django user? diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index 1cdb75fba..dc3c75fb4 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -19,6 +19,7 @@ "lilya": ["lilya"], "quart": ["quart", "quart_schema"], "falcon": ["falcon"], + "sanic": ["sanic", "sanic_ext"], } ROUTERS = list(ROUTER_DEPENDENCIES.keys()) diff --git a/piccolo/apps/asgi/commands/templates/app/_sanic_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_sanic_app.py.jinja new file mode 100644 index 000000000..d76a13d5e --- /dev/null +++ b/piccolo/apps/asgi/commands/templates/app/_sanic_app.py.jinja @@ -0,0 +1,121 @@ +import asyncio +import typing as t + +from hypercorn.middleware import DispatcherMiddleware +from piccolo.engine import engine_finder +from piccolo_admin.endpoints import create_admin +from piccolo_api.crud.serializers import create_pydantic_model +from sanic import Request, Sanic, json +from sanic_ext import openapi + +from home.endpoints import index +from home.piccolo_app import APP_CONFIG +from home.tables import Task + +app = Sanic(__name__) +app.static("/static/", "static") + + +TaskModelIn: t.Any = create_pydantic_model( + table=Task, + model_name="TaskModelIn", +) +TaskModelOut: t.Any = create_pydantic_model( + table=Task, + include_default_columns=True, + model_name="TaskModelOut", +) + + +@app.get("/") +@openapi.exclude() +def home(request: Request): + return index() + + +@app.get("/tasks/") +@openapi.tag("Task") +@openapi.response(200, {"application/json": TaskModelOut.model_json_schema()}) +async def tasks(request: Request): + return json( + await Task.select().order_by(Task._meta.primary_key, ascending=False), + status=200, + ) + + +@app.post("/tasks/") +@openapi.definition( + body={"application/json": TaskModelIn.model_json_schema()}, + tag="Task", +) +@openapi.response(201, {"application/json": TaskModelOut.model_json_schema()}) +async def create_task(request: Request): + task = Task(**request.json) + await task.save() + return json(task.to_dict(), status=201) + + +@app.put("/tasks//") +@openapi.definition( + body={"application/json": TaskModelIn.model_json_schema()}, + tag="Task", +) +@openapi.response(200, {"application/json": TaskModelOut.model_json_schema()}) +async def update_task(request: Request, task_id: int): + task = await Task.objects().get(Task._meta.primary_key == task_id) + if not task: + return json({}, status=404) + for key, value in request.json.items(): + setattr(task, key, value) + + await task.save() + return json(task.to_dict(), status=200) + + +@app.delete("/tasks//") +@openapi.tag("Task") +async def delete_task(request: Request, task_id: int): + task = await Task.objects().get(Task._meta.primary_key == task_id) + if not task: + return json({}, status=404) + await task.remove() + return json({}, status=200) + + +async def open_database_connection_pool(): + try: + engine = engine_finder() + await engine.start_connection_pool() + except Exception: + print("Unable to connect to the database") + + +async def close_database_connection_pool(): + try: + engine = engine_finder() + await engine.close_connection_pool() + except Exception: + print("Unable to connect to the database") + + +@app.after_server_start +async def startup(app, loop): + await open_database_connection_pool() + + +@app.before_server_stop +async def shutdown(app, loop): + await close_database_connection_pool() + + +# enable the admin application using DispatcherMiddleware +app = DispatcherMiddleware( # type: ignore + { + "/admin": create_admin( + tables=APP_CONFIG.table_classes, + # Required when running under HTTPS: + # allowed_hosts=['my_site.com'] + ), + "": app, + } +) diff --git a/piccolo/apps/asgi/commands/templates/app/app.py.jinja b/piccolo/apps/asgi/commands/templates/app/app.py.jinja index 4b056d679..a4bf71ee2 100644 --- a/piccolo/apps/asgi/commands/templates/app/app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/app.py.jinja @@ -13,5 +13,7 @@ {% elif router == 'quart' %} {% include '_quart_app.py.jinja' %} {% elif router == 'falcon' %} - {% include '_falcon_app.py.jinja' %} + {% include '_falcon_app.py.jinja' %} +{% elif router == 'sanic' %} + {% include '_sanic_app.py.jinja' %} {% endif %} diff --git a/piccolo/apps/asgi/commands/templates/app/home/_sanic_endpoints.py.jinja b/piccolo/apps/asgi/commands/templates/app/home/_sanic_endpoints.py.jinja new file mode 100644 index 000000000..e6a5f3416 --- /dev/null +++ b/piccolo/apps/asgi/commands/templates/app/home/_sanic_endpoints.py.jinja @@ -0,0 +1,17 @@ +import os + +import jinja2 + +from sanic import HTTPResponse + +ENVIRONMENT = jinja2.Environment( + loader=jinja2.FileSystemLoader( + searchpath=os.path.join(os.path.dirname(__file__), "templates") + ) +) + + +def index(): + template = ENVIRONMENT.get_template("home.html.jinja") + content = template.render(title="Piccolo + ASGI") + return HTTPResponse(content) \ No newline at end of file diff --git a/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja b/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja index 733808964..4f0023134 100644 --- a/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja @@ -12,4 +12,6 @@ {% include '_quart_endpoints.py.jinja' %} {% elif router == 'falcon' %} {% include '_falcon_endpoints.py.jinja' %} +{% elif router == 'sanic' %} + {% include '_sanic_endpoints.py.jinja' %} {% endif %} diff --git a/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw b/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw index c1ae75276..47cb9a039 100644 --- a/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw +++ b/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw @@ -76,6 +76,11 @@
  • Admin
  • JSON endpoint
    • +

    Sanic

    + {% endblock content %} diff --git a/pyproject.toml b/pyproject.toml index eed50963a..0c94768d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,8 @@ module = [ "IPython.core.interactiveshell", "jinja2", "orjson", - "aiosqlite" + "aiosqlite", + "uvicorn" ] ignore_missing_imports = true diff --git a/tests/apps/asgi/commands/files/dummy_server.py b/tests/apps/asgi/commands/files/dummy_server.py index a4807aa66..307fe1c80 100644 --- a/tests/apps/asgi/commands/files/dummy_server.py +++ b/tests/apps/asgi/commands/files/dummy_server.py @@ -4,9 +4,10 @@ import typing as t from httpx import ASGITransport, AsyncClient +from uvicorn import Config, Server -async def dummy_server(app: t.Union[str, t.Callable] = "app:app"): +async def dummy_server(app: t.Union[str, t.Callable] = "app:app") -> None: """ A very simplistic ASGI server. It's used to run the generated ASGI applications in unit tests. @@ -24,10 +25,16 @@ async def dummy_server(app: t.Union[str, t.Callable] = "app:app"): module = importlib.import_module(path) app = t.cast(t.Callable, getattr(module, app_name)) - async with AsyncClient(transport=ASGITransport(app=app)) as client: - response = await client.get("http://localhost:8000") - if response.status_code != 200: - sys.exit("The app isn't callable!") + try: + async with AsyncClient(transport=ASGITransport(app=app)) as client: + response = await client.get("http://localhost:8000") + if response.status_code != 200: + sys.exit("The app isn't callable!") + except Exception: + config = Config(app=app) + server = Server(config=config) + asyncio.create_task(server.serve()) + await asyncio.sleep(0.1) if __name__ == "__main__": From 40a31b2483678c9b2460c20d5fb9986a5f6ac994 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Wed, 12 Feb 2025 12:22:31 +0100 Subject: [PATCH 652/727] update asgi frameworks and servers in docs (#1150) --- docs/src/piccolo/asgi/index.rst | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/src/piccolo/asgi/index.rst b/docs/src/piccolo/asgi/index.rst index a3d57899e..a0756ca6f 100644 --- a/docs/src/piccolo/asgi/index.rst +++ b/docs/src/piccolo/asgi/index.rst @@ -20,21 +20,19 @@ will ask for your preferences on which libraries to use. Routing frameworks ****************** -Currently, `Starlette `_, `FastAPI `_, +`Starlette `_, `FastAPI `_, `BlackSheep `_, -`Litestar `_, `Esmerald `_ and -`Lilya `_ are supported. - -Other great ASGI routing frameworks exist, and may be supported in the future -(`Quart `_ , -`Sanic `_ , -`Django `_ etc). +`Litestar `_, `Esmerald `_, +`Lilya `_, +`Quart `_, +`Falcon `_ +and `Sanic `_ are supported. Which to use? ============= All are great choices. FastAPI is built on top of Starlette and Esmerald is built on top of Lilya, so they're -very similar. FastAPI, BlackSheep and Esmerald are great if you want to document a REST +very similar. FastAPI, BlackSheep, Litestar and Esmerald are great if you want to document a REST API, as they have built-in OpenAPI support. ------------------------------------------------------------------------------- @@ -42,7 +40,9 @@ API, as they have built-in OpenAPI support. Web servers ************ -`Hypercorn `_ and -`Uvicorn `_ are available as ASGI servers. +`Uvicorn `_, +`Hypercorn `_ +and `Granian `_ +are available as ASGI servers. `Daphne `_ can't be used programatically so was omitted at this time. From 8fd69e710bc8a4e577c10e57d0a55058805c4acd Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 12 Feb 2025 23:53:53 +0000 Subject: [PATCH 653/727] bumped version --- CHANGES.rst | 11 +++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2a88f740c..78e5c8fad 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,17 @@ Changes ======= +1.23.0 +------ + +* Added Quart, Sanic, and Falcon as supported ASGI frameworks (thanks to + @sinisaos for this). +* Fixed a bug with very large integers in SQLite. +* Fixed type annotation for ``Timestamptz`` default values (thanks to @Skelmis + for this). + +------------------------------------------------------------------------------- + 1.22.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 43503fd5a..5edc72b35 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.22.0" +__VERSION__ = "1.23.0" From 2a637b870bf14857d1579dcd87e8a4bf31aa2b61 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 13 Mar 2025 12:56:55 +0000 Subject: [PATCH 654/727] 1162 Add DSN support to `sql_shell` (#1163) * allow dsn into sql_shell * improve tests * reduce repetition --- piccolo/apps/sql_shell/commands/run.py | 27 +++++++++++----------- tests/apps/sql_shell/commands/test_run.py | 28 +++++++++++++++++++++-- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/piccolo/apps/sql_shell/commands/run.py b/piccolo/apps/sql_shell/commands/run.py index 7de03dfcd..8286b7cce 100644 --- a/piccolo/apps/sql_shell/commands/run.py +++ b/piccolo/apps/sql_shell/commands/run.py @@ -28,24 +28,23 @@ def run() -> None: args = ["psql"] - host = engine.config.get("host") - port = engine.config.get("port") - user = engine.config.get("user") - password = engine.config.get("password") - database = engine.config.get("database") + config = engine.config - if user: - args += ["-U", user] - if host: - args += ["-h", host] - if port: - args += ["-p", str(port)] - if database: - args += [database] + if dsn := config.get("dsn"): + args += [dsn] + else: + if user := config.get("user"): + args += ["-U", user] + if host := config.get("host"): + args += ["-h", host] + if port := config.get("port"): + args += ["-p", str(port)] + if database := config.get("database"): + args += [database] sigint_handler = signal.getsignal(signal.SIGINT) subprocess_env = os.environ.copy() - if password: + if password := config.get("password"): subprocess_env["PGPASSWORD"] = str(password) try: # Allow SIGINT to pass to psql to abort queries. diff --git a/tests/apps/sql_shell/commands/test_run.py b/tests/apps/sql_shell/commands/test_run.py index be4790ced..8d0c5689c 100644 --- a/tests/apps/sql_shell/commands/test_run.py +++ b/tests/apps/sql_shell/commands/test_run.py @@ -2,13 +2,37 @@ from unittest.mock import MagicMock, patch from piccolo.apps.sql_shell.commands.run import run +from tests.base import postgres_only, sqlite_only class TestRun(TestCase): + @postgres_only @patch("piccolo.apps.sql_shell.commands.run.subprocess") - def test_run(self, subprocess: MagicMock): + def test_psql(self, subprocess: MagicMock): """ - A simple test to make sure it executes without raising any exceptions. + Make sure psql was called correctly. """ run() self.assertTrue(subprocess.run.called) + + assert subprocess.run.call_args.args[0] == [ + "psql", + "-U", + "postgres", + "-h", + "localhost", + "-p", + "5432", + "piccolo", + ] + + @sqlite_only + @patch("piccolo.apps.sql_shell.commands.run.subprocess") + def test_sqlite3(self, subprocess: MagicMock): + """ + Make sure sqlite3 was called correctly. + """ + run() + self.assertTrue(subprocess.run.called) + + assert subprocess.run.call_args.args[0] == ["sqlite3", "test.sqlite"] From 0cdb6dc662e5e92bc90ef27169e5cff6c2f668a6 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 14 Mar 2025 15:11:59 +0000 Subject: [PATCH 655/727] 1152 `get_or_create` raises `ValueError` (#1153) * set `_ignore_missing=True` * populate data dict, instead of setting attributes afterwards * add a test * add name to test table --- piccolo/query/methods/objects.py | 12 +++++------- tests/table/test_objects.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index f351f1319..a870a14e3 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -65,22 +65,20 @@ async def run( instance._was_created = False return instance - instance = self.table_class(_data=self.defaults) + data = {**self.defaults} # If it's a complex `where`, there can be several column values to # extract e.g. (Band.name == 'Pythonistas') & (Band.popularity == 1000) if isinstance(self.where, Where): - setattr( - instance, - self.where.column._meta.name, # type: ignore - self.where.value, # type: ignore - ) + data[self.where.column] = self.where.value elif isinstance(self.where, And): for column, value in self.where.get_column_values().items(): if len(column._meta.call_chain) == 0: # Make sure we only set the value if the column belongs # to this table. - setattr(instance, column._meta.name, value) + data[column] = value + + instance = self.table_class(_data=data) await instance.save().run(node=node, in_pool=in_pool) diff --git a/tests/table/test_objects.py b/tests/table/test_objects.py index e2db53ba9..9f065eecc 100644 --- a/tests/table/test_objects.py +++ b/tests/table/test_objects.py @@ -1,3 +1,5 @@ +from piccolo.columns.column_types import ForeignKey +from piccolo.testing.test_case import AsyncTableTest from tests.base import DBTestCase, engines_only, sqlite_only from tests.example_apps.music.tables import Band, Manager @@ -268,3 +270,32 @@ def test_prefetch_new_object(self): self.assertIsInstance(band.manager, Manager) self.assertEqual(band.name, "New Band 2") self.assertEqual(band.manager.name, "Guido") + + +class BandNotNull(Band, tablename="band"): + manager = ForeignKey(Manager, null=False) + + +class TestGetOrCreateNotNull(AsyncTableTest): + + tables = [BandNotNull, Manager] + + async def test_not_null(self): + """ + There was a bug where `get_or_create` would fail for columns with + `default=None` and `null=False`, even if the value for those columns + was specified in the where clause. + + https://github.com/piccolo-orm/piccolo/issues/1152 + + """ + + manager = Manager({Manager.name: "Test"}) + await manager.save() + + self.assertIsInstance( + await BandNotNull.objects().get_or_create( + BandNotNull.manager == manager + ), + BandNotNull, + ) From 0aa0d90dd33216c18296ea72141e3b4d742f6772 Mon Sep 17 00:00:00 2001 From: Compro Prasad Date: Fri, 14 Mar 2025 23:23:08 +0530 Subject: [PATCH 656/727] Fixes #1164 - Add support for None typehint in Varchar length attribute (#1165) * Add support for None typehint for length option in Varchar * fix linter warning --------- Co-authored-by: Daniel Townsend --- piccolo/apps/user/tables.py | 4 +++- piccolo/columns/column_types.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/piccolo/apps/user/tables.py b/piccolo/apps/user/tables.py index 1d61dfc2b..18f4915fa 100644 --- a/piccolo/apps/user/tables.py +++ b/piccolo/apps/user/tables.py @@ -202,7 +202,9 @@ async def login(cls, username: str, password: str) -> t.Optional[int]: The id of the user if a match is found, otherwise ``None``. """ - if len(username) > cls.username.length: + if (max_username_length := cls.username.length) and len( + username + ) > max_username_length: logger.warning("Excessively long username provided.") return None diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 5d0b04acc..e80fb254a 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -315,7 +315,7 @@ class Band(Table): def __init__( self, - length: int = 255, + length: t.Optional[int] = 255, default: t.Union[str, Enum, t.Callable[[], str], None] = "", **kwargs, ) -> None: From d7063bdc34ed1170f6a6cd5a3ea802732689dea6 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 14 Mar 2025 22:55:59 +0000 Subject: [PATCH 657/727] bumped version --- CHANGES.rst | 14 ++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 78e5c8fad..8cd926da3 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,20 @@ Changes ======= +1.24.0 +------ + +* Fixed a bug with ``get_or_create`` when a table has a column with both + ``null=False`` and ``default=None`` - thanks to @bymoye for reporting this + issue. +* If a ``PostgresEngine`` uses the ``dsn`` argument for ``asyncpg``, this is + now used by ``piccolo sql_shell run``. Thanks to @abhishek-compro for + suggesting this. +* Fixed the type annotation for the ``length`` argument of ``Varchar`` - it + is allowed to be ``None``. Thanks to @Compro-Prasad for this. + +------------------------------------------------------------------------------- + 1.23.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 5edc72b35..c2c9193d1 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.23.0" +__VERSION__ = "1.24.0" From 319faeaf008cf7eb81368bf2ad4e88db05249b6e Mon Sep 17 00:00:00 2001 From: Spencer Churchill Date: Fri, 21 Mar 2025 15:40:22 -0700 Subject: [PATCH 658/727] Fix `day` and `minute` in `TimestamptzCustom`'s `from_datetime` (#1168) * fix day set as month * add minute to datetime * add minute to from_datetime * update `TimestampCustom` too * add a test --------- Co-authored-by: Daniel Townsend --- piccolo/columns/defaults/timestamp.py | 6 ++++- piccolo/columns/defaults/timestamptz.py | 4 ++- tests/columns/test_defaults.py | 34 +++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/piccolo/columns/defaults/timestamp.py b/piccolo/columns/defaults/timestamp.py index 73e06d1ef..8fb984f77 100644 --- a/piccolo/columns/defaults/timestamp.py +++ b/piccolo/columns/defaults/timestamp.py @@ -70,6 +70,7 @@ def __init__( month: int = 1, day: int = 1, hour: int = 0, + minute: int = 0, second: int = 0, microsecond: int = 0, ): @@ -77,6 +78,7 @@ def __init__( self.month = month self.day = day self.hour = hour + self.minute = minute self.second = second self.microsecond = microsecond @@ -87,6 +89,7 @@ def datetime(self): month=self.month, day=self.day, hour=self.hour, + minute=self.minute, second=self.second, microsecond=self.microsecond, ) @@ -113,8 +116,9 @@ def from_datetime(cls, instance: datetime.datetime): # type: ignore return cls( year=instance.year, month=instance.month, - day=instance.month, + day=instance.day, hour=instance.hour, + minute=instance.minute, second=instance.second, microsecond=instance.microsecond, ) diff --git a/piccolo/columns/defaults/timestamptz.py b/piccolo/columns/defaults/timestamptz.py index 82864b593..f0e792973 100644 --- a/piccolo/columns/defaults/timestamptz.py +++ b/piccolo/columns/defaults/timestamptz.py @@ -47,6 +47,7 @@ def datetime(self): month=self.month, day=self.day, hour=self.hour, + minute=self.minute, second=self.second, microsecond=self.microsecond, tzinfo=datetime.timezone.utc, @@ -59,8 +60,9 @@ def from_datetime(cls, instance: datetime.datetime): # type: ignore return cls( year=instance.year, month=instance.month, - day=instance.month, + day=instance.day, hour=instance.hour, + minute=instance.minute, second=instance.second, microsecond=instance.microsecond, ) diff --git a/tests/columns/test_defaults.py b/tests/columns/test_defaults.py index 77df731bf..dbe9ee522 100644 --- a/tests/columns/test_defaults.py +++ b/tests/columns/test_defaults.py @@ -22,6 +22,8 @@ TimestampNow, Varchar, ) +from piccolo.columns.defaults.timestamp import TimestampCustom +from piccolo.columns.defaults.timestamptz import TimestamptzCustom from piccolo.table import Table @@ -98,3 +100,35 @@ class MyTable(Table): ForeignKey(references=MyTable, default=1) with self.assertRaises(ValueError): ForeignKey(references=MyTable, default="hello world") + + +class TestDatetime(TestCase): + + def test_datetime(self): + """ + Make sure we can create a `TimestampCustom` / `TimestamptzCustom` from + a datetime, and then convert it back into the same datetime again. + + https://github.com/piccolo-orm/piccolo/issues/1169 + + """ + datetime_obj = datetime.datetime( + year=2025, + month=1, + day=30, + hour=12, + minute=10, + second=15, + microsecond=100, + ) + + self.assertEqual( + TimestampCustom.from_datetime(datetime_obj).datetime, + datetime_obj, + ) + + datetime_obj = datetime_obj.astimezone(tz=datetime.timezone.utc) + self.assertEqual( + TimestamptzCustom.from_datetime(datetime_obj).datetime, + datetime_obj, + ) From f4a23a3590979c178671e0c648e21a7be682df3e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 21 Mar 2025 22:43:08 +0000 Subject: [PATCH 659/727] bumped version --- CHANGES.rst | 8 ++++++++ piccolo/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 8cd926da3..4b2dc0894 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,14 @@ Changes ======= +1.24.1 +------ + +Fixed a bug with default values in ``Timestamp`` and ``Timestamptz`` columns. +Thanks to @splch for this. + +------------------------------------------------------------------------------- + 1.24.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index c2c9193d1..88a38dfbf 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.24.0" +__VERSION__ = "1.24.1" From a979b68c36d44e940c78322ab96c3320e5bd1d50 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Thu, 3 Apr 2025 23:44:21 +0200 Subject: [PATCH 660/727] 1171 Delete subquery (#1173) * enable delete queries with FK columns in where clause * rename querystring_for_update to querystring_for_joins * name method `querystring_for_update_and_delete` * tweak test --------- Co-authored-by: Daniel Townsend --- piccolo/columns/combination.py | 10 +++++----- piccolo/query/methods/delete.py | 2 +- piccolo/query/methods/update.py | 2 +- tests/table/test_delete.py | 15 +++++++++++++++ 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/piccolo/columns/combination.py b/piccolo/columns/combination.py index e080cced2..ede58f09b 100644 --- a/piccolo/columns/combination.py +++ b/piccolo/columns/combination.py @@ -45,11 +45,11 @@ def querystring(self) -> QueryString: ) @property - def querystring_for_update(self) -> QueryString: + def querystring_for_update_and_delete(self) -> QueryString: return QueryString( "({} " + self.operator + " {})", - self.first.querystring_for_update, - self.second.querystring_for_update, + self.first.querystring_for_update_and_delete, + self.second.querystring_for_update_and_delete, ) def __str__(self): @@ -131,7 +131,7 @@ def __init__(self, sql: str, *args: t.Any) -> None: self.querystring = QueryString(sql, *args) @property - def querystring_for_update(self) -> QueryString: + def querystring_for_update_and_delete(self) -> QueryString: return self.querystring def __str__(self): @@ -218,7 +218,7 @@ def querystring(self) -> QueryString: return QueryString(template, *args) @property - def querystring_for_update(self) -> QueryString: + def querystring_for_update_and_delete(self) -> QueryString: args: t.List[t.Any] = [] if self.value != UNDEFINED: args.append(self.value) diff --git a/piccolo/query/methods/delete.py b/piccolo/query/methods/delete.py index 628b89b8e..5570ddde9 100644 --- a/piccolo/query/methods/delete.py +++ b/piccolo/query/methods/delete.py @@ -61,7 +61,7 @@ def default_querystrings(self) -> t.Sequence[QueryString]: querystring = QueryString( "{} WHERE {}", querystring, - self.where_delegate._where.querystring, + self.where_delegate._where.querystring_for_update_and_delete, ) if self.returning_delegate._returning: diff --git a/piccolo/query/methods/update.py b/piccolo/query/methods/update.py index f75854c43..5cd2e5073 100644 --- a/piccolo/query/methods/update.py +++ b/piccolo/query/methods/update.py @@ -104,7 +104,7 @@ def default_querystrings(self) -> t.Sequence[QueryString]: querystring = QueryString( "{} WHERE {}", querystring, - self.where_delegate._where.querystring_for_update, + self.where_delegate._where.querystring_for_update_and_delete, ) if self.returning_delegate._returning: diff --git a/tests/table/test_delete.py b/tests/table/test_delete.py index 0fb3df112..218acd458 100644 --- a/tests/table/test_delete.py +++ b/tests/table/test_delete.py @@ -44,3 +44,18 @@ def test_validation(self): Band.delete().run_sync() Band.delete(force=True).run_sync() + + def test_delete_with_joins(self): + """ + Make sure delete works if the `where` clause specifies joins. + """ + + self.insert_rows() + + Band.delete().where(Band.manager._.name == "Guido").run_sync() + + response = ( + Band.count().where(Band.manager._.name == "Guido").run_sync() + ) + + self.assertEqual(response, 0) From dd466ae469165c2f1c5eac1f1033cbbef1a1d255 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 3 Apr 2025 23:03:54 +0100 Subject: [PATCH 661/727] bumped version --- CHANGES.rst | 14 ++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 4b2dc0894..14f50ec6f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,20 @@ Changes ======= +1.24.2 +------ + +Fixed a bug with ``delete`` queries which had joins in the ``where`` clause. +For example: + +.. code-block:: python + + >>> await Band.delete().where(Band.manager.name == 'Guido') + +Thanks to @HakierGrzonzo for reporting the issue, and @sinisaos for the fix. + +------------------------------------------------------------------------------- + 1.24.1 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 88a38dfbf..2f669327c 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.24.1" +__VERSION__ = "1.24.2" From f9c77cc89dd3f5ef6d64ac6426395b684e70bbbe Mon Sep 17 00:00:00 2001 From: sinisaos Date: Sat, 5 Apr 2025 00:08:10 +0200 Subject: [PATCH 662/727] fix remove method bug (#1175) --- piccolo/table.py | 2 ++ tests/table/instance/test_remove.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/piccolo/table.py b/piccolo/table.py index bae9b8a47..ee98c1bfa 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -573,6 +573,8 @@ def remove(self) -> Delete: setattr(self, self._meta.primary_key._meta.name, None) + self._exists_in_db = False + return self.__class__.delete().where( self.__class__._meta.primary_key == primary_key_value ) diff --git a/tests/table/instance/test_remove.py b/tests/table/instance/test_remove.py index be9f7b511..494b4fc89 100644 --- a/tests/table/instance/test_remove.py +++ b/tests/table/instance/test_remove.py @@ -17,9 +17,11 @@ def test_remove(self): "Maz" in Manager.select(Manager.name).output(as_list=True).run_sync() ) + self.assertEqual(manager._exists_in_db, True) manager.remove().run_sync() self.assertTrue( "Maz" not in Manager.select(Manager.name).output(as_list=True).run_sync() ) + self.assertEqual(manager._exists_in_db, False) From b7937ab0a3e3c76264e589d6df5dd078d367b66e Mon Sep 17 00:00:00 2001 From: sinisaos Date: Mon, 14 Apr 2025 15:39:21 +0200 Subject: [PATCH 663/727] fix esmerald asgi template (#1181) --- piccolo/apps/asgi/commands/templates/app/_esmerald_app.py.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piccolo/apps/asgi/commands/templates/app/_esmerald_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_esmerald_app.py.jinja index 978300bf6..01e6dfe45 100644 --- a/piccolo/apps/asgi/commands/templates/app/_esmerald_app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/_esmerald_app.py.jinja @@ -12,7 +12,7 @@ from esmerald import ( post, put, ) -from esmerald.config import StaticFilesConfig +from esmerald.core.config import StaticFilesConfig from piccolo.engine import engine_finder from piccolo.utils.pydantic import create_pydantic_model from piccolo_admin.endpoints import create_admin From 9f9febbcd1bb9da1d8076b1e7258520cad14a334 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 16 Apr 2025 20:07:51 +0100 Subject: [PATCH 664/727] 1178 Improve imports when app is created in a custom root (#1179) * improve imports when app is created in a custom root * use a relative path as an example * fix bug * make intermediate folders * remove unused import --- piccolo/apps/app/commands/new.py | 4 +-- .../commands/templates/piccolo_app.py.jinja | 5 +-- piccolo/conf/apps.py | 31 ++++++++++++++++--- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/piccolo/apps/app/commands/new.py b/piccolo/apps/app/commands/new.py index 973134653..2d43287d0 100644 --- a/piccolo/apps/app/commands/new.py +++ b/piccolo/apps/app/commands/new.py @@ -44,7 +44,7 @@ def new_app(app_name: str, root: str = "."): "Python module. Please choose a different name for your app." ) - os.mkdir(app_root) + os.makedirs(app_root) with open(os.path.join(app_root, "__init__.py"), "w"): pass @@ -77,7 +77,7 @@ def new(app_name: str, root: str = "."): :param app_name: The name of the new app. :param root: - Where to create the app e.g. /my/folder. By default it creates the + Where to create the app e.g. ./my/folder. By default it creates the app in the current directory. """ diff --git a/piccolo/apps/app/commands/templates/piccolo_app.py.jinja b/piccolo/apps/app/commands/templates/piccolo_app.py.jinja index 4527161bc..39d629814 100644 --- a/piccolo/apps/app/commands/templates/piccolo_app.py.jinja +++ b/piccolo/apps/app/commands/templates/piccolo_app.py.jinja @@ -5,7 +5,7 @@ the APP_CONFIG. import os -from piccolo.conf.apps import AppConfig, table_finder +from piccolo.conf.apps import AppConfig, table_finder, get_package CURRENT_DIRECTORY = os.path.dirname(os.path.abspath(__file__)) @@ -18,7 +18,8 @@ APP_CONFIG = AppConfig( 'piccolo_migrations' ), table_classes=table_finder( - modules=["{{ app_name }}.tables"], + modules=[".tables"], + package=get_package(__name__), exclude_imported=True ), migration_dependencies=[], diff --git a/piccolo/conf/apps.py b/piccolo/conf/apps.py index 6c16c9e81..c36d389a7 100644 --- a/piccolo/conf/apps.py +++ b/piccolo/conf/apps.py @@ -32,8 +32,18 @@ class PiccoloAppModule(ModuleType): APP_CONFIG: AppConfig +def get_package(name: str) -> str: + """ + :param name: + The __name__ variable from a Python file. + + """ + return ".".join(name.split(".")[:-1]) + + def table_finder( modules: t.Sequence[str], + package: t.Optional[str] = None, include_tags: t.Optional[t.Sequence[str]] = None, exclude_tags: t.Optional[t.Sequence[str]] = None, exclude_imported: bool = False, @@ -46,8 +56,10 @@ def table_finder( :param modules: The module paths to check for ``Table`` subclasses. For example, - ``['blog.tables']``. The path should be from the root of your project, - not a relative path. + ``['blog.tables']``. + :param package: + This must be passed in if the modules are relative paths (e.g. + ``['.foo']``). :param include_tags: If the ``Table`` subclass has one of these tags, it will be imported. The special tag ``'__all__'`` will import all ``Table`` @@ -83,10 +95,19 @@ class Task(Table): # included table_subclasses: t.List[t.Type[Table]] = [] for module_path in modules: + full_module_path = ( + ".".join([package, module_path.lstrip(".")]) + if package + else module_path + ) + try: - module = import_module(module_path) + module = import_module( + module_path, + package=package, + ) except ImportError as exception: - print(f"Unable to import {module_path}") + print(f"Unable to import {full_module_path}") raise exception from exception object_names = [i for i in dir(module) if not i.startswith("_")] @@ -100,7 +121,7 @@ class Task(Table): # included ): table: Table = _object # type: ignore - if exclude_imported and table.__module__ != module_path: + if exclude_imported and table.__module__ != full_module_path: continue if exclude_tags and set(table._meta.tags).intersection( From 45775a7c9acb5e68e3668494bc4d8b8a64bcf9a9 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 17 Apr 2025 18:20:21 +0100 Subject: [PATCH 665/727] 1184 Be able to auto register apps (#1182) * auto register apps * update docs * add tests * move `get_app_identifier` out * rename var to `app_module` * fix test * add docs for root --- .../projects_and_apps/piccolo_apps.rst | 17 +++- piccolo/apps/app/commands/new.py | 19 +++- piccolo/conf/apps.py | 99 +++++++++++++++++++ tests/apps/app/commands/test_new.py | 20 +++- tests/conf/test_apps.py | 50 +++++++++- 5 files changed, 198 insertions(+), 7 deletions(-) diff --git a/docs/src/piccolo/projects_and_apps/piccolo_apps.rst b/docs/src/piccolo/projects_and_apps/piccolo_apps.rst index fe13f9e1a..631eb4f4f 100644 --- a/docs/src/piccolo/projects_and_apps/piccolo_apps.rst +++ b/docs/src/piccolo/projects_and_apps/piccolo_apps.rst @@ -18,7 +18,7 @@ Run the following command within your project: .. code-block:: bash - piccolo app new my_app + piccolo app new my_app --register Where ``my_app`` is your new app's name. This will create a folder like this: @@ -34,7 +34,8 @@ Where ``my_app`` is your new app's name. This will create a folder like this: It's important to register your new app with the ``APP_REGISTRY`` in -``piccolo_conf.py``. +``piccolo_conf.py``. If you used the ``--register`` flag, then this is done +automatically. Otherwise, add it manually: .. code-block:: python @@ -45,6 +46,18 @@ It's important to register your new app with the ``APP_REGISTRY`` in Anytime you invoke the ``piccolo`` command, you will now be able to perform operations on your app, such as :ref:`Migrations`. +root +~~~~ + +By default the app is created in the current directory. If you want the app to +be in a sub folder, you can use the ``--root`` option: + +.. code-block:: bash + + piccolo app new my_app --register --root=./apps + +The app will then be created in the ``apps`` folder. + ------------------------------------------------------------------------------- AppConfig diff --git a/piccolo/apps/app/commands/new.py b/piccolo/apps/app/commands/new.py index 2d43287d0..a3cd0c0e8 100644 --- a/piccolo/apps/app/commands/new.py +++ b/piccolo/apps/app/commands/new.py @@ -2,12 +2,15 @@ import importlib import os +import pathlib import sys import typing as t import black import jinja2 +from piccolo.conf.apps import PiccoloConfUpdater + TEMPLATE_DIRECTORY = os.path.join( os.path.dirname(os.path.abspath(__file__)), "templates" ) @@ -30,7 +33,11 @@ def module_exists(module_name: str) -> bool: return True -def new_app(app_name: str, root: str = "."): +def get_app_module(app_name: str, root: str) -> str: + return ".".join([*pathlib.Path(root).parts, app_name, "piccolo_app"]) + + +def new_app(app_name: str, root: str = ".", register: bool = False): print(f"Creating {app_name} app ...") app_root = os.path.join(root, app_name) @@ -69,8 +76,12 @@ def new_app(app_name: str, root: str = "."): with open(os.path.join(migrations_folder_path, "__init__.py"), "w"): pass + if register: + app_module = get_app_module(app_name=app_name, root=root) + PiccoloConfUpdater().register_app(app_module=app_module) + -def new(app_name: str, root: str = "."): +def new(app_name: str, root: str = ".", register: bool = False): """ Creates a new Piccolo app. @@ -79,6 +90,8 @@ def new(app_name: str, root: str = "."): :param root: Where to create the app e.g. ./my/folder. By default it creates the app in the current directory. + :param register: + If True, the app is registered automatically in piccolo_conf.py. """ - new_app(app_name=app_name, root=root) + new_app(app_name=app_name, root=root, register=register) diff --git a/piccolo/conf/apps.py b/piccolo/conf/apps.py index c36d389a7..27379a993 100644 --- a/piccolo/conf/apps.py +++ b/piccolo/conf/apps.py @@ -1,5 +1,6 @@ from __future__ import annotations +import ast import inspect import itertools import os @@ -11,6 +12,8 @@ from importlib import import_module from types import ModuleType +import black + from piccolo.apps.migrations.auto.migration_manager import MigrationManager from piccolo.engine.base import Engine from piccolo.table import Table @@ -437,6 +440,17 @@ def get_piccolo_conf_module( else: return module + def get_piccolo_conf_path(self) -> str: + piccolo_conf_module = self.get_piccolo_conf_module() + + if piccolo_conf_module is None: + raise ModuleNotFoundError("piccolo_conf.py not found.") + + module_file_path = piccolo_conf_module.__file__ + assert module_file_path + + return module_file_path + def get_app_registry(self) -> AppRegistry: """ Returns the ``AppRegistry`` instance within piccolo_conf. @@ -583,3 +597,88 @@ def get_table_classes( tables.extend(app_config.table_classes) return tables + + +############################################################################### + + +class PiccoloConfUpdater: + + def __init__(self, piccolo_conf_path: t.Optional[str] = None): + """ + :param piccolo_conf_path: + The path to the piccolo_conf.py (e.g. `./piccolo_conf.py`). If not + passed in, we use our ``Finder`` class to get it. + """ + self.piccolo_conf_path = ( + piccolo_conf_path or Finder().get_piccolo_conf_path() + ) + + def _modify_app_registry_src(self, src: str, app_module: str) -> str: + """ + :param src: + The contents of the ``piccolo_conf.py`` file. + :param app_module: + The app to add to the registry e.g. ``'music.piccolo_app'``. + :returns: + Updated Python source code string. + + """ + ast_root = ast.parse(src) + + parsing_successful = False + + for node in ast.walk(ast_root): + if isinstance(node, ast.Call): + if ( + isinstance(node.func, ast.Name) + and node.func.id == "AppRegistry" + ): + if len(node.keywords) > 0: + keyword = node.keywords[0] + if keyword.arg == "apps": + apps = keyword.value + if isinstance(apps, ast.List): + apps.elts.append( + ast.Constant(app_module, kind="str") + ) + parsing_successful = True + break + + if not parsing_successful: + raise SyntaxError( + "Unable to parse piccolo_conf.py - `AppRegistry(apps=...)` " + "not found)." + ) + + new_contents = ast.unparse(ast_root) + + formatted_contents = black.format_str( + new_contents, mode=black.FileMode(line_length=80) + ) + + return formatted_contents + + def register_app(self, app_module: str): + """ + Adds the given app to the ``AppRegistry`` in ``piccolo_conf.py``. + + This is used by command line tools like: + + .. code-block:: bash + + piccolo app new my_app --register + + :param app_module: + The module of the app, e.g. ``'music.piccolo_app'``. + + """ + with open(self.piccolo_conf_path) as f: + piccolo_conf_src = f.read() + + new_contents = self._modify_app_registry_src( + src=piccolo_conf_src, app_module=app_module + ) + + with open(self.piccolo_conf_path, "wt") as f: + f.write(new_contents) diff --git a/tests/apps/app/commands/test_new.py b/tests/apps/app/commands/test_new.py index fe2addd88..7c6cbdbc7 100644 --- a/tests/apps/app/commands/test_new.py +++ b/tests/apps/app/commands/test_new.py @@ -3,7 +3,7 @@ import tempfile from unittest import TestCase -from piccolo.apps.app.commands.new import module_exists, new +from piccolo.apps.app.commands.new import get_app_module, module_exists, new class TestModuleExists(TestCase): @@ -43,3 +43,21 @@ def test_new_with_clashing_name(self): "A module called sys already exists" ) ) + + +class TestGetAppIdentifier(TestCase): + + def test_get_app_module(self): + """ + Make sure the the ``root`` argument is handled correctly. + """ + self.assertEqual( + get_app_module(app_name="music", root="."), + "music.piccolo_app", + ) + + for root in ("apps", "./apps", "./apps/"): + self.assertEqual( + get_app_module(app_name="music", root=root), + "apps.music.piccolo_app", + ) diff --git a/tests/conf/test_apps.py b/tests/conf/test_apps.py index 0749f5f25..e98c978a0 100644 --- a/tests/conf/test_apps.py +++ b/tests/conf/test_apps.py @@ -1,10 +1,17 @@ from __future__ import annotations import pathlib +import tempfile from unittest import TestCase from piccolo.apps.user.tables import BaseUser -from piccolo.conf.apps import AppConfig, AppRegistry, Finder, table_finder +from piccolo.conf.apps import ( + AppConfig, + AppRegistry, + Finder, + PiccoloConfUpdater, + table_finder, +) from tests.example_apps.mega.tables import MegaTable, SmallTable from tests.example_apps.music.tables import ( Band, @@ -310,3 +317,44 @@ def test_sort_app_configs(self): self.assertListEqual( [i.app_name for i in sorted_app_configs], ["app_2", "app_1"] ) + + +class TestPiccoloConfUpdater(TestCase): + + def test_modify_app_registry_src(self): + """ + Make sure the `piccolo_conf.py` source code can be modified + successfully. + """ + updater = PiccoloConfUpdater() + + new_src = updater._modify_app_registry_src( + src="APP_REGISTRY = AppRegistry(apps=[])", + app_module="music.piccolo_app", + ) + self.assertEqual( + new_src.strip(), + 'APP_REGISTRY = AppRegistry(apps=["music.piccolo_app"])', + ) + + def test_register_app(self): + """ + Make sure the new contents get written to disk. + """ + temp_dir = tempfile.gettempdir() + piccolo_conf_path = pathlib.Path(temp_dir) / "piccolo_conf.py" + + src = "APP_REGISTRY = AppRegistry(apps=[])" + + with open(piccolo_conf_path, "wt") as f: + f.write(src) + + updater = PiccoloConfUpdater(piccolo_conf_path=str(piccolo_conf_path)) + updater.register_app(app_module="music.piccolo_app") + + with open(piccolo_conf_path) as f: + contents = f.read().strip() + + self.assertEqual( + contents, 'APP_REGISTRY = AppRegistry(apps=["music.piccolo_app"])' + ) From 9517f49eef60a3e90bea21f8d25f3d99b6126181 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 17 Apr 2025 23:18:38 +0100 Subject: [PATCH 666/727] validate app name (#1186) --- piccolo/apps/app/commands/new.py | 33 +++++++++++++++++++++++++---- tests/apps/app/commands/test_new.py | 23 +++++++++++++++++++- 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/piccolo/apps/app/commands/new.py b/piccolo/apps/app/commands/new.py index a3cd0c0e8..e35a7f7db 100644 --- a/piccolo/apps/app/commands/new.py +++ b/piccolo/apps/app/commands/new.py @@ -3,6 +3,7 @@ import importlib import os import pathlib +import string import sys import typing as t @@ -33,6 +34,25 @@ def module_exists(module_name: str) -> bool: return True +APP_NAME_ALLOWED_CHARACTERS = [*string.ascii_lowercase, *string.digits, "_"] + + +def validate_app_name(app_name: str): + """ + Make sure the app name is something which is a valid Python package name. + + :raises ValueError: + If ``app_name`` isn't valid. + + """ + for char in app_name: + if not char.lower() in APP_NAME_ALLOWED_CHARACTERS: + raise ValueError( + f"The app name contains a disallowed character: `{char}`. " + "It must only include a-z, 0-9, and _ characters." + ) + + def get_app_module(app_name: str, root: str) -> str: return ".".join([*pathlib.Path(root).parts, app_name, "piccolo_app"]) @@ -40,10 +60,10 @@ def get_app_module(app_name: str, root: str) -> str: def new_app(app_name: str, root: str = ".", register: bool = False): print(f"Creating {app_name} app ...") - app_root = os.path.join(root, app_name) - - if os.path.exists(app_root): - sys.exit("Folder already exists - exiting.") + try: + validate_app_name(app_name=app_name) + except ValueError as exception: + sys.exit(str(exception)) if module_exists(app_name): sys.exit( @@ -51,6 +71,11 @@ def new_app(app_name: str, root: str = ".", register: bool = False): "Python module. Please choose a different name for your app." ) + app_root = os.path.join(root, app_name) + + if os.path.exists(app_root): + sys.exit("Folder already exists - exiting.") + os.makedirs(app_root) with open(os.path.join(app_root, "__init__.py"), "w"): diff --git a/tests/apps/app/commands/test_new.py b/tests/apps/app/commands/test_new.py index 7c6cbdbc7..bcd89a045 100644 --- a/tests/apps/app/commands/test_new.py +++ b/tests/apps/app/commands/test_new.py @@ -3,7 +3,12 @@ import tempfile from unittest import TestCase -from piccolo.apps.app.commands.new import get_app_module, module_exists, new +from piccolo.apps.app.commands.new import ( + get_app_module, + module_exists, + new, + validate_app_name, +) class TestModuleExists(TestCase): @@ -45,6 +50,22 @@ def test_new_with_clashing_name(self): ) +class TestValidateAppName(TestCase): + + def test_validate_app_name(self): + """ + Make sure only app names which work as valid Python package names are + allowed. + """ + # Should be rejected: + for app_name in ("MY APP", "app/my_app", "my.app"): + with self.assertRaises(ValueError): + validate_app_name(app_name=app_name) + + # Should work fine: + validate_app_name(app_name="music") + + class TestGetAppIdentifier(TestCase): def test_get_app_module(self): From 81d102eb56c1e9687b88ddd9d80a57cd3aa9b3a5 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 26 Apr 2025 23:12:11 +0100 Subject: [PATCH 667/727] tweak `table_finder` docs --- docs/src/piccolo/projects_and_apps/piccolo_apps.rst | 5 +++-- piccolo/conf/apps.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/src/piccolo/projects_and_apps/piccolo_apps.rst b/docs/src/piccolo/projects_and_apps/piccolo_apps.rst index 631eb4f4f..36a3b6330 100644 --- a/docs/src/piccolo/projects_and_apps/piccolo_apps.rst +++ b/docs/src/piccolo/projects_and_apps/piccolo_apps.rst @@ -213,8 +213,9 @@ list of modules. commands=[] ) -The module path should be from the root of the project (the same directory as -your ``piccolo_conf.py`` file, rather than a relative path). +The ``modules`` list can contain absolute paths (e.g. ``'blog.tables'``) or +relative paths (e.g. ``'.tables'``). If relative paths are used, then the +``package`` argument must be passed in (``'blog'`` in this case). You can filter the ``Table`` subclasses returned using :ref:`tags `. diff --git a/piccolo/conf/apps.py b/piccolo/conf/apps.py index 27379a993..c5a1fd7dc 100644 --- a/piccolo/conf/apps.py +++ b/piccolo/conf/apps.py @@ -62,7 +62,7 @@ def table_finder( ``['blog.tables']``. :param package: This must be passed in if the modules are relative paths (e.g. - ``['.foo']``). + if ``modules=['.tables']`` then ``package='blog'``). :param include_tags: If the ``Table`` subclass has one of these tags, it will be imported. The special tag ``'__all__'`` will import all ``Table`` From 7cbd871c7b2fcdabbcfed0db9c69c6a72cdb3b37 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 26 Apr 2025 23:12:36 +0100 Subject: [PATCH 668/727] bumped version --- CHANGES.rst | 29 +++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 14f50ec6f..eaad1ccf2 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,35 @@ Changes ======= +1.25.0 +------ + +Improvements to Piccolo app creation. When running the following: + +.. code-block:: bash + + piccolo app new my_app + +It now validates that the app name (``my_app`` in this case) is valid as a +Python package. + +Also, there is now a ``--register`` flag, which automatically adds the new app +to the ``APP_REGISTRY`` in ``piccolo_conf.py``. + +.. code-block:: python + + piccolo app new my_app --register + +Other changes: + +* ``table_finder`` can now use relative modules. +* Updated the Esmerald ASGI template (thanks to @sinisaos for this). +* When using the ``remove`` method to delete a row from the database + (``await some_band.remove()``), ``some_band._exists_in_db`` is now set to + ``False``. Thanks to @sinisaos for this fix. + +------------------------------------------------------------------------------- + 1.24.2 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 2f669327c..80c223d96 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.24.2" +__VERSION__ = "1.25.0" From 9d1356b25fcfceb3559a76cc10950a77d4929530 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Mon, 5 May 2025 19:17:53 +0200 Subject: [PATCH 669/727] Altering `Ondelete` and `OnUpdate` with `MigrationManager` (#1183) * fix OnDelete and OnUpdate in migrations * fix test docstring * remove for loop * refactor `add_foreign_key_constraint` If a `ForeignKey` is passed in, we can get most of the info we need from it, rather than requiring additional params to be passed in. * add `get_fk_constraint_name` * rename pk constraint on table renaming in migrations * `RenamePkConstraint` -> `RenameConstraint` * skip renaming pk constraint in SQLite * revert changes * update param names in `rename_constraint` method * add docstring * add `get_fk_constraint_rules` function So the logic for getting constraints is centralised. * add TODO for supporting SQLite * add TODO for moving logic from `generate.py` to `constraints.py` * remove redundant lookup in `generate.py` We should refactor a bunch of this - I came across this dict lookup, which isn't required because we can just call the enum directly. --------- Co-authored-by: Daniel Townsend --- .../apps/migrations/auto/migration_manager.py | 38 +++++++- piccolo/apps/schema/commands/generate.py | 23 +---- piccolo/query/constraints.py | 92 +++++++++++++++++++ piccolo/query/methods/alter.py | 75 +++++++++++++-- piccolo/utils/sync.py | 10 +- .../migrations/auto/test_migration_manager.py | 50 ++++++++++ .../foreign_key/test_on_delete_on_update.py | 43 ++------- tests/type_checking.py | 12 +++ 8 files changed, 276 insertions(+), 67 deletions(-) create mode 100644 piccolo/query/constraints.py diff --git a/piccolo/apps/migrations/auto/migration_manager.py b/piccolo/apps/migrations/auto/migration_manager.py index 31ad5c120..fc7449ccb 100644 --- a/piccolo/apps/migrations/auto/migration_manager.py +++ b/piccolo/apps/migrations/auto/migration_manager.py @@ -19,6 +19,7 @@ from piccolo.engine import engine_finder from piccolo.query import Query from piccolo.query.base import DDL +from piccolo.query.constraints import get_fk_constraint_name from piccolo.schema import SchemaDDLBase from piccolo.table import Table, create_table_class, sort_table_classes from piccolo.utils.warnings import colored_warning @@ -423,8 +424,8 @@ async def _print_query(query: t.Union[DDL, Query, SchemaDDLBase]): async def _run_query(self, query: t.Union[DDL, Query, SchemaDDLBase]): """ - If MigrationManager is not in the preview mode, - executes the queries. else, prints the query. + If MigrationManager is in preview mode then it just print the query + instead of executing it. """ if self.preview: await self._print_query(query) @@ -534,6 +535,39 @@ async def _run_alter_columns(self, backwards: bool = False): ############################################################### + on_delete = params.get("on_delete") + on_update = params.get("on_update") + if on_delete is not None or on_update is not None: + existing_table = await self.get_table_from_snapshot( + table_class_name=table_class_name, + app_name=self.app_name, + ) + + fk_column = existing_table._meta.get_column_by_name( + alter_column.column_name + ) + + assert isinstance(fk_column, ForeignKey) + + # First drop the existing foreign key constraint + constraint_name = await get_fk_constraint_name( + column=fk_column + ) + await self._run_query( + _Table.alter().drop_constraint( + constraint_name=constraint_name + ) + ) + + # Then add a new foreign key constraint + await self._run_query( + _Table.alter().add_foreign_key_constraint( + column=fk_column, + on_delete=on_delete, + on_update=on_update, + ) + ) + null = params.get("null") if null is not None: await self._run_query( diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index da97d247b..c0fe1e95a 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -501,6 +501,8 @@ async def get_fk_triggers( Any Table subclass - just used to execute raw queries on the database. """ + # TODO - Move this query to `piccolo.query.constraints` or use: + # `piccolo.query.constraints.referential_constraints` triggers = await table_class.raw( ( "SELECT tc.constraint_name, " @@ -537,23 +539,6 @@ async def get_fk_triggers( ) -ONDELETE_MAP = { - "NO ACTION": OnDelete.no_action, - "RESTRICT": OnDelete.restrict, - "CASCADE": OnDelete.cascade, - "SET NULL": OnDelete.set_null, - "SET DEFAULT": OnDelete.set_default, -} - -ONUPDATE_MAP = { - "NO ACTION": OnUpdate.no_action, - "RESTRICT": OnUpdate.restrict, - "CASCADE": OnUpdate.cascade, - "SET NULL": OnUpdate.set_null, - "SET DEFAULT": OnUpdate.set_default, -} - - async def get_constraints( table_class: t.Type[Table], tablename: str, schema_name: str = "public" ) -> TableConstraints: @@ -765,8 +750,8 @@ async def create_table_class_from_db( column_name, constraint_table.name ) if trigger: - kwargs["on_update"] = ONUPDATE_MAP[trigger.on_update] - kwargs["on_delete"] = ONDELETE_MAP[trigger.on_delete] + kwargs["on_update"] = OnUpdate(trigger.on_update) + kwargs["on_delete"] = OnDelete(trigger.on_delete) else: output_schema.trigger_warnings.append( f"{tablename}.{column_name}" diff --git a/piccolo/query/constraints.py b/piccolo/query/constraints.py new file mode 100644 index 000000000..7f6d1f565 --- /dev/null +++ b/piccolo/query/constraints.py @@ -0,0 +1,92 @@ +from dataclasses import dataclass + +from piccolo.columns import ForeignKey +from piccolo.columns.base import OnDelete, OnUpdate + + +async def get_fk_constraint_name(column: ForeignKey) -> str: + """ + Checks what the foreign key constraint is called in the database. + """ + + table = column._meta.table + + if table._meta.db.engine_type == "sqlite": + # TODO - add the query for SQLite + raise ValueError("SQLite isn't currently supported.") + + schema = table._meta.schema or "public" + table_name = table._meta.tablename + column_name = column._meta.db_column_name + + constraints = await table.raw( + """ + SELECT + kcu.constraint_name AS fk_constraint_name + FROM + information_schema.referential_constraints AS rc + INNER JOIN + information_schema.key_column_usage AS kcu + ON kcu.constraint_catalog = rc.constraint_catalog + AND kcu.constraint_schema = rc.constraint_schema + AND kcu.constraint_name = rc.constraint_name + WHERE + kcu.table_schema = {} AND + kcu.table_name = {} AND + kcu.column_name = {} + """, + schema, + table_name, + column_name, + ) + + return constraints[0]["fk_constraint_name"] + + +@dataclass +class ConstraintRules: + on_delete: OnDelete + on_update: OnUpdate + + +async def get_fk_constraint_rules(column: ForeignKey) -> ConstraintRules: + """ + Checks the constraint rules for this foreign key in the database. + """ + table = column._meta.table + + if table._meta.db.engine_type == "sqlite": + # TODO - add the query for SQLite + raise ValueError("SQLite isn't currently supported.") + + schema = table._meta.schema or "public" + table_name = table._meta.tablename + column_name = column._meta.db_column_name + + constraints = await table.raw( + """ + SELECT + kcu.constraint_name, + kcu.table_name, + kcu.column_name, + rc.update_rule, + rc.delete_rule + FROM + information_schema.key_column_usage AS kcu + INNER JOIN + information_schema.referential_constraints AS rc + ON kcu.constraint_name = rc.constraint_name + WHERE + kcu.table_schema = {} AND + kcu.table_name = {} AND + kcu.column_name = {} + """, + schema, + table_name, + column_name, + ) + + return ConstraintRules( + on_delete=OnDelete(constraints[0]["delete_rule"]), + on_update=OnUpdate(constraints[0]["update_rule"]), + ) diff --git a/piccolo/query/methods/alter.py b/piccolo/query/methods/alter.py index 040b2f883..b0adba2c4 100644 --- a/piccolo/query/methods/alter.py +++ b/piccolo/query/methods/alter.py @@ -36,6 +36,18 @@ def ddl(self) -> str: return f"RENAME TO {self.new_name}" +@dataclass +class RenameConstraint(AlterStatement): + __slots__ = ("old_name", "new_name") + + old_name: str + new_name: str + + @property + def ddl(self) -> str: + return f"RENAME CONSTRAINT {self.old_name} TO {self.new_name}" + + @dataclass class AlterColumnStatement(AlterStatement): __slots__ = ("column",) @@ -194,6 +206,7 @@ class AddForeignKeyConstraint(AlterStatement): "constraint_name", "foreign_key_column_name", "referenced_table_name", + "referenced_column_name", "on_delete", "on_update", ) @@ -201,9 +214,9 @@ class AddForeignKeyConstraint(AlterStatement): constraint_name: str foreign_key_column_name: str referenced_table_name: str + referenced_column_name: str on_delete: t.Optional[OnDelete] on_update: t.Optional[OnUpdate] - referenced_column_name: str = "id" @property def ddl(self) -> str: @@ -273,8 +286,8 @@ def ddl(self) -> str: class Alter(DDL): __slots__ = ( - "_add_foreign_key_constraint", "_add", + "_add_foreign_key_constraint", "_drop_constraint", "_drop_default", "_drop_table", @@ -288,6 +301,7 @@ class Alter(DDL): "_set_null", "_set_schema", "_set_unique", + "_rename_constraint", ) def __init__(self, table: t.Type[Table], **kwargs): @@ -307,6 +321,7 @@ def __init__(self, table: t.Type[Table], **kwargs): self._set_null: t.List[SetNull] = [] self._set_schema: t.List[SetSchema] = [] self._set_unique: t.List[SetUnique] = [] + self._rename_constraint: t.List[RenameConstraint] = [] def add_column(self: Self, name: str, column: Column) -> Self: """ @@ -372,6 +387,24 @@ def rename_table(self, new_name: str) -> Alter: self._rename_table = [RenameTable(new_name=new_name)] return self + def rename_constraint(self, old_name: str, new_name: str) -> Alter: + """ + Rename a constraint on the table:: + + >>> await Band.alter().rename_constraint( + ... 'old_constraint_name', + ... 'new_constraint_name', + ... ) + + """ + self._rename_constraint = [ + RenameConstraint( + old_name=old_name, + new_name=new_name, + ) + ] + return self + def rename_column( self, column: t.Union[str, Column], new_name: str ) -> Alter: @@ -488,7 +521,7 @@ def set_length(self, column: t.Union[str, Varchar], length: int) -> Alter: def _get_constraint_name(self, column: t.Union[str, ForeignKey]) -> str: column_name = AlterColumnStatement(column=column).column_name tablename = self.table._meta.tablename - return f"{tablename}_{column_name}_fk" + return f"{tablename}_{column_name}_fkey" def drop_constraint(self, constraint_name: str) -> Alter: self._drop_constraint.append( @@ -500,37 +533,58 @@ def drop_foreign_key_constraint( self, column: t.Union[str, ForeignKey] ) -> Alter: constraint_name = self._get_constraint_name(column=column) - return self.drop_constraint(constraint_name=constraint_name) + self._drop_constraint.append( + DropConstraint(constraint_name=constraint_name) + ) + return self def add_foreign_key_constraint( self, column: t.Union[str, ForeignKey], - referenced_table_name: str, + referenced_table_name: t.Optional[str] = None, + referenced_column_name: t.Optional[str] = None, + constraint_name: t.Optional[str] = None, on_delete: t.Optional[OnDelete] = None, on_update: t.Optional[OnUpdate] = None, - referenced_column_name: str = "id", ) -> Alter: """ Add a new foreign key constraint:: >>> await Band.alter().add_foreign_key_constraint( ... Band.manager, - ... referenced_table_name='manager', ... on_delete=OnDelete.cascade ... ) """ - constraint_name = self._get_constraint_name(column=column) + constraint_name = constraint_name or self._get_constraint_name( + column=column + ) column_name = AlterColumnStatement(column=column).column_name + if referenced_column_name is None: + if isinstance(column, ForeignKey): + referenced_column_name = ( + column._foreign_key_meta.resolved_target_column._meta.db_column_name # noqa: E501 + ) + else: + raise ValueError("Please pass in `referenced_column_name`.") + + if referenced_table_name is None: + if isinstance(column, ForeignKey): + referenced_table_name = ( + column._foreign_key_meta.resolved_references._meta.tablename # noqa: E501 + ) + else: + raise ValueError("Please pass in `referenced_table_name`.") + self._add_foreign_key_constraint.append( AddForeignKeyConstraint( constraint_name=constraint_name, foreign_key_column_name=column_name, referenced_table_name=referenced_table_name, + referenced_column_name=referenced_column_name, on_delete=on_delete, on_update=on_update, - referenced_column_name=referenced_column_name, ) ) return self @@ -579,9 +633,12 @@ def default_ddl(self) -> t.Sequence[str]: i.ddl for i in itertools.chain( self._add, + self._add_foreign_key_constraint, self._rename_columns, self._rename_table, + self._rename_constraint, self._drop, + self._drop_constraint, self._drop_default, self._set_column_type, self._set_unique, diff --git a/piccolo/utils/sync.py b/piccolo/utils/sync.py index 55b44e499..348829c9e 100644 --- a/piccolo/utils/sync.py +++ b/piccolo/utils/sync.py @@ -2,10 +2,14 @@ import asyncio import typing as t -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import Future, ThreadPoolExecutor +ReturnType = t.TypeVar("ReturnType") -def run_sync(coroutine: t.Coroutine): + +def run_sync( + coroutine: t.Coroutine[t.Any, t.Any, ReturnType], +) -> ReturnType: """ Run the coroutine synchronously - trying to accommodate as many edge cases as possible. @@ -20,5 +24,5 @@ def run_sync(coroutine: t.Coroutine): except RuntimeError: # An event loop already exists. with ThreadPoolExecutor(max_workers=1) as executor: - future = executor.submit(asyncio.run, coroutine) + future: Future = executor.submit(asyncio.run, coroutine) return future.result() diff --git a/tests/apps/migrations/auto/test_migration_manager.py b/tests/apps/migrations/auto/test_migration_manager.py index 6e71846e0..68b550ddd 100644 --- a/tests/apps/migrations/auto/test_migration_manager.py +++ b/tests/apps/migrations/auto/test_migration_manager.py @@ -12,8 +12,10 @@ from piccolo.columns.column_types import ForeignKey from piccolo.conf.apps import AppConfig from piccolo.engine import engine_finder +from piccolo.query.constraints import get_fk_constraint_rules from piccolo.table import Table, sort_table_classes from piccolo.utils.lazy_loader import LazyLoader +from piccolo.utils.sync import run_sync from tests.base import AsyncMock, DBTestCase, engine_is, engines_only from tests.example_apps.music.tables import Band, Concert, Manager, Venue @@ -618,6 +620,54 @@ def test_rename_table(self): response = self.run_sync("SELECT * FROM manager;") self.assertEqual(response, [{"id": id[0]["id"], "name": "Dave"}]) + @engines_only("postgres", "cockroach") + def test_alter_fk_on_delete_on_update(self): + """ + Test altering OnDelete and OnUpdate with MigrationManager. + """ + # before performing migrations - OnDelete.no_action + self.assertEqual( + run_sync(get_fk_constraint_rules(column=Band.manager)).on_delete, + OnDelete.no_action, + ) + + manager = MigrationManager(app_name="music") + manager.alter_column( + table_class_name="Band", + tablename="band", + column_name="manager", + db_column_name="manager", + params={ + "on_delete": OnDelete.set_null, + "on_update": OnUpdate.set_null, + }, + old_params={ + "on_delete": OnDelete.no_action, + "on_update": OnUpdate.no_action, + }, + column_class=ForeignKey, + old_column_class=ForeignKey, + schema=None, + ) + + asyncio.run(manager.run()) + + # after performing migrations - OnDelete.set_null + self.assertEqual( + run_sync(get_fk_constraint_rules(column=Band.manager)).on_delete, + OnDelete.set_null, + ) + + # Reverse + asyncio.run(manager.run(backwards=True)) + + # after performing reverse migrations we have + # OnDelete.no_action again + self.assertEqual( + run_sync(get_fk_constraint_rules(column=Band.manager)).on_delete, + OnDelete.no_action, + ) + @engines_only("postgres") def test_alter_column_unique(self): """ diff --git a/tests/columns/foreign_key/test_on_delete_on_update.py b/tests/columns/foreign_key/test_on_delete_on_update.py index aa5239942..c7356e171 100644 --- a/tests/columns/foreign_key/test_on_delete_on_update.py +++ b/tests/columns/foreign_key/test_on_delete_on_update.py @@ -1,8 +1,8 @@ -from unittest import TestCase - from piccolo.columns import ForeignKey, Varchar from piccolo.columns.base import OnDelete, OnUpdate +from piccolo.query.constraints import get_fk_constraint_rules from piccolo.table import Table +from piccolo.testing.test_case import AsyncTableTest from tests.base import engines_only @@ -23,40 +23,15 @@ class Band(Table): @engines_only("postgres", "cockroach") -class TestOnDeleteOnUpdate(TestCase): +class TestOnDeleteOnUpdate(AsyncTableTest): """ Make sure that on_delete, and on_update are correctly applied in the database. """ - def setUp(self): - for table_class in (Manager, Band): - table_class.create_table().run_sync() - - def tearDown(self): - for table_class in (Band, Manager): - table_class.alter().drop_table(if_exists=True).run_sync() - - def test_on_delete_on_update(self): - response = Band.raw( - """ - SELECT - rc.update_rule AS on_update, - rc.delete_rule AS on_delete - FROM information_schema.table_constraints tc - LEFT JOIN information_schema.key_column_usage kcu - ON tc.constraint_catalog = kcu.constraint_catalog - AND tc.constraint_schema = kcu.constraint_schema - AND tc.constraint_name = kcu.constraint_name - LEFT JOIN information_schema.referential_constraints rc - ON tc.constraint_catalog = rc.constraint_catalog - AND tc.constraint_schema = rc.constraint_schema - AND tc.constraint_name = rc.constraint_name - WHERE - lower(tc.constraint_type) in ('foreign key') - AND tc.table_name = 'band' - AND kcu.column_name = 'manager'; - """ - ).run_sync() - self.assertTrue(response[0]["on_update"] == "SET NULL") - self.assertTrue(response[0]["on_delete"] == "SET NULL") + tables = [Manager, Band] + + async def test_on_delete_on_update(self): + constraint_rules = await get_fk_constraint_rules(Band.manager) + self.assertEqual(constraint_rules.on_delete, OnDelete.set_null) + self.assertEqual(constraint_rules.on_update, OnDelete.set_null) diff --git a/tests/type_checking.py b/tests/type_checking.py index 7288768e1..717d62206 100644 --- a/tests/type_checking.py +++ b/tests/type_checking.py @@ -11,6 +11,7 @@ from piccolo.columns import ForeignKey, Varchar from piccolo.testing.model_builder import ModelBuilder +from piccolo.utils.sync import run_sync from .example_apps.music.tables import Band, Concert, Manager @@ -117,3 +118,14 @@ async def insert() -> None: async def model_builder() -> None: assert_type(await ModelBuilder.build(Band), Band) assert_type(ModelBuilder.build_sync(Band), Band) + + def run_sync_return_type() -> None: + """ + Make sure `run_sync` returns the same type as the coroutine which is + passed in. + """ + + async def my_func() -> str: + return "hello" + + assert_type(run_sync(my_func()), str) From 85dc203946f3329995ce8191a301d8eaed60ee7c Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 12 May 2025 15:23:58 +0100 Subject: [PATCH 670/727] bumped version --- CHANGES.rst | 8 ++++++++ piccolo/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index eaad1ccf2..e3997badd 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,14 @@ Changes ======= +1.26.0 +------ + +Improved auto migrations - ``ON DELETE`` and ``ON UPDATE`` can be modified +on ``ForeignKey`` columns. Thanks to @sinisaos for this. + +------------------------------------------------------------------------------- + 1.25.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 80c223d96..05e434c62 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.25.0" +__VERSION__ = "1.26.0" From 452e3222ed390851f11a9e18665e6769d4e8aa79 Mon Sep 17 00:00:00 2001 From: Sarvesh Dwivedi Date: Mon, 12 May 2025 20:33:24 +0530 Subject: [PATCH 671/727] docs: add tutorial for moving a table between Piccolo apps (#1191) * docs: add tutorial for moving a table between Piccolo apps without data loss * docs: update migration commands in table moving tutorial for clarity --------- Co-authored-by: sarvesh4396 --- docs/src/piccolo/tutorials/index.rst | 1 + .../tutorials/moving_table_between_apps.rst | 82 +++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 docs/src/piccolo/tutorials/moving_table_between_apps.rst diff --git a/docs/src/piccolo/tutorials/index.rst b/docs/src/piccolo/tutorials/index.rst index 1fae535a0..e017ad7d5 100644 --- a/docs/src/piccolo/tutorials/index.rst +++ b/docs/src/piccolo/tutorials/index.rst @@ -12,3 +12,4 @@ help you solve common problems: ./deployment ./fastapi ./avoiding_circular_imports + ./moving_table_between_apps diff --git a/docs/src/piccolo/tutorials/moving_table_between_apps.rst b/docs/src/piccolo/tutorials/moving_table_between_apps.rst new file mode 100644 index 000000000..5d72c52f5 --- /dev/null +++ b/docs/src/piccolo/tutorials/moving_table_between_apps.rst @@ -0,0 +1,82 @@ +Moving a Table Between Piccolo Apps Without Data Loss +====================================================== + +Piccolo ORM makes it easy to manage models within individual apps. But what if you need to move a table (model) from one app to another—say, from ``app_a`` to ``app_b``—without losing your data? + +This tutorial walks you through the safest way to move a table between Piccolo apps using migrations and the ``--fake`` flag. + +Use Case +-------- + +You're working on a project structured with multiple Piccolo apps, and you want to reorganize your models by moving a table (``TableA``) from one app (``app_a``) to another (``app_b``), without affecting the data in your database. + +Prerequisites +------------- + +- Piccolo ORM installed and configured +- Both ``app_a`` and ``app_b`` listed in ``piccolo_conf.py`` under ``PICCOLO_APPS`` +- Basic familiarity with Piccolo migrations + +Step-by-Step Instructions +------------------------- + +1. Remove the Table from ``app_a`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In ``app_a/tables.py``, delete or comment out the ``TableA`` class definition. + +2. Create a Migration in ``app_a`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Run the following command in your terminal: + +.. code-block:: bash + + piccolo migrations new app_a --auto + +This will create a migration that removes the table from ``app_a``. + +3. Fake Apply the Migration +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To prevent the table from actually being dropped from the database, apply the migration using the ``--fake`` flag: + +.. code-block:: bash + + piccolo migrations forwards app_a --fake + +This marks the migration as applied without making real changes to the database. + +4. Move the Table to ``app_b`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Copy the ``TableA`` class definition into ``app_b/tables.py``. + +Ensure the definition matches exactly what it was in ``app_a``. + +5. Create a Migration in ``app_b`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Generate a new fake migration for ``app_b`` to register ``TableA``: + +.. code-block:: bash + + piccolo migrations new app_b --auto + +6. Apply the Migration in ``app_b`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Apply the new migration: + +.. code-block:: bash + + piccolo migrations forwards app_b --fake + +Because the table already exists in the database, Piccolo will associate it with ``app_b`` without duplicating or altering it. + +Notes & Tips +------------ + +- This process preserves your data because it avoids actually dropping or creating the table. +- Always back up your database before doing schema changes. +- Inspect the migration files to understand what Piccolo is tracking. From bf4e64ccbd53ad081d8e6f400423e7e3ca407d95 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Tue, 13 May 2025 12:45:00 +0200 Subject: [PATCH 672/727] update BlackSheep template (#1195) --- .../apps/asgi/commands/templates/app/_blacksheep_app.py.jinja | 2 ++ 1 file changed, 2 insertions(+) diff --git a/piccolo/apps/asgi/commands/templates/app/_blacksheep_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_blacksheep_app.py.jinja index 60d06f33b..d2e9e4d8b 100644 --- a/piccolo/apps/asgi/commands/templates/app/_blacksheep_app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/_blacksheep_app.py.jinja @@ -115,3 +115,5 @@ async def close_database_connection_pool(application): app.on_start += open_database_connection_pool app.on_stop += close_database_connection_pool + +app.router.apply_routes() From 04e1330419691ee6ba358766084480563855b6ae Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 14 May 2025 23:30:53 +0100 Subject: [PATCH 673/727] check if the `target_column` is the primary key (#1198) --- piccolo/apps/migrations/auto/serialisation.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/piccolo/apps/migrations/auto/serialisation.py b/piccolo/apps/migrations/auto/serialisation.py index b3644b853..c82e4856a 100644 --- a/piccolo/apps/migrations/auto/serialisation.py +++ b/piccolo/apps/migrations/auto/serialisation.py @@ -379,6 +379,12 @@ def __repr__(self) -> str: else: raise ValueError("Unrecognised `target_column` value.") + if column._meta.name == pk_column._meta.name: + # The target column is the foreign key, so no need to add + # it again. + # https://github.com/piccolo-orm/piccolo/issues/1197 + continue + serialised_target_columns.add( SerialisedColumnInstance( column, From 0eeec95bb86b7c3191b074eaff17f334dba156e3 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 16 May 2025 18:29:31 +0100 Subject: [PATCH 674/727] bumped version --- CHANGES.rst | 11 +++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index e3997badd..fafb75a7c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,17 @@ Changes ======= +1.26.1 +------ + +Updated the BlackSheep ASGI template - thanks to @sinisaos for this. + +Fixed a bug with auto migrations when a ``ForeignKey`` specifies +``target_column`` - multiple primary key columns were added to the migration +file. Thanks to @waldner for reporting this issue. + +------------------------------------------------------------------------------- + 1.26.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 05e434c62..093399c19 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.26.0" +__VERSION__ = "1.26.1" From 1e69d7b16bad0a31ba329e79581723f95df5195b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 16 May 2025 18:30:50 +0100 Subject: [PATCH 675/727] Update CHANGES.rst --- CHANGES.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index fafb75a7c..ea0dd3b4c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -10,6 +10,9 @@ Fixed a bug with auto migrations when a ``ForeignKey`` specifies ``target_column`` - multiple primary key columns were added to the migration file. Thanks to @waldner for reporting this issue. +Added a tutorial for moving tables between Piccolo apps - thanks to +@sarvesh4396 for this. + ------------------------------------------------------------------------------- 1.26.0 From 1dd0613b63de498067108b7163d8e7f443a5e8c1 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 3 Jun 2025 20:31:32 +0100 Subject: [PATCH 676/727] 1204 Remove `primary` arg from old files (#1205) * rename `primary` and `key` to `primary_key` * remove `primary` arg from tests * just check for `primary` --- .../piccolo_migrations/2019-11-14T21-52-21.py | 15 +++++---------- .../piccolo_migrations/2020-06-11T21-38-55.py | 6 ++---- .../piccolo_migrations/2021-04-30T16-14-15.py | 6 ++---- piccolo/columns/base.py | 7 +++---- .../migrations/auto/test_migration_manager.py | 15 +++++---------- .../test_migrations/2020-03-31T20-38-22.py | 3 +-- tests/columns/test_primary_key.py | 4 ++-- .../piccolo_migrations/2020-12-17T18-44-30.py | 12 ++++-------- .../piccolo_migrations/2020-12-17T18-44-39.py | 18 ++++++------------ .../piccolo_migrations/2020-12-17T18-44-44.py | 3 +-- .../2021-07-25T22-38-48-009306.py | 9 +++------ 11 files changed, 34 insertions(+), 64 deletions(-) diff --git a/piccolo/apps/user/piccolo_migrations/2019-11-14T21-52-21.py b/piccolo/apps/user/piccolo_migrations/2019-11-14T21-52-21.py index 600e54946..8205aef64 100644 --- a/piccolo/apps/user/piccolo_migrations/2019-11-14T21-52-21.py +++ b/piccolo/apps/user/piccolo_migrations/2019-11-14T21-52-21.py @@ -15,8 +15,7 @@ async def forwards(): "length": 100, "default": "", "null": False, - "primary": False, - "key": False, + "primary_key": False, "unique": True, "index": False, }, @@ -30,8 +29,7 @@ async def forwards(): "length": 255, "default": "", "null": False, - "primary": False, - "key": False, + "primary_key": False, "unique": False, "index": False, }, @@ -45,8 +43,7 @@ async def forwards(): "length": 255, "default": "", "null": False, - "primary": False, - "key": False, + "primary_key": False, "unique": True, "index": False, }, @@ -59,8 +56,7 @@ async def forwards(): params={ "default": False, "null": False, - "primary": False, - "key": False, + "primary_key": False, "unique": False, "index": False, }, @@ -73,8 +69,7 @@ async def forwards(): params={ "default": False, "null": False, - "primary": False, - "key": False, + "primary_key": False, "unique": False, "index": False, }, diff --git a/piccolo/apps/user/piccolo_migrations/2020-06-11T21-38-55.py b/piccolo/apps/user/piccolo_migrations/2020-06-11T21-38-55.py index 73701adb0..b5dc10908 100644 --- a/piccolo/apps/user/piccolo_migrations/2020-06-11T21-38-55.py +++ b/piccolo/apps/user/piccolo_migrations/2020-06-11T21-38-55.py @@ -15,8 +15,7 @@ async def forwards(): "length": 255, "default": "", "null": True, - "primary": False, - "key": False, + "primary_key": False, "unique": False, "index": False, }, @@ -31,8 +30,7 @@ async def forwards(): "length": 255, "default": "", "null": True, - "primary": False, - "key": False, + "primary_key": False, "unique": False, "index": False, }, diff --git a/piccolo/apps/user/piccolo_migrations/2021-04-30T16-14-15.py b/piccolo/apps/user/piccolo_migrations/2021-04-30T16-14-15.py index 7737e6767..ac1a6ecd1 100644 --- a/piccolo/apps/user/piccolo_migrations/2021-04-30T16-14-15.py +++ b/piccolo/apps/user/piccolo_migrations/2021-04-30T16-14-15.py @@ -18,8 +18,7 @@ async def forwards(): params={ "default": False, "null": False, - "primary": False, - "key": False, + "primary_key": False, "unique": False, "index": False, "index_method": IndexMethod.btree, @@ -35,8 +34,7 @@ async def forwards(): params={ "default": None, "null": True, - "primary": False, - "key": False, + "primary_key": False, "unique": False, "index": False, "index_method": IndexMethod.btree, diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 4725b78ad..8626f4dc6 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -467,10 +467,9 @@ def __init__( auto_update: t.Any = ..., **kwargs, ) -> None: - # This is for backwards compatibility - originally there were two - # separate arguments `primary` and `key`, but they have now been merged - # into `primary_key`. - if (kwargs.get("primary") is True) and (kwargs.get("key") is True): + # This is for backwards compatibility - originally the `primary_key` + # argument was called `primary`. + if kwargs.get("primary") is True: primary_key = True # Used for migrations. diff --git a/tests/apps/migrations/auto/test_migration_manager.py b/tests/apps/migrations/auto/test_migration_manager.py index 68b550ddd..e4d1a1d41 100644 --- a/tests/apps/migrations/auto/test_migration_manager.py +++ b/tests/apps/migrations/auto/test_migration_manager.py @@ -286,8 +286,7 @@ def test_add_column(self) -> None: "length": 100, "default": "", "null": True, - "primary": False, - "key": False, + "primary_key": False, "unique": True, "index": False, }, @@ -355,8 +354,7 @@ def test_add_column_with_index(self): "length": 100, "default": "", "null": True, - "primary": False, - "key": False, + "primary_key": False, "unique": True, "index": True, }, @@ -402,8 +400,7 @@ def test_add_foreign_key_self_column(self): "on_update": OnUpdate.cascade, "default": None, "null": True, - "primary": False, - "key": False, + "primary_key": False, "unique": False, "index": False, }, @@ -449,8 +446,7 @@ def test_add_foreign_key_self_column_alt(self): "on_update": OnUpdate.cascade, "default": None, "null": True, - "primary": False, - "key": False, + "primary_key": False, "unique": False, "index": False, }, @@ -505,8 +501,7 @@ def test_add_non_nullable_column(self): "length": 100, "default": "", "null": False, - "primary": False, - "key": False, + "primary_key": False, "unique": True, "index": False, }, diff --git a/tests/apps/migrations/commands/test_migrations/2020-03-31T20-38-22.py b/tests/apps/migrations/commands/test_migrations/2020-03-31T20-38-22.py index 347020d44..bcdf8e8e2 100644 --- a/tests/apps/migrations/commands/test_migrations/2020-03-31T20-38-22.py +++ b/tests/apps/migrations/commands/test_migrations/2020-03-31T20-38-22.py @@ -15,8 +15,7 @@ async def forwards(): "length": 150, "default": "", "null": True, - "primary": False, - "key": False, + "primary_key": False, "unique": False, "index": False, }, diff --git a/tests/columns/test_primary_key.py b/tests/columns/test_primary_key.py index 55bff8ee5..86868a2c8 100644 --- a/tests/columns/test_primary_key.py +++ b/tests/columns/test_primary_key.py @@ -75,12 +75,12 @@ def test_return_type(self): class Manager(Table): - pk = UUID(primary=True, key=True) + pk = UUID(primary_key=True) name = Varchar() class Band(Table): - pk = UUID(primary=True, key=True) + pk = UUID(primary_key=True) name = Varchar() manager = ForeignKey(Manager) diff --git a/tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-30.py b/tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-30.py index e9a5e42e0..be87769ea 100644 --- a/tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-30.py +++ b/tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-30.py @@ -26,8 +26,7 @@ async def forwards(): "length": 50, "default": "", "null": False, - "primary": False, - "key": False, + "primary_key": False, "unique": False, "index": False, }, @@ -44,8 +43,7 @@ async def forwards(): "on_update": OnUpdate.cascade, "default": None, "null": True, - "primary": False, - "key": False, + "primary_key": False, "unique": False, "index": False, }, @@ -59,8 +57,7 @@ async def forwards(): params={ "default": 0, "null": False, - "primary": False, - "key": False, + "primary_key": False, "unique": False, "index": False, }, @@ -75,8 +72,7 @@ async def forwards(): "length": 50, "default": "", "null": False, - "primary": False, - "key": False, + "primary_key": False, "unique": False, "index": False, }, diff --git a/tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-39.py b/tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-39.py index 7e4ed09d2..48048ce5a 100644 --- a/tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-39.py +++ b/tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-39.py @@ -33,8 +33,7 @@ async def forwards(): "default": Decimal("0"), "digits": (5, 2), "null": False, - "primary": False, - "key": False, + "primary_key": False, "unique": False, "index": False, }, @@ -51,8 +50,7 @@ async def forwards(): "on_update": OnUpdate.cascade, "default": None, "null": True, - "primary": False, - "key": False, + "primary_key": False, "unique": False, "index": False, }, @@ -69,8 +67,7 @@ async def forwards(): "on_update": OnUpdate.cascade, "default": None, "null": True, - "primary": False, - "key": False, + "primary_key": False, "unique": False, "index": False, }, @@ -87,8 +84,7 @@ async def forwards(): "on_update": OnUpdate.cascade, "default": None, "null": True, - "primary": False, - "key": False, + "primary_key": False, "unique": False, "index": False, }, @@ -103,8 +99,7 @@ async def forwards(): "length": 100, "default": "", "null": False, - "primary": False, - "key": False, + "primary_key": False, "unique": False, "index": False, }, @@ -118,8 +113,7 @@ async def forwards(): params={ "default": 0, "null": False, - "primary": False, - "key": False, + "primary_key": False, "unique": False, "index": False, }, diff --git a/tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-44.py b/tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-44.py index 2b4aec32a..52a94fff8 100644 --- a/tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-44.py +++ b/tests/example_apps/music/piccolo_migrations/2020-12-17T18-44-44.py @@ -17,8 +17,7 @@ async def forwards(): params={ "default": "", "null": False, - "primary": False, - "key": False, + "primary_key": False, "unique": False, "index": False, }, diff --git a/tests/example_apps/music/piccolo_migrations/2021-07-25T22-38-48-009306.py b/tests/example_apps/music/piccolo_migrations/2021-07-25T22-38-48-009306.py index 484bd3390..0bddaf7cf 100644 --- a/tests/example_apps/music/piccolo_migrations/2021-07-25T22-38-48-009306.py +++ b/tests/example_apps/music/piccolo_migrations/2021-07-25T22-38-48-009306.py @@ -25,8 +25,7 @@ async def forwards(): "length": 1, "default": "l", "null": False, - "primary": False, - "key": False, + "primary_key": False, "unique": False, "index": False, "index_method": IndexMethod.btree, @@ -45,8 +44,7 @@ async def forwards(): params={ "default": "{}", "null": False, - "primary": False, - "key": False, + "primary_key": False, "unique": False, "index": False, "index_method": IndexMethod.btree, @@ -63,8 +61,7 @@ async def forwards(): params={ "default": "{}", "null": False, - "primary": False, - "key": False, + "primary_key": False, "unique": False, "index": False, "index_method": IndexMethod.btree, From d865af16b99914466a60111f3e3c814606eef9cb Mon Sep 17 00:00:00 2001 From: sinisaos Date: Tue, 3 Jun 2025 22:59:37 +0200 Subject: [PATCH 677/727] Invalid column arguments (#1203) * add invalid column argument checking * remove unnecessary type conversion * remove the hardcoded arguments * change boolean to default argument in test_boolean.py * experiment with TypeDict and Unpack for better autocomplete * added @dantownsend suggestions * move ColumnKwargs and fix digits argument * shift `ColumnKwargs` a bit closer to `Column` --------- Co-authored-by: Daniel Townsend --- piccolo/apps/migrations/auto/serialisation.py | 2 +- piccolo/columns/base.py | 14 +++ piccolo/columns/column_types.py | 101 ++++++++++-------- tests/columns/test_boolean.py | 2 +- 4 files changed, 71 insertions(+), 48 deletions(-) diff --git a/piccolo/apps/migrations/auto/serialisation.py b/piccolo/apps/migrations/auto/serialisation.py index c82e4856a..fee9675d5 100644 --- a/piccolo/apps/migrations/auto/serialisation.py +++ b/piccolo/apps/migrations/auto/serialisation.py @@ -61,7 +61,7 @@ def __new__(mcs, name, bases, class_attributes): @staticmethod def get_unique_class_attribute_values( - class_attributes: t.Dict[str, t.Any] + class_attributes: t.Dict[str, t.Any], ) -> t.Set[t.Any]: """ Return class attribute values. diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 8626f4dc6..07d74fd59 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -350,6 +350,20 @@ def __deepcopy__(self, memo) -> ColumnMeta: return self.copy() +class ColumnKwargs(t.TypedDict, total=False): + null: bool + primary_key: bool + unique: bool + index: bool + index_method: IndexMethod + required: bool + help_text: str + choices: t.Type[Enum] + db_column_name: str + secret: bool + auto_update: t.Any + + class Column(Selectable): """ All other columns inherit from ``Column``. Don't use it directly. diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index e80fb254a..2cef6625d 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -35,8 +35,11 @@ class Band(Table): from datetime import date, datetime, time, timedelta from enum import Enum +from typing_extensions import Unpack + from piccolo.columns.base import ( Column, + ColumnKwargs, ForeignKeyMeta, OnDelete, OnUpdate, @@ -317,14 +320,13 @@ def __init__( self, length: t.Optional[int] = 255, default: t.Union[str, Enum, t.Callable[[], str], None] = "", - **kwargs, + **kwargs: Unpack[ColumnKwargs], ) -> None: self._validate_default(default, (str, None)) self.length = length self.default = default - kwargs.update({"length": length, "default": default}) - super().__init__(**kwargs) + super().__init__(length=length, default=default, **kwargs) @property def column_type(self): @@ -426,12 +428,11 @@ class Band(Table): def __init__( self, default: t.Union[str, Enum, None, t.Callable[[], str]] = "", - **kwargs, + **kwargs: Unpack[ColumnKwargs], ) -> None: self._validate_default(default, (str, None)) self.default = default - kwargs.update({"default": default}) - super().__init__(**kwargs) + super().__init__(default=default, **kwargs) ########################################################################### # For update queries @@ -490,7 +491,11 @@ class Band(Table): value_type = uuid.UUID - def __init__(self, default: UUIDArg = UUID4(), **kwargs) -> None: + def __init__( + self, + default: UUIDArg = UUID4(), + **kwargs: Unpack[ColumnKwargs], + ) -> None: if default is UUID4: # In case the class is passed in, instead of an instance. default = UUID4() @@ -509,8 +514,7 @@ def __init__(self, default: UUIDArg = UUID4(), **kwargs) -> None: ) from e self.default = default - kwargs.update({"default": default}) - super().__init__(**kwargs) + super().__init__(default=default, **kwargs) ########################################################################### # Descriptors @@ -553,12 +557,11 @@ class Band(Table): def __init__( self, default: t.Union[int, Enum, t.Callable[[], int], None] = 0, - **kwargs, + **kwargs: Unpack[ColumnKwargs], ) -> None: self._validate_default(default, (int, None)) self.default = default - kwargs.update({"default": default}) - super().__init__(**kwargs) + super().__init__(default=default, **kwargs) ########################################################################### # For update queries @@ -833,7 +836,10 @@ def __set__(self, obj, value: t.Union[int, None]): class PrimaryKey(Serial): - def __init__(self, **kwargs) -> None: + def __init__( + self, + **kwargs: Unpack[ColumnKwargs], + ) -> None: # Set the index to False, as a database should automatically create # an index for a PrimaryKey column. kwargs.update({"primary_key": True, "index": False}) @@ -896,7 +902,9 @@ class Concert(Table): timedelta_delegate = TimedeltaDelegate() def __init__( - self, default: TimestampArg = TimestampNow(), **kwargs + self, + default: TimestampArg = TimestampNow(), + **kwargs: Unpack[ColumnKwargs], ) -> None: self._validate_default(default, TimestampArg.__args__) # type: ignore @@ -912,8 +920,7 @@ def __init__( default = TimestampNow() self.default = default - kwargs.update({"default": default}) - super().__init__(**kwargs) + super().__init__(default=default, **kwargs) ########################################################################### # For update queries @@ -994,7 +1001,9 @@ class Concert(Table): timedelta_delegate = TimedeltaDelegate() def __init__( - self, default: TimestamptzArg = TimestamptzNow(), **kwargs + self, + default: TimestamptzArg = TimestamptzNow(), + **kwargs: Unpack[ColumnKwargs], ) -> None: self._validate_default( default, TimestamptzArg.__args__ # type: ignore @@ -1007,8 +1016,7 @@ def __init__( default = TimestamptzNow() self.default = default - kwargs.update({"default": default}) - super().__init__(**kwargs) + super().__init__(default=default, **kwargs) ########################################################################### # For update queries @@ -1075,7 +1083,11 @@ class Concert(Table): value_type = date timedelta_delegate = TimedeltaDelegate() - def __init__(self, default: DateArg = DateNow(), **kwargs) -> None: + def __init__( + self, + default: DateArg = DateNow(), + **kwargs: Unpack[ColumnKwargs], + ) -> None: self._validate_default(default, DateArg.__args__) # type: ignore if isinstance(default, date): @@ -1085,8 +1097,7 @@ def __init__(self, default: DateArg = DateNow(), **kwargs) -> None: default = DateNow() self.default = default - kwargs.update({"default": default}) - super().__init__(**kwargs) + super().__init__(default=default, **kwargs) ########################################################################### # For update queries @@ -1153,15 +1164,18 @@ class Concert(Table): value_type = time timedelta_delegate = TimedeltaDelegate() - def __init__(self, default: TimeArg = TimeNow(), **kwargs) -> None: + def __init__( + self, + default: TimeArg = TimeNow(), + **kwargs: Unpack[ColumnKwargs], + ) -> None: self._validate_default(default, TimeArg.__args__) # type: ignore if isinstance(default, time): default = TimeCustom.from_time(default) self.default = default - kwargs.update({"default": default}) - super().__init__(**kwargs) + super().__init__(default=default, **kwargs) ########################################################################### # For update queries @@ -1229,7 +1243,9 @@ class Concert(Table): timedelta_delegate = TimedeltaDelegate() def __init__( - self, default: IntervalArg = IntervalCustom(), **kwargs + self, + default: IntervalArg = IntervalCustom(), + **kwargs: Unpack[ColumnKwargs], ) -> None: self._validate_default(default, IntervalArg.__args__) # type: ignore @@ -1237,8 +1253,7 @@ def __init__( default = IntervalCustom.from_timedelta(default) self.default = default - kwargs.update({"default": default}) - super().__init__(**kwargs) + super().__init__(default=default, **kwargs) @property def column_type(self): @@ -1319,12 +1334,11 @@ class Band(Table): def __init__( self, default: t.Union[bool, Enum, t.Callable[[], bool], None] = False, - **kwargs, + **kwargs: Unpack[ColumnKwargs], ) -> None: self._validate_default(default, (bool, None)) self.default = default - kwargs.update({"default": default}) - super().__init__(**kwargs) + super().__init__(default=default, **kwargs) def eq(self, value) -> Where: """ @@ -1448,7 +1462,7 @@ def __init__( default: t.Union[ decimal.Decimal, Enum, t.Callable[[], decimal.Decimal], None ] = decimal.Decimal(0.0), - **kwargs, + **kwargs: Unpack[ColumnKwargs], ) -> None: if isinstance(digits, tuple): if len(digits) != 2: @@ -1464,8 +1478,7 @@ def __init__( self.default = default self.digits = digits - kwargs.update({"default": default, "digits": digits}) - super().__init__(**kwargs) + super().__init__(default=default, digits=digits, **kwargs) ########################################################################### # Descriptors @@ -1530,12 +1543,11 @@ class Concert(Table): def __init__( self, default: t.Union[float, Enum, t.Callable[[], float], None] = 0.0, - **kwargs, + **kwargs: Unpack[ColumnKwargs], ) -> None: self._validate_default(default, (float, None)) self.default = default - kwargs.update({"default": default}) - super().__init__(**kwargs) + super().__init__(default=default, **kwargs) ########################################################################### # Descriptors @@ -2302,7 +2314,7 @@ def __init__( t.Callable[[], t.Union[str, t.List, t.Dict]], None, ] = "{}", - **kwargs, + **kwargs: Unpack[ColumnKwargs], ) -> None: self._validate_default(default, (str, list, dict, None)) @@ -2310,8 +2322,7 @@ def __init__( default = dump_json(default) self.default = default - kwargs.update({"default": default}) - super().__init__(**kwargs) + super().__init__(default=default, **kwargs) self.json_operator: t.Optional[str] = None @@ -2486,7 +2497,7 @@ def __init__( t.Callable[[], bytearray], None, ] = b"", - **kwargs, + **kwargs: Unpack[ColumnKwargs], ) -> None: self._validate_default(default, (bytes, bytearray, None)) @@ -2494,8 +2505,7 @@ def __init__( default = bytes(default) self.default = default - kwargs.update({"default": default}) - super().__init__(**kwargs) + super().__init__(default=default, **kwargs) ########################################################################### # Descriptors @@ -2587,7 +2597,7 @@ def __init__( default: t.Union[ t.List, Enum, t.Callable[[], t.List], None ] = ListProxy(), - **kwargs, + **kwargs: Unpack[ColumnKwargs], ) -> None: if isinstance(base_column, ForeignKey): raise ValueError("Arrays of ForeignKeys aren't allowed.") @@ -2613,8 +2623,7 @@ def __init__( self.base_column = base_column self.default = default self.index: t.Optional[int] = None - kwargs.update({"base_column": base_column, "default": default}) - super().__init__(**kwargs) + super().__init__(default=default, base_column=base_column, **kwargs) @property def column_type(self): diff --git a/tests/columns/test_boolean.py b/tests/columns/test_boolean.py index 08f2a504b..1e1efc5c2 100644 --- a/tests/columns/test_boolean.py +++ b/tests/columns/test_boolean.py @@ -6,7 +6,7 @@ class MyTable(Table): - boolean = Boolean(boolean=False, null=True) + boolean = Boolean(default=False, null=True) class TestBoolean(TableTest): From 433dabfd1521cdac4521146196a7398fb8ae614e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 3 Jun 2025 22:07:17 +0100 Subject: [PATCH 678/727] bumped version --- CHANGES.rst | 16 ++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index ea0dd3b4c..9bb48b1c1 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,22 @@ Changes ======= +1.27.0 +------ + +Improved auto completion / typo detection for column arguments. + +For example: + +.. code-block:: bash + + class Band(Table): + name = Varchar(nul=True) # linters will now warn that nul is a typo (should be null) + +Thanks to @sinisaos for this. + +------------------------------------------------------------------------------- + 1.26.1 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 093399c19..55e15f103 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.26.1" +__VERSION__ = "1.27.0" From 00017f0065a0a49d32cd5967171007f653c7b15f Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 3 Jun 2025 22:22:22 +0100 Subject: [PATCH 679/727] Update CHANGES.rst --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 9bb48b1c1..1570b0641 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,7 +8,7 @@ Improved auto completion / typo detection for column arguments. For example: -.. code-block:: bash +.. code-block:: python class Band(Table): name = Varchar(nul=True) # linters will now warn that nul is a typo (should be null) From ec2d27b50ea40cba379f511280a15fffd6864eae Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 5 Jun 2025 23:36:35 +0100 Subject: [PATCH 680/727] make some `ColumnKwargs` attributes optional (#1207) --- piccolo/columns/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 07d74fd59..4b80a9ff6 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -357,9 +357,9 @@ class ColumnKwargs(t.TypedDict, total=False): index: bool index_method: IndexMethod required: bool - help_text: str - choices: t.Type[Enum] - db_column_name: str + help_text: t.Optional[str] + choices: t.Optional[t.Type[Enum]] + db_column_name: t.Optional[str] secret: bool auto_update: t.Any From 8d97ab27abf265292bb4925b41f7f68d5e0d3137 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 5 Jun 2025 23:38:22 +0100 Subject: [PATCH 681/727] bumped version --- CHANGES.rst | 8 ++++++++ piccolo/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 1570b0641..4f42f433d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,14 @@ Changes ======= +1.27.1 +------ + +Improve the type annotations in ``ColumnKwargs`` - make some optional. Thanks +to @stronk7 and @sinisaos for their help with this. + +------------------------------------------------------------------------------- + 1.27.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 55e15f103..4e863d10e 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.27.0" +__VERSION__ = "1.27.1" From 7ee7538e9132c12eb641a95b0a2d7cc48925540a Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 5 Jun 2025 23:38:52 +0100 Subject: [PATCH 682/727] Update CHANGES.rst --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 4f42f433d..ebdb631b1 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,7 +4,7 @@ Changes 1.27.1 ------ -Improve the type annotations in ``ColumnKwargs`` - make some optional. Thanks +Improved the type annotations in ``ColumnKwargs`` - made some optional. Thanks to @stronk7 and @sinisaos for their help with this. ------------------------------------------------------------------------------- From 73651e0e118056bf1adbde6aed3cf167cb98ea66 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Tue, 10 Jun 2025 00:03:06 +0200 Subject: [PATCH 683/727] Improve type hints (#1209) * type hint update for Python 3.9 and later * remove unnecessary commented line --- piccolo/apps/app/commands/new.py | 6 +- piccolo/apps/asgi/commands/new.py | 3 +- piccolo/apps/fixtures/commands/dump.py | 16 +- piccolo/apps/fixtures/commands/load.py | 10 +- piccolo/apps/fixtures/commands/shared.py | 18 +- .../apps/migrations/auto/diffable_table.py | 24 +- .../apps/migrations/auto/migration_manager.py | 125 +++---- piccolo/apps/migrations/auto/operations.py | 28 +- piccolo/apps/migrations/auto/schema_differ.py | 69 ++-- .../apps/migrations/auto/schema_snapshot.py | 7 +- piccolo/apps/migrations/auto/serialisation.py | 49 +-- .../migrations/auto/serialisation_legacy.py | 4 +- piccolo/apps/migrations/commands/backwards.py | 3 +- piccolo/apps/migrations/commands/base.py | 24 +- piccolo/apps/migrations/commands/check.py | 5 +- piccolo/apps/migrations/commands/clean.py | 6 +- piccolo/apps/migrations/commands/forwards.py | 3 +- piccolo/apps/migrations/commands/new.py | 12 +- piccolo/apps/migrations/tables.py | 6 +- piccolo/apps/schema/commands/generate.py | 98 ++--- piccolo/apps/schema/commands/graph.py | 10 +- piccolo/apps/shell/commands/run.py | 3 +- piccolo/apps/sql_shell/commands/run.py | 8 +- piccolo/apps/tester/commands/run.py | 6 +- .../apps/user/commands/change_permissions.py | 12 +- piccolo/apps/user/commands/create.py | 14 +- piccolo/apps/user/commands/list.py | 4 +- piccolo/apps/user/tables.py | 16 +- piccolo/columns/base.py | 97 ++--- piccolo/columns/choices.py | 4 +- piccolo/columns/column_types.py | 338 +++++++++--------- piccolo/columns/combination.py | 22 +- piccolo/columns/defaults/base.py | 8 +- piccolo/columns/defaults/date.py | 7 +- piccolo/columns/defaults/interval.py | 7 +- piccolo/columns/defaults/time.py | 7 +- piccolo/columns/defaults/timestamp.py | 7 +- piccolo/columns/defaults/timestamptz.py | 7 +- piccolo/columns/defaults/uuid.py | 5 +- piccolo/columns/m2m.py | 37 +- piccolo/columns/readable.py | 7 +- piccolo/columns/reference.py | 18 +- piccolo/conf/apps.py | 107 +++--- piccolo/custom_types.py | 13 +- piccolo/engine/base.py | 28 +- piccolo/engine/cockroach.py | 9 +- piccolo/engine/finder.py | 4 +- piccolo/engine/postgres.py | 39 +- piccolo/engine/sqlite.py | 45 +-- piccolo/query/base.py | 59 +-- piccolo/query/functions/aggregate.py | 7 +- piccolo/query/functions/base.py | 6 +- piccolo/query/functions/datetime.py | 44 +-- piccolo/query/functions/string.py | 8 +- piccolo/query/functions/type_conversion.py | 8 +- piccolo/query/methods/alter.py | 93 ++--- piccolo/query/methods/count.py | 21 +- piccolo/query/methods/create.py | 11 +- piccolo/query/methods/create_index.py | 17 +- piccolo/query/methods/delete.py | 13 +- piccolo/query/methods/drop_index.py | 13 +- piccolo/query/methods/exists.py | 11 +- piccolo/query/methods/indexes.py | 8 +- piccolo/query/methods/insert.py | 35 +- piccolo/query/methods/objects.py | 110 +++--- piccolo/query/methods/raw.py | 13 +- piccolo/query/methods/refresh.py | 15 +- piccolo/query/methods/select.py | 105 +++--- piccolo/query/methods/table_exists.py | 10 +- piccolo/query/methods/update.py | 15 +- piccolo/query/mixins.py | 117 +++--- piccolo/query/operators/json.py | 22 +- piccolo/query/proxy.py | 17 +- piccolo/querystring.py | 29 +- piccolo/schema.py | 20 +- piccolo/table.py | 187 +++++----- piccolo/table_reflection.py | 18 +- piccolo/testing/model_builder.py | 23 +- piccolo/testing/random_builder.py | 4 +- piccolo/testing/test_case.py | 8 +- piccolo/utils/dictionary.py | 6 +- piccolo/utils/encoding.py | 10 +- piccolo/utils/lazy_loader.py | 6 +- piccolo/utils/list.py | 15 +- piccolo/utils/objects.py | 10 +- piccolo/utils/pydantic.py | 45 ++- piccolo/utils/sql_values.py | 6 +- piccolo/utils/sync.py | 7 +- piccolo/utils/warnings.py | 3 +- setup.py | 5 +- .../apps/asgi/commands/files/dummy_server.py | 7 +- .../apps/fixtures/commands/test_dump_load.py | 3 +- .../auto/integration/test_migrations.py | 11 +- .../migrations/auto/test_migration_manager.py | 4 +- .../migrations/auto/test_schema_differ.py | 45 ++- .../commands/test_forwards_backwards.py | 6 +- tests/apps/schema/commands/test_generate.py | 6 +- tests/base.py | 8 +- tests/columns/m2m/base.py | 4 +- tests/columns/test_boolean.py | 6 +- tests/columns/test_db_column_name.py | 4 +- tests/engine/test_extra_nodes.py | 4 +- tests/engine/test_pool.py | 6 +- tests/engine/test_transaction.py | 8 +- tests/query/test_freeze.py | 8 +- tests/table/instance/test_get_related.py | 4 +- tests/table/test_alter.py | 8 +- tests/table/test_indexes.py | 3 +- tests/table/test_refresh.py | 4 +- tests/table/test_update.py | 6 +- tests/testing/test_model_builder.py | 3 +- tests/type_checking.py | 52 +-- tests/utils/test_pydantic.py | 72 ++-- tests/utils/test_table_reflection.py | 3 +- 114 files changed, 1451 insertions(+), 1419 deletions(-) diff --git a/piccolo/apps/app/commands/new.py b/piccolo/apps/app/commands/new.py index e35a7f7db..c8beee26a 100644 --- a/piccolo/apps/app/commands/new.py +++ b/piccolo/apps/app/commands/new.py @@ -5,7 +5,7 @@ import pathlib import string import sys -import typing as t +from typing import Any import black import jinja2 @@ -46,7 +46,7 @@ def validate_app_name(app_name: str): """ for char in app_name: - if not char.lower() in APP_NAME_ALLOWED_CHARACTERS: + if char.lower() not in APP_NAME_ALLOWED_CHARACTERS: raise ValueError( f"The app name contains a disallowed character: `{char}`. " "It must only include a-z, 0-9, and _ characters." @@ -81,7 +81,7 @@ def new_app(app_name: str, root: str = ".", register: bool = False): with open(os.path.join(app_root, "__init__.py"), "w"): pass - templates: t.Dict[str, t.Any] = { + templates: dict[str, Any] = { "piccolo_app.py": {"app_name": app_name}, "tables.py": {}, } diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index dc3c75fb4..4db7ee2a3 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -2,7 +2,6 @@ import os import shutil -import typing as t import black import colorama @@ -28,7 +27,7 @@ def print_instruction(message: str): print(f"{colorama.Fore.CYAN}{message}{colorama.Fore.RESET}") -def get_options_string(options: t.List[str]): +def get_options_string(options: list[str]): return ", ".join(f"{name} [{index}]" for index, name in enumerate(options)) diff --git a/piccolo/apps/fixtures/commands/dump.py b/piccolo/apps/fixtures/commands/dump.py index 4d78eadba..1114a6c86 100644 --- a/piccolo/apps/fixtures/commands/dump.py +++ b/piccolo/apps/fixtures/commands/dump.py @@ -1,6 +1,6 @@ from __future__ import annotations -import typing as t +from typing import Any, Optional from piccolo.apps.fixtures.commands.shared import ( FixtureConfig, @@ -11,8 +11,8 @@ async def get_dump( - fixture_configs: t.List[FixtureConfig], -) -> t.Dict[str, t.Any]: + fixture_configs: list[FixtureConfig], +) -> dict[str, Any]: """ Gets the data for each table specified and returns a data structure like: @@ -32,7 +32,7 @@ async def get_dump( """ finder = Finder() - output: t.Dict[str, t.Any] = {} + output: dict[str, Any] = {} for fixture_config in fixture_configs: app_config = finder.get_app_config(app_name=fixture_config.app_name) @@ -53,7 +53,7 @@ async def get_dump( async def dump_to_json_string( - fixture_configs: t.List[FixtureConfig], + fixture_configs: list[FixtureConfig], ) -> str: """ Dumps all of the data for the given tables into a JSON string. @@ -65,7 +65,7 @@ async def dump_to_json_string( return pydantic_model(**dump).model_dump_json(indent=4) -def parse_args(apps: str, tables: str) -> t.List[FixtureConfig]: +def parse_args(apps: str, tables: str) -> list[FixtureConfig]: """ Works out which apps and tables the user is referring to. """ @@ -80,11 +80,11 @@ def parse_args(apps: str, tables: str) -> t.List[FixtureConfig]: # Must be a single app name app_names.append(apps) - table_class_names: t.Optional[t.List[str]] = None + table_class_names: Optional[list[str]] = None if tables != "all": table_class_names = tables.split(",") if "," in tables else [tables] - output: t.List[FixtureConfig] = [] + output: list[FixtureConfig] = [] for app_name in app_names: app_config = finder.get_app_config(app_name=app_name) diff --git a/piccolo/apps/fixtures/commands/load.py b/piccolo/apps/fixtures/commands/load.py index 64e4c3334..835f814a9 100644 --- a/piccolo/apps/fixtures/commands/load.py +++ b/piccolo/apps/fixtures/commands/load.py @@ -1,7 +1,7 @@ from __future__ import annotations import sys -import typing as t +from typing import Optional import typing_extensions @@ -20,7 +20,7 @@ async def load_json_string( json_string: str, chunk_size: int = 1000, - on_conflict_action: t.Optional[OnConflictAction] = None, + on_conflict_action: Optional[OnConflictAction] = None, ): """ Parses the JSON string, and inserts the parsed data into the database. @@ -55,7 +55,7 @@ async def load_json_string( raise Exception("Unable to find the engine.") # This is what we want to the insert into the database: - data: t.Dict[t.Type[Table], t.List[Table]] = {} + data: dict[type[Table], list[Table]] = {} for app_name in app_names: app_model = getattr(fixture_pydantic_model, app_name) @@ -94,7 +94,7 @@ async def load_json_string( async def load( path: str = "fixture.json", chunk_size: int = 1000, - on_conflict: t.Optional[ + on_conflict: Optional[ typing_extensions.Literal["DO NOTHING", "DO UPDATE"] ] = None, ): @@ -118,7 +118,7 @@ async def load( with open(path, "r") as f: contents = f.read() - on_conflict_action: t.Optional[OnConflictAction] = None + on_conflict_action: Optional[OnConflictAction] = None if on_conflict: try: diff --git a/piccolo/apps/fixtures/commands/shared.py b/piccolo/apps/fixtures/commands/shared.py index 99915a85a..d2d67a819 100644 --- a/piccolo/apps/fixtures/commands/shared.py +++ b/piccolo/apps/fixtures/commands/shared.py @@ -1,42 +1,42 @@ from __future__ import annotations -import typing as t from dataclasses import dataclass +from typing import TYPE_CHECKING, Any import pydantic from piccolo.conf.apps import Finder from piccolo.utils.pydantic import create_pydantic_model -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from piccolo.table import Table @dataclass class FixtureConfig: app_name: str - table_class_names: t.List[str] + table_class_names: list[str] -def create_pydantic_fixture_model(fixture_configs: t.List[FixtureConfig]): +def create_pydantic_fixture_model(fixture_configs: list[FixtureConfig]): """ Returns a nested Pydantic model for serialising and deserialising fixtures. """ - columns: t.Dict[str, t.Any] = {} + columns: dict[str, Any] = {} finder = Finder() for fixture_config in fixture_configs: - app_columns: t.Dict[str, t.Any] = {} + app_columns: dict[str, Any] = {} for table_class_name in fixture_config.table_class_names: - table_class: t.Type[Table] = finder.get_table_with_name( + table_class: type[Table] = finder.get_table_with_name( app_name=fixture_config.app_name, table_class_name=table_class_name, ) app_columns[table_class_name] = ( - t.List[ # type: ignore + list[ # type: ignore create_pydantic_model( table_class, include_default_columns=True ) @@ -44,7 +44,7 @@ def create_pydantic_fixture_model(fixture_configs: t.List[FixtureConfig]): ..., ) - app_model: t.Any = pydantic.create_model( + app_model: Any = pydantic.create_model( f"{fixture_config.app_name.title()}Model", **app_columns ) diff --git a/piccolo/apps/migrations/auto/diffable_table.py b/piccolo/apps/migrations/auto/diffable_table.py index 522f4f001..a3e80500f 100644 --- a/piccolo/apps/migrations/auto/diffable_table.py +++ b/piccolo/apps/migrations/auto/diffable_table.py @@ -1,7 +1,7 @@ from __future__ import annotations -import typing as t from dataclasses import dataclass, field +from typing import Any, Optional from piccolo.apps.migrations.auto.operations import ( AddColumn, @@ -17,8 +17,8 @@ def compare_dicts( - dict_1: t.Dict[str, t.Any], dict_2: t.Dict[str, t.Any] -) -> t.Dict[str, t.Any]: + dict_1: dict[str, Any], dict_2: dict[str, Any] +) -> dict[str, Any]: """ Returns a new dictionary which only contains key, value pairs which are in the first dictionary and not the second. @@ -59,9 +59,9 @@ def compare_dicts( @dataclass class TableDelta: - add_columns: t.List[AddColumn] = field(default_factory=list) - drop_columns: t.List[DropColumn] = field(default_factory=list) - alter_columns: t.List[AlterColumn] = field(default_factory=list) + add_columns: list[AddColumn] = field(default_factory=list) + drop_columns: list[DropColumn] = field(default_factory=list) + alter_columns: list[AlterColumn] = field(default_factory=list) def __eq__(self, value: TableDelta) -> bool: # type: ignore """ @@ -101,12 +101,12 @@ class DiffableTable: class_name: str tablename: str - schema: t.Optional[str] = None - columns: t.List[Column] = field(default_factory=list) - previous_class_name: t.Optional[str] = None + schema: Optional[str] = None + columns: list[Column] = field(default_factory=list) + previous_class_name: Optional[str] = None def __post_init__(self) -> None: - self.columns_map: t.Dict[str, Column] = { + self.columns_map: dict[str, Column] = { i._meta.name: i for i in self.columns } @@ -163,7 +163,7 @@ def __sub__(self, value: DiffableTable) -> TableDelta: ####################################################################### - alter_columns: t.List[AlterColumn] = [] + alter_columns: list[AlterColumn] = [] for existing_column in value.columns: column = self.columns_map.get(existing_column._meta.name) @@ -221,7 +221,7 @@ def __eq__(self, value) -> bool: def __str__(self): return f"{self.class_name} - {self.tablename}" - def to_table_class(self) -> t.Type[Table]: + def to_table_class(self) -> type[Table]: """ Converts the DiffableTable into a Table subclass. """ diff --git a/piccolo/apps/migrations/auto/migration_manager.py b/piccolo/apps/migrations/auto/migration_manager.py index fc7449ccb..53f4a3a7c 100644 --- a/piccolo/apps/migrations/auto/migration_manager.py +++ b/piccolo/apps/migrations/auto/migration_manager.py @@ -2,8 +2,9 @@ import inspect import logging -import typing as t +from collections.abc import Callable, Coroutine from dataclasses import dataclass, field +from typing import Any, Optional, Union from piccolo.apps.migrations.auto.diffable_table import DiffableTable from piccolo.apps.migrations.auto.operations import ( @@ -32,19 +33,19 @@ class AddColumnClass: column: Column table_class_name: str tablename: str - schema: t.Optional[str] + schema: Optional[str] @dataclass class AddColumnCollection: - add_columns: t.List[AddColumnClass] = field(default_factory=list) + add_columns: list[AddColumnClass] = field(default_factory=list) def append(self, add_column: AddColumnClass): self.add_columns.append(add_column) def for_table_class_name( self, table_class_name: str - ) -> t.List[AddColumnClass]: + ) -> list[AddColumnClass]: return [ i for i in self.add_columns @@ -53,7 +54,7 @@ def for_table_class_name( def columns_for_table_class_name( self, table_class_name: str - ) -> t.List[Column]: + ) -> list[Column]: return [ i.column for i in self.add_columns @@ -61,20 +62,18 @@ def columns_for_table_class_name( ] @property - def table_class_names(self) -> t.List[str]: + def table_class_names(self) -> list[str]: return list({i.table_class_name for i in self.add_columns}) @dataclass class DropColumnCollection: - drop_columns: t.List[DropColumn] = field(default_factory=list) + drop_columns: list[DropColumn] = field(default_factory=list) def append(self, drop_column: DropColumn): self.drop_columns.append(drop_column) - def for_table_class_name( - self, table_class_name: str - ) -> t.List[DropColumn]: + def for_table_class_name(self, table_class_name: str) -> list[DropColumn]: return [ i for i in self.drop_columns @@ -82,20 +81,20 @@ def for_table_class_name( ] @property - def table_class_names(self) -> t.List[str]: + def table_class_names(self) -> list[str]: return list({i.table_class_name for i in self.drop_columns}) @dataclass class RenameColumnCollection: - rename_columns: t.List[RenameColumn] = field(default_factory=list) + rename_columns: list[RenameColumn] = field(default_factory=list) def append(self, rename_column: RenameColumn): self.rename_columns.append(rename_column) def for_table_class_name( self, table_class_name: str - ) -> t.List[RenameColumn]: + ) -> list[RenameColumn]: return [ i for i in self.rename_columns @@ -103,20 +102,18 @@ def for_table_class_name( ] @property - def table_class_names(self) -> t.List[str]: + def table_class_names(self) -> list[str]: return list({i.table_class_name for i in self.rename_columns}) @dataclass class AlterColumnCollection: - alter_columns: t.List[AlterColumn] = field(default_factory=list) + alter_columns: list[AlterColumn] = field(default_factory=list) def append(self, alter_column: AlterColumn): self.alter_columns.append(alter_column) - def for_table_class_name( - self, table_class_name: str - ) -> t.List[AlterColumn]: + def for_table_class_name(self, table_class_name: str) -> list[AlterColumn]: return [ i for i in self.alter_columns @@ -124,11 +121,11 @@ def for_table_class_name( ] @property - def table_class_names(self) -> t.List[str]: + def table_class_names(self) -> list[str]: return list({i.table_class_name for i in self.alter_columns}) -AsyncFunction = t.Callable[[], t.Coroutine] +AsyncFunction = Callable[[], Coroutine] class SkippedTransaction: @@ -158,12 +155,10 @@ class MigrationManager: app_name: str = "" description: str = "" preview: bool = False - add_tables: t.List[DiffableTable] = field(default_factory=list) - drop_tables: t.List[DiffableTable] = field(default_factory=list) - rename_tables: t.List[RenameTable] = field(default_factory=list) - change_table_schemas: t.List[ChangeTableSchema] = field( - default_factory=list - ) + add_tables: list[DiffableTable] = field(default_factory=list) + drop_tables: list[DiffableTable] = field(default_factory=list) + rename_tables: list[RenameTable] = field(default_factory=list) + change_table_schemas: list[ChangeTableSchema] = field(default_factory=list) add_columns: AddColumnCollection = field( default_factory=AddColumnCollection ) @@ -176,10 +171,8 @@ class MigrationManager: alter_columns: AlterColumnCollection = field( default_factory=AlterColumnCollection ) - raw: t.List[t.Union[t.Callable, AsyncFunction]] = field( - default_factory=list - ) - raw_backwards: t.List[t.Union[t.Callable, AsyncFunction]] = field( + raw: list[Union[Callable, AsyncFunction]] = field(default_factory=list) + raw_backwards: list[Union[Callable, AsyncFunction]] = field( default_factory=list ) fake: bool = False @@ -189,8 +182,8 @@ def add_table( self, class_name: str, tablename: str, - schema: t.Optional[str] = None, - columns: t.Optional[t.List[Column]] = None, + schema: Optional[str] = None, + columns: Optional[list[Column]] = None, ): if not columns: columns = [] @@ -205,7 +198,7 @@ def add_table( ) def drop_table( - self, class_name: str, tablename: str, schema: t.Optional[str] = None + self, class_name: str, tablename: str, schema: Optional[str] = None ): self.drop_tables.append( DiffableTable( @@ -217,8 +210,8 @@ def change_table_schema( self, class_name: str, tablename: str, - new_schema: t.Optional[str] = None, - old_schema: t.Optional[str] = None, + new_schema: Optional[str] = None, + old_schema: Optional[str] = None, ): self.change_table_schemas.append( ChangeTableSchema( @@ -235,7 +228,7 @@ def rename_table( old_tablename: str, new_class_name: str, new_tablename: str, - schema: t.Optional[str] = None, + schema: Optional[str] = None, ): self.rename_tables.append( RenameTable( @@ -252,11 +245,11 @@ def add_column( table_class_name: str, tablename: str, column_name: str, - db_column_name: t.Optional[str] = None, + db_column_name: Optional[str] = None, column_class_name: str = "", - column_class: t.Optional[t.Type[Column]] = None, - params: t.Optional[t.Dict[str, t.Any]] = None, - schema: t.Optional[str] = None, + column_class: Optional[type[Column]] = None, + params: Optional[dict[str, Any]] = None, + schema: Optional[str] = None, ): """ Add a new column to the table. @@ -297,8 +290,8 @@ def drop_column( table_class_name: str, tablename: str, column_name: str, - db_column_name: t.Optional[str] = None, - schema: t.Optional[str] = None, + db_column_name: Optional[str] = None, + schema: Optional[str] = None, ): self.drop_columns.append( DropColumn( @@ -316,9 +309,9 @@ def rename_column( tablename: str, old_column_name: str, new_column_name: str, - old_db_column_name: t.Optional[str] = None, - new_db_column_name: t.Optional[str] = None, - schema: t.Optional[str] = None, + old_db_column_name: Optional[str] = None, + new_db_column_name: Optional[str] = None, + schema: Optional[str] = None, ): self.rename_columns.append( RenameColumn( @@ -337,12 +330,12 @@ def alter_column( table_class_name: str, tablename: str, column_name: str, - db_column_name: t.Optional[str] = None, - params: t.Optional[t.Dict[str, t.Any]] = None, - old_params: t.Optional[t.Dict[str, t.Any]] = None, - column_class: t.Optional[t.Type[Column]] = None, - old_column_class: t.Optional[t.Type[Column]] = None, - schema: t.Optional[str] = None, + db_column_name: Optional[str] = None, + params: Optional[dict[str, Any]] = None, + old_params: Optional[dict[str, Any]] = None, + column_class: Optional[type[Column]] = None, + old_column_class: Optional[type[Column]] = None, + schema: Optional[str] = None, ): """ All possible alterations aren't currently supported. @@ -365,14 +358,14 @@ def alter_column( ) ) - def add_raw(self, raw: t.Union[t.Callable, AsyncFunction]): + def add_raw(self, raw: Union[Callable, AsyncFunction]): """ A migration manager can execute arbitrary functions or coroutines when run. This is useful if you want to execute raw SQL. """ self.raw.append(raw) - def add_raw_backwards(self, raw: t.Union[t.Callable, AsyncFunction]): + def add_raw_backwards(self, raw: Union[Callable, AsyncFunction]): """ When reversing a migration, you may want to run extra code to help clean up. @@ -384,10 +377,10 @@ def add_raw_backwards(self, raw: t.Union[t.Callable, AsyncFunction]): async def get_table_from_snapshot( self, table_class_name: str, - app_name: t.Optional[str], + app_name: Optional[str], offset: int = 0, - migration_id: t.Optional[str] = None, - ) -> t.Type[Table]: + migration_id: Optional[str] = None, + ) -> type[Table]: """ Returns a Table subclass which can be used for modifying data within a migration. @@ -416,13 +409,13 @@ async def get_table_from_snapshot( ########################################################################### @staticmethod - async def _print_query(query: t.Union[DDL, Query, SchemaDDLBase]): + async def _print_query(query: Union[DDL, Query, SchemaDDLBase]): if isinstance(query, DDL): print("\n", ";".join(query.ddl) + ";") else: print(str(query)) - async def _run_query(self, query: t.Union[DDL, Query, SchemaDDLBase]): + async def _run_query(self, query: Union[DDL, Query, SchemaDDLBase]): """ If MigrationManager is in preview mode then it just print the query instead of executing it. @@ -441,7 +434,7 @@ async def _run_alter_columns(self, backwards: bool = False): if not alter_columns: continue - _Table: t.Type[Table] = create_table_class( + _Table: type[Table] = create_table_class( class_name=table_class_name, class_kwargs={ "tablename": alter_columns[0].tablename, @@ -494,7 +487,7 @@ async def _run_alter_columns(self, backwards: bool = False): alter_column.db_column_name ) - using_expression: t.Optional[str] = None + using_expression: Optional[str] = None # Postgres won't automatically cast some types to # others. We may as well try, as it will definitely @@ -739,7 +732,7 @@ async def _run_rename_tables(self, backwards: bool = False): else rename_table.new_tablename ) - _Table: t.Type[Table] = create_table_class( + _Table: type[Table] = create_table_class( class_name=class_name, class_kwargs={ "tablename": tablename, @@ -760,7 +753,7 @@ async def _run_rename_columns(self, backwards: bool = False): if not columns: continue - _Table: t.Type[Table] = create_table_class( + _Table: type[Table] = create_table_class( class_name=table_class_name, class_kwargs={ "tablename": columns[0].tablename, @@ -788,12 +781,12 @@ async def _run_rename_columns(self, backwards: bool = False): ) async def _run_add_tables(self, backwards: bool = False): - table_classes: t.List[t.Type[Table]] = [] + table_classes: list[type[Table]] = [] for add_table in self.add_tables: - add_columns: t.List[AddColumnClass] = ( + add_columns: list[AddColumnClass] = ( self.add_columns.for_table_class_name(add_table.class_name) ) - _Table: t.Type[Table] = create_table_class( + _Table: type[Table] = create_table_class( class_name=add_table.class_name, class_kwargs={ "tablename": add_table.tablename, @@ -845,7 +838,7 @@ async def _run_add_columns(self, backwards: bool = False): if table_class_name in [i.class_name for i in self.add_tables]: continue # No need to add columns to new tables - add_columns: t.List[AddColumnClass] = ( + add_columns: list[AddColumnClass] = ( self.add_columns.for_table_class_name(table_class_name) ) diff --git a/piccolo/apps/migrations/auto/operations.py b/piccolo/apps/migrations/auto/operations.py index 0676bdbd4..84e0d261a 100644 --- a/piccolo/apps/migrations/auto/operations.py +++ b/piccolo/apps/migrations/auto/operations.py @@ -1,5 +1,5 @@ -import typing as t from dataclasses import dataclass +from typing import Any, Optional from piccolo.columns.base import Column @@ -10,15 +10,15 @@ class RenameTable: old_tablename: str new_class_name: str new_tablename: str - schema: t.Optional[str] = None + schema: Optional[str] = None @dataclass class ChangeTableSchema: class_name: str tablename: str - old_schema: t.Optional[str] - new_schema: t.Optional[str] + old_schema: Optional[str] + new_schema: Optional[str] @dataclass @@ -29,7 +29,7 @@ class RenameColumn: new_column_name: str old_db_column_name: str new_db_column_name: str - schema: t.Optional[str] = None + schema: Optional[str] = None @dataclass @@ -38,11 +38,11 @@ class AlterColumn: column_name: str db_column_name: str tablename: str - params: t.Dict[str, t.Any] - old_params: t.Dict[str, t.Any] - column_class: t.Optional[t.Type[Column]] = None - old_column_class: t.Optional[t.Type[Column]] = None - schema: t.Optional[str] = None + params: dict[str, Any] + old_params: dict[str, Any] + column_class: Optional[type[Column]] = None + old_column_class: Optional[type[Column]] = None + schema: Optional[str] = None @dataclass @@ -51,7 +51,7 @@ class DropColumn: column_name: str db_column_name: str tablename: str - schema: t.Optional[str] = None + schema: Optional[str] = None @dataclass @@ -60,6 +60,6 @@ class AddColumn: column_name: str db_column_name: str column_class_name: str - column_class: t.Type[Column] - params: t.Dict[str, t.Any] - schema: t.Optional[str] = None + column_class: type[Column] + params: dict[str, Any] + schema: Optional[str] = None diff --git a/piccolo/apps/migrations/auto/schema_differ.py b/piccolo/apps/migrations/auto/schema_differ.py index 1d095b938..7dbc9a469 100644 --- a/piccolo/apps/migrations/auto/schema_differ.py +++ b/piccolo/apps/migrations/auto/schema_differ.py @@ -1,9 +1,10 @@ from __future__ import annotations import inspect -import typing as t +from collections.abc import Callable from copy import deepcopy from dataclasses import dataclass, field +from typing import Any, Optional from piccolo.apps.migrations.auto.diffable_table import ( DiffableTable, @@ -26,7 +27,7 @@ @dataclass class RenameTableCollection: - rename_tables: t.List[RenameTable] = field(default_factory=list) + rename_tables: list[RenameTable] = field(default_factory=list) def append(self, renamed_table: RenameTable): self.rename_tables.append(renamed_table) @@ -48,7 +49,7 @@ def was_renamed_from(self, old_class_name: str) -> bool: return True return False - def renamed_from(self, new_class_name: str) -> t.Optional[str]: + def renamed_from(self, new_class_name: str) -> Optional[str]: """ Returns the old class name, if it exists. """ @@ -60,7 +61,7 @@ def renamed_from(self, new_class_name: str) -> t.Optional[str]: @dataclass class ChangeTableSchemaCollection: - collection: t.List[ChangeTableSchema] = field(default_factory=list) + collection: list[ChangeTableSchema] = field(default_factory=list) def append(self, change_table_schema: ChangeTableSchema): self.collection.append(change_table_schema) @@ -68,14 +69,14 @@ def append(self, change_table_schema: ChangeTableSchema): @dataclass class RenameColumnCollection: - rename_columns: t.List[RenameColumn] = field(default_factory=list) + rename_columns: list[RenameColumn] = field(default_factory=list) def append(self, rename_column: RenameColumn): self.rename_columns.append(rename_column) def for_table_class_name( self, table_class_name: str - ) -> t.List[RenameColumn]: + ) -> list[RenameColumn]: return [ i for i in self.rename_columns @@ -93,9 +94,9 @@ def new_column_names(self): @dataclass class AlterStatements: - statements: t.List[str] = field(default_factory=list) - extra_imports: t.List[Import] = field(default_factory=list) - extra_definitions: t.List[Definition] = field(default_factory=list) + statements: list[str] = field(default_factory=list) + extra_imports: list[Import] = field(default_factory=list) + extra_definitions: list[Definition] = field(default_factory=list) def extend(self, alter_statements: AlterStatements): self.statements.extend(alter_statements.statements) @@ -112,19 +113,19 @@ class SchemaDiffer: sure - for example, whether a column was renamed. """ - schema: t.List[DiffableTable] - schema_snapshot: t.List[DiffableTable] + schema: list[DiffableTable] + schema_snapshot: list[DiffableTable] # Sometimes the SchemaDiffer requires input from a user - for example, # asking if a table was renamed or not. When running in non-interactive # mode (like in a unittest), we can set a default to be used instead, like # 'y'. - auto_input: t.Optional[str] = None + auto_input: Optional[str] = None ########################################################################### def __post_init__(self) -> None: - self.schema_snapshot_map: t.Dict[str, DiffableTable] = { + self.schema_snapshot_map: dict[str, DiffableTable] = { i.class_name: i for i in self.schema_snapshot } self.table_schema_changes_collection = ( @@ -137,11 +138,11 @@ def check_rename_tables(self) -> RenameTableCollection: """ Work out whether any of the tables were renamed. """ - drop_tables: t.List[DiffableTable] = list( + drop_tables: list[DiffableTable] = list( set(self.schema_snapshot) - set(self.schema) ) - new_tables: t.List[DiffableTable] = list( + new_tables: list[DiffableTable] = list( set(self.schema) - set(self.schema_snapshot) ) @@ -267,7 +268,7 @@ def check_renamed_columns(self) -> RenameColumnCollection: # We track which dropped columns have already been identified by # the user as renames, so we don't ask them if another column # was also renamed from it. - used_drop_column_names: t.List[str] = [] + used_drop_column_names: list[str] = [] for add_column in delta.add_columns: for drop_column in delta.drop_columns: @@ -300,9 +301,9 @@ def check_renamed_columns(self) -> RenameColumnCollection: def _stringify_func( self, - func: t.Callable, - params: t.Dict[str, t.Any], - prefix: t.Optional[str] = None, + func: Callable, + params: dict[str, Any], + prefix: Optional[str] = None, ) -> AlterStatements: """ Generates a string representing how to call the given function with the @@ -348,7 +349,7 @@ def my_callable(arg_1: str, arg_2: str): @property def create_tables(self) -> AlterStatements: - new_tables: t.List[DiffableTable] = list( + new_tables: list[DiffableTable] = list( set(self.schema) - set(self.schema_snapshot) ) @@ -379,7 +380,7 @@ def create_tables(self) -> AlterStatements: @property def drop_tables(self) -> AlterStatements: - drop_tables: t.List[DiffableTable] = list( + drop_tables: list[DiffableTable] = list( set(self.schema_snapshot) - set(self.schema) ) @@ -442,7 +443,7 @@ def change_table_schemas(self) -> AlterStatements: def _get_snapshot_table( self, table_class_name: str - ) -> t.Optional[DiffableTable]: + ) -> Optional[DiffableTable]: snapshot_table = self.schema_snapshot_map.get(table_class_name, None) if snapshot_table: return snapshot_table @@ -463,9 +464,9 @@ def _get_snapshot_table( @property def alter_columns(self) -> AlterStatements: - response: t.List[str] = [] - extra_imports: t.List[Import] = [] - extra_definitions: t.List[Definition] = [] + response: list[str] = [] + extra_imports: list[Import] = [] + extra_definitions: list[Definition] = [] for table in self.schema: snapshot_table = self._get_snapshot_table(table.class_name) if snapshot_table: @@ -563,9 +564,9 @@ def drop_columns(self) -> AlterStatements: @property def add_columns(self) -> AlterStatements: - response: t.List[str] = [] - extra_imports: t.List[Import] = [] - extra_definitions: t.List[Definition] = [] + response: list[str] = [] + extra_imports: list[Import] = [] + extra_definitions: list[Definition] = [] for table in self.schema: snapshot_table = self._get_snapshot_table(table.class_name) if snapshot_table: @@ -632,13 +633,13 @@ def rename_columns(self) -> AlterStatements: @property def new_table_columns(self) -> AlterStatements: - new_tables: t.List[DiffableTable] = list( + new_tables: list[DiffableTable] = list( set(self.schema) - set(self.schema_snapshot) ) - response: t.List[str] = [] - extra_imports: t.List[Import] = [] - extra_definitions: t.List[Definition] = [] + response: list[str] = [] + extra_imports: list[Import] = [] + extra_definitions: list[Definition] = [] for table in new_tables: if ( table.class_name @@ -681,11 +682,11 @@ def new_table_columns(self) -> AlterStatements: ########################################################################### - def get_alter_statements(self) -> t.List[AlterStatements]: + def get_alter_statements(self) -> list[AlterStatements]: """ Call to execute the necessary alter commands on the database. """ - alter_statements: t.Dict[str, AlterStatements] = { + alter_statements: dict[str, AlterStatements] = { "Created tables": self.create_tables, "Dropped tables": self.drop_tables, "Renamed tables": self.rename_tables, diff --git a/piccolo/apps/migrations/auto/schema_snapshot.py b/piccolo/apps/migrations/auto/schema_snapshot.py index 45963b717..5bf343063 100644 --- a/piccolo/apps/migrations/auto/schema_snapshot.py +++ b/piccolo/apps/migrations/auto/schema_snapshot.py @@ -1,6 +1,5 @@ from __future__ import annotations -import typing as t from dataclasses import dataclass, field from piccolo.apps.migrations.auto.diffable_table import DiffableTable @@ -15,7 +14,7 @@ class SchemaSnapshot: """ # In ascending order of date created. - managers: t.List[MigrationManager] = field(default_factory=list) + managers: list[MigrationManager] = field(default_factory=list) ########################################################################### @@ -28,8 +27,8 @@ def get_table_from_snapshot(self, table_class_name: str) -> DiffableTable: ########################################################################### - def get_snapshot(self) -> t.List[DiffableTable]: - tables: t.List[DiffableTable] = [] + def get_snapshot(self) -> list[DiffableTable]: + tables: list[DiffableTable] = [] # Make sure the managers are sorted correctly: sorted_managers = sorted(self.managers, key=lambda x: x.migration_id) diff --git a/piccolo/apps/migrations/auto/serialisation.py b/piccolo/apps/migrations/auto/serialisation.py index fee9675d5..bfafc43d6 100644 --- a/piccolo/apps/migrations/auto/serialisation.py +++ b/piccolo/apps/migrations/auto/serialisation.py @@ -5,12 +5,13 @@ import datetime import decimal import inspect -import typing as t import uuid import warnings +from collections.abc import Callable, Iterable from copy import deepcopy from dataclasses import dataclass, field from enum import Enum +from typing import Any, Optional from piccolo.columns import Column from piccolo.columns.defaults.base import Default @@ -61,8 +62,8 @@ def __new__(mcs, name, bases, class_attributes): @staticmethod def get_unique_class_attribute_values( - class_attributes: t.Dict[str, t.Any], - ) -> t.Set[t.Any]: + class_attributes: dict[str, Any], + ) -> set[Any]: """ Return class attribute values. @@ -87,9 +88,9 @@ def get_unique_class_attribute_values( @staticmethod def merge_class_attributes( - class_attributes1: t.Dict[str, t.Any], - class_attributes2: t.Dict[str, t.Any], - ) -> t.Dict[str, t.Any]: + class_attributes1: dict[str, Any], + class_attributes2: dict[str, Any], + ) -> dict[str, Any]: """ Merges two class attribute dictionaries. @@ -104,12 +105,12 @@ def merge_class_attributes( return dict(**class_attributes1, **class_attributes2) @staticmethod - def get_column_class_attributes() -> t.Dict[str, str]: + def get_column_class_attributes() -> dict[str, str]: """Automatically generates global names for each column type.""" import piccolo.columns.column_types - class_attributes: t.Dict[str, str] = {} + class_attributes: dict[str, str] = {} for module_global in piccolo.columns.column_types.__dict__.values(): try: if module_global is not Column and issubclass( @@ -150,7 +151,7 @@ class to ensure that no conflicts arise during serialisation. EXTERNAL_UUID = f"{EXTERNAL_MODULE_UUID}.{uuid.UUID.__name__}" # This attribute is set in metaclass - unique_names: t.Set[str] + unique_names: set[str] @classmethod def warn_if_is_conflicting_name( @@ -172,7 +173,7 @@ def is_conflicting_name(cls, name: str) -> bool: @staticmethod def warn_if_are_conflicting_objects( - objects: t.Iterable[CanConflictWithGlobalNames], + objects: Iterable[CanConflictWithGlobalNames], ) -> None: """ Call each object's ``raise_if_is_conflicting_with_global_name`` method. @@ -192,8 +193,8 @@ class UniqueGlobalNameConflictWarning(UserWarning): @dataclass class Import(CanConflictWithGlobalNames): module: str - target: t.Optional[str] = None - expect_conflict_with_global_name: t.Optional[str] = None + target: Optional[str] = None + expect_conflict_with_global_name: Optional[str] = None def __post_init__(self) -> None: if ( @@ -256,9 +257,9 @@ def __ge__(self, value): @dataclass class SerialisedParams: - params: t.Dict[str, t.Any] - extra_imports: t.List[Import] - extra_definitions: t.List[Definition] = field(default_factory=list) + params: dict[str, Any] + extra_imports: list[Import] + extra_definitions: list[Definition] = field(default_factory=list) ############################################################################### @@ -273,7 +274,7 @@ def check_equality(self, other): @dataclass class SerialisedBuiltin: - builtin: t.Any + builtin: Any def __hash__(self): return hash(self.builtin.__name__) @@ -335,7 +336,7 @@ def __repr__(self): @dataclass class SerialisedTableType(Definition): - table_type: t.Type[Table] + table_type: type[Table] def __hash__(self): return hash( @@ -361,7 +362,7 @@ def __repr__(self) -> str: # When creating a ForeignKey, the user can specify a column other than # the primary key to reference. - serialised_target_columns: t.Set[SerialisedColumnInstance] = set() + serialised_target_columns: set[SerialisedColumnInstance] = set() for fk_column in self.table_type._meta._foreign_key_references: target_column = fk_column._foreign_key_meta.target_column @@ -426,7 +427,7 @@ def warn_if_is_conflicting_with_global_name(self) -> None: @dataclass class SerialisedEnumType: - enum_type: t.Type[Enum] + enum_type: type[Enum] def __hash__(self): return hash(self.__repr__()) @@ -442,7 +443,7 @@ def __repr__(self): @dataclass class SerialisedCallable: - callable_: t.Callable + callable_: Callable def __hash__(self): return hash(self.callable_.__name__) @@ -487,14 +488,14 @@ def __repr__(self): ############################################################################### -def serialise_params(params: t.Dict[str, t.Any]) -> SerialisedParams: +def serialise_params(params: dict[str, Any]) -> SerialisedParams: """ When writing column params to a migration file, we need to serialise some of the values. """ params = deepcopy(params) - extra_imports: t.List[Import] = [] - extra_definitions: t.List[Definition] = [] + extra_imports: list[Import] = [] + extra_definitions: list[Definition] = [] for key, value in params.items(): # Builtins, such as str, list and dict. @@ -725,7 +726,7 @@ def serialise_params(params: t.Dict[str, t.Any]) -> SerialisedParams: ) -def deserialise_params(params: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]: +def deserialise_params(params: dict[str, Any]) -> dict[str, Any]: """ When reading column params from a migration file, we need to convert them from their serialised form. diff --git a/piccolo/apps/migrations/auto/serialisation_legacy.py b/piccolo/apps/migrations/auto/serialisation_legacy.py index b212943bc..7ccfbf740 100644 --- a/piccolo/apps/migrations/auto/serialisation_legacy.py +++ b/piccolo/apps/migrations/auto/serialisation_legacy.py @@ -1,14 +1,14 @@ from __future__ import annotations import datetime -import typing as t +from typing import Any from piccolo.columns.column_types import OnDelete, OnUpdate from piccolo.columns.defaults.timestamp import TimestampNow from piccolo.table import create_table_class -def deserialise_legacy_params(name: str, value: str) -> t.Any: +def deserialise_legacy_params(name: str, value: str) -> Any: """ Earlier versions of Piccolo serialised parameters differently. This is here purely for backwards compatibility. diff --git a/piccolo/apps/migrations/commands/backwards.py b/piccolo/apps/migrations/commands/backwards.py index 363992510..25b72e8e3 100644 --- a/piccolo/apps/migrations/commands/backwards.py +++ b/piccolo/apps/migrations/commands/backwards.py @@ -2,7 +2,6 @@ import os import sys -import typing as t from piccolo.apps.migrations.auto.migration_manager import MigrationManager from piccolo.apps.migrations.commands.base import ( @@ -31,7 +30,7 @@ def __init__( super().__init__() async def run_migrations_backwards(self, app_config: AppConfig): - migration_modules: t.Dict[str, MigrationModule] = ( + migration_modules: dict[str, MigrationModule] = ( self.get_migration_modules( app_config.resolved_migrations_folder_path ) diff --git a/piccolo/apps/migrations/commands/base.py b/piccolo/apps/migrations/commands/base.py index bcc5cbc55..5a4e9615d 100644 --- a/piccolo/apps/migrations/commands/base.py +++ b/piccolo/apps/migrations/commands/base.py @@ -3,8 +3,8 @@ import importlib import os import sys -import typing as t from dataclasses import dataclass +from typing import Optional, cast from piccolo.apps.migrations.auto.diffable_table import DiffableTable from piccolo.apps.migrations.auto.migration_manager import MigrationManager @@ -16,7 +16,7 @@ @dataclass class MigrationResult: success: bool - message: t.Optional[str] = None + message: Optional[str] = None class BaseMigrationManager(Finder): @@ -32,7 +32,7 @@ async def create_migration_table(self) -> bool: def get_migration_modules( self, folder_path: str - ) -> t.Dict[str, MigrationModule]: + ) -> dict[str, MigrationModule]: """ Imports the migration modules in the given folder path, and returns a mapping of migration ID to the corresponding migration module. @@ -50,8 +50,8 @@ def get_migration_modules( if ((i not in excluded) and i.endswith(".py")) ] - modules: t.List[MigrationModule] = [ - t.cast(MigrationModule, importlib.import_module(name)) + modules: list[MigrationModule] = [ + cast(MigrationModule, importlib.import_module(name)) for name in migration_names ] for m in modules: @@ -62,8 +62,8 @@ def get_migration_modules( return migration_modules def get_migration_ids( - self, migration_module_dict: t.Dict[str, MigrationModule] - ) -> t.List[str]: + self, migration_module_dict: dict[str, MigrationModule] + ) -> list[str]: """ Returns a list of migration IDs, from the Python migration files. """ @@ -72,9 +72,9 @@ def get_migration_ids( async def get_migration_managers( self, app_config: AppConfig, - max_migration_id: t.Optional[str] = None, + max_migration_id: Optional[str] = None, offset: int = 0, - ) -> t.List[MigrationManager]: + ) -> list[MigrationManager]: """ Call the forwards coroutine in each migration module. Each one should return a `MigrationManger`. Combine all of the results, and return in @@ -84,11 +84,11 @@ async def get_migration_managers( If set, only MigrationManagers up to and including the given migration ID will be returned. """ - migration_managers: t.List[MigrationManager] = [] + migration_managers: list[MigrationManager] = [] migrations_folder = app_config.resolved_migrations_folder_path - migration_modules: t.Dict[str, MigrationModule] = ( + migration_modules: dict[str, MigrationModule] = ( self.get_migration_modules(migrations_folder) ) @@ -118,7 +118,7 @@ async def get_table_from_snapshot( self, app_name: str, table_class_name: str, - max_migration_id: t.Optional[str] = None, + max_migration_id: Optional[str] = None, offset: int = 0, ) -> DiffableTable: """ diff --git a/piccolo/apps/migrations/commands/check.py b/piccolo/apps/migrations/commands/check.py index 53e20840a..ee4bf12a3 100644 --- a/piccolo/apps/migrations/commands/check.py +++ b/piccolo/apps/migrations/commands/check.py @@ -1,5 +1,4 @@ import dataclasses -import typing as t from piccolo.apps.migrations.commands.base import BaseMigrationManager from piccolo.apps.migrations.tables import Migration @@ -19,11 +18,11 @@ def __init__(self, app_name: str): self.app_name = app_name super().__init__() - async def get_migration_statuses(self) -> t.List[MigrationStatus]: + async def get_migration_statuses(self) -> list[MigrationStatus]: # Make sure the migration table exists, otherwise we'll get an error. await self.create_migration_table() - migration_statuses: t.List[MigrationStatus] = [] + migration_statuses: list[MigrationStatus] = [] app_modules = self.get_app_modules() diff --git a/piccolo/apps/migrations/commands/clean.py b/piccolo/apps/migrations/commands/clean.py index 687ff64e9..5a95015a0 100644 --- a/piccolo/apps/migrations/commands/clean.py +++ b/piccolo/apps/migrations/commands/clean.py @@ -1,6 +1,6 @@ from __future__ import annotations -import typing as t +from typing import cast from piccolo.apps.migrations.commands.base import BaseMigrationManager from piccolo.apps.migrations.tables import Migration @@ -12,7 +12,7 @@ def __init__(self, app_name: str, auto_agree: bool = False): self.auto_agree = auto_agree super().__init__() - def get_migration_ids_to_remove(self) -> t.List[str]: + def get_migration_ids_to_remove(self) -> list[str]: """ Returns a list of migration ID strings, which are rows in the table, but don't have a corresponding migration module on disk. @@ -37,7 +37,7 @@ def get_migration_ids_to_remove(self) -> t.List[str]: if len(migration_ids) > 0: query = query.where(Migration.name.not_in(migration_ids)) - return t.cast(t.List[str], query.run_sync()) + return cast(list[str], query.run_sync()) async def run(self): print("Checking the migration table ...") diff --git a/piccolo/apps/migrations/commands/forwards.py b/piccolo/apps/migrations/commands/forwards.py index 560f117b2..adc7e657d 100644 --- a/piccolo/apps/migrations/commands/forwards.py +++ b/piccolo/apps/migrations/commands/forwards.py @@ -1,7 +1,6 @@ from __future__ import annotations import sys -import typing as t from piccolo.apps.migrations.auto.migration_manager import MigrationManager from piccolo.apps.migrations.commands.base import ( @@ -32,7 +31,7 @@ async def run_migrations(self, app_config: AppConfig) -> MigrationResult: app_name=app_config.app_name ) - migration_modules: t.Dict[str, MigrationModule] = ( + migration_modules: dict[str, MigrationModule] = ( self.get_migration_modules( app_config.resolved_migrations_folder_path ) diff --git a/piccolo/apps/migrations/commands/new.py b/piccolo/apps/migrations/commands/new.py index 082868435..172de96ed 100644 --- a/piccolo/apps/migrations/commands/new.py +++ b/piccolo/apps/migrations/commands/new.py @@ -3,10 +3,10 @@ import datetime import os import string -import typing as t from dataclasses import dataclass from itertools import chain from types import ModuleType +from typing import Optional import black import jinja2 @@ -33,7 +33,7 @@ loader=jinja2.FileSystemLoader(searchpath=TEMPLATE_DIRECTORY), ) -MIGRATION_MODULES: t.Dict[str, ModuleType] = {} +MIGRATION_MODULES: dict[str, ModuleType] = {} VALID_PYTHON_MODULE_CHARACTERS = string.ascii_lowercase + string.digits + "_" @@ -115,7 +115,7 @@ async def _create_new_migration( app_config: AppConfig, auto: bool = False, description: str = "", - auto_input: t.Optional[str] = None, + auto_input: Optional[str] = None, ) -> NewMigrationMeta: """ Creates a new migration file on disk. @@ -170,13 +170,13 @@ async def _create_new_migration( class AutoMigrationManager(BaseMigrationManager): - def __init__(self, auto_input: t.Optional[str] = None, *args, **kwargs): + def __init__(self, auto_input: Optional[str] = None, *args, **kwargs): self.auto_input = auto_input super().__init__(*args, **kwargs) async def get_alter_statements( self, app_config: AppConfig - ) -> t.List[AlterStatements]: + ) -> list[AlterStatements]: """ Works out which alter statements are required. """ @@ -214,7 +214,7 @@ async def new( app_name: str, auto: bool = False, desc: str = "", - auto_input: t.Optional[str] = None, + auto_input: Optional[str] = None, ): """ Creates a new migration file in the migrations folder. diff --git a/piccolo/apps/migrations/tables.py b/piccolo/apps/migrations/tables.py index 91906be4f..782172bbb 100644 --- a/piccolo/apps/migrations/tables.py +++ b/piccolo/apps/migrations/tables.py @@ -1,6 +1,6 @@ from __future__ import annotations -import typing as t +from typing import Optional from piccolo.columns import Timestamp, Varchar from piccolo.columns.defaults.timestamp import TimestampNow @@ -14,8 +14,8 @@ class Migration(Table): @classmethod async def get_migrations_which_ran( - cls, app_name: t.Optional[str] = None - ) -> t.List[str]: + cls, app_name: Optional[str] = None + ) -> list[str]: """ Returns the names of migrations which have already run, by inspecting the database. diff --git a/piccolo/apps/schema/commands/generate.py b/piccolo/apps/schema/commands/generate.py index c0fe1e95a..5e1785784 100644 --- a/piccolo/apps/schema/commands/generate.py +++ b/piccolo/apps/schema/commands/generate.py @@ -5,9 +5,9 @@ import itertools import json import re -import typing as t import uuid from datetime import date, datetime +from typing import TYPE_CHECKING, Any, Literal, Optional, Union import black @@ -43,7 +43,7 @@ from piccolo.table import Table, create_table_class, sort_table_classes from piccolo.utils.naming import _snake_to_camel -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from piccolo.engine.base import Engine @@ -61,13 +61,13 @@ class ConstraintTable: class RowMeta: column_default: str column_name: str - is_nullable: t.Literal["YES", "NO"] + is_nullable: Literal["YES", "NO"] table_name: str - character_maximum_length: t.Optional[int] + character_maximum_length: Optional[int] data_type: str - numeric_precision: t.Optional[t.Union[int, str]] - numeric_scale: t.Optional[t.Union[int, str]] - numeric_precision_radix: t.Optional[t.Literal[2, 10]] + numeric_precision: Optional[Union[int, str]] + numeric_scale: Optional[Union[int, str]] + numeric_precision_radix: Optional[Literal[2, 10]] @classmethod def get_column_name_str(cls) -> str: @@ -76,10 +76,10 @@ def get_column_name_str(cls) -> str: @dataclasses.dataclass class Constraint: - constraint_type: t.Literal["PRIMARY KEY", "UNIQUE", "FOREIGN KEY", "CHECK"] + constraint_type: Literal["PRIMARY KEY", "UNIQUE", "FOREIGN KEY", "CHECK"] constraint_name: str - constraint_schema: t.Optional[str] = None - column_name: t.Optional[str] = None + constraint_schema: Optional[str] = None + column_name: Optional[str] = None @dataclasses.dataclass @@ -89,12 +89,12 @@ class TableConstraints: """ tablename: str - constraints: t.List[Constraint] + constraints: list[Constraint] def __post_init__(self) -> None: - foreign_key_constraints: t.List[Constraint] = [] - unique_constraints: t.List[Constraint] = [] - primary_key_constraints: t.List[Constraint] = [] + foreign_key_constraints: list[Constraint] = [] + unique_constraints: list[Constraint] = [] + primary_key_constraints: list[Constraint] = [] for constraint in self.constraints: if constraint.constraint_type == "FOREIGN KEY": @@ -141,7 +141,7 @@ class Trigger: table_name: str column_name: str on_update: str - on_delete: t.Literal[ + on_delete: Literal[ "NO ACTION", "RESTRICT", "CASCADE", "SET NULL", "SET_DEFAULT" ] references_table: str @@ -155,14 +155,14 @@ class TableTriggers: """ tablename: str - triggers: t.List[Trigger] + triggers: list[Trigger] - def get_column_triggers(self, column_name: str) -> t.List[Trigger]: + def get_column_triggers(self, column_name: str) -> list[Trigger]: return [i for i in self.triggers if i.column_name == column_name] def get_column_ref_trigger( self, column_name: str, references_table: str - ) -> t.Optional[Trigger]: + ) -> Optional[Trigger]: for trigger in self.triggers: if ( trigger.column_name == column_name @@ -221,14 +221,14 @@ class TableIndexes: """ tablename: str - indexes: t.List[Index] + indexes: list[Index] - def get_column_index(self, column_name: str) -> t.Optional[Index]: + def get_column_index(self, column_name: str) -> Optional[Index]: return next( (i for i in self.indexes if i.column_name == column_name), None ) - def get_warnings(self) -> t.List[str]: + def get_warnings(self) -> list[str]: return list( itertools.chain(*[index.warnings for index in self.indexes]) ) @@ -250,13 +250,13 @@ class OutputSchema: e.g. ["class MyTable(Table): ..."] """ - imports: t.List[str] = dataclasses.field(default_factory=list) - warnings: t.List[str] = dataclasses.field(default_factory=list) - index_warnings: t.List[str] = dataclasses.field(default_factory=list) - trigger_warnings: t.List[str] = dataclasses.field(default_factory=list) - tables: t.List[t.Type[Table]] = dataclasses.field(default_factory=list) + imports: list[str] = dataclasses.field(default_factory=list) + warnings: list[str] = dataclasses.field(default_factory=list) + index_warnings: list[str] = dataclasses.field(default_factory=list) + trigger_warnings: list[str] = dataclasses.field(default_factory=list) + tables: list[type[Table]] = dataclasses.field(default_factory=list) - def get_table_with_name(self, tablename: str) -> t.Optional[t.Type[Table]]: + def get_table_with_name(self, tablename: str) -> Optional[type[Table]]: """ Used to search for a table by name. """ @@ -287,7 +287,7 @@ def __add__(self, value: OutputSchema) -> OutputSchema: return self -COLUMN_TYPE_MAP: t.Dict[str, t.Type[Column]] = { +COLUMN_TYPE_MAP: dict[str, type[Column]] = { "bigint": BigInt, "boolean": Boolean, "bytea": Bytea, @@ -308,12 +308,12 @@ def __add__(self, value: OutputSchema) -> OutputSchema: } # Re-map for Cockroach compatibility. -COLUMN_TYPE_MAP_COCKROACH: t.Dict[str, t.Type[Column]] = { +COLUMN_TYPE_MAP_COCKROACH: dict[str, type[Column]] = { **COLUMN_TYPE_MAP, **{"integer": BigInt, "json": JSONB}, } -COLUMN_DEFAULT_PARSER: t.Dict[t.Type[Column], t.Any] = { +COLUMN_DEFAULT_PARSER: dict[type[Column], Any] = { BigInt: re.compile(r"^'?(?P-?[0-9]\d*)'?(?:::bigint)?$"), Boolean: re.compile(r"^(?Ptrue|false)$"), Bytea: re.compile(r"'(?P.*)'::bytea$"), @@ -373,15 +373,15 @@ def __add__(self, value: OutputSchema) -> OutputSchema: } # Re-map for Cockroach compatibility. -COLUMN_DEFAULT_PARSER_COCKROACH: t.Dict[t.Type[Column], t.Any] = { +COLUMN_DEFAULT_PARSER_COCKROACH: dict[type[Column], Any] = { **COLUMN_DEFAULT_PARSER, BigInt: re.compile(r"^(?P-?\d+)$"), } def get_column_default( - column_type: t.Type[Column], column_default: str, engine_type: str -) -> t.Any: + column_type: type[Column], column_default: str, engine_type: str +) -> Any: if engine_type == "cockroach": pat = COLUMN_DEFAULT_PARSER_COCKROACH.get(column_type) else: @@ -455,7 +455,7 @@ def get_column_default( return column_type.value_type(value["value"]) -INDEX_METHOD_MAP: t.Dict[str, IndexMethod] = { +INDEX_METHOD_MAP: dict[str, IndexMethod] = { "btree": IndexMethod.btree, "hash": IndexMethod.hash, "gist": IndexMethod.gist, @@ -465,7 +465,7 @@ def get_column_default( # 'Indices' seems old-fashioned and obscure in this context. async def get_indexes( # noqa: E302 - table_class: t.Type[Table], tablename: str, schema_name: str = "public" + table_class: type[Table], tablename: str, schema_name: str = "public" ) -> TableIndexes: """ Get all of the constraints for a table. @@ -492,7 +492,7 @@ async def get_indexes( # noqa: E302 async def get_fk_triggers( - table_class: t.Type[Table], tablename: str, schema_name: str = "public" + table_class: type[Table], tablename: str, schema_name: str = "public" ) -> TableTriggers: """ Get all of the constraints for a table. @@ -540,7 +540,7 @@ async def get_fk_triggers( async def get_constraints( - table_class: t.Type[Table], tablename: str, schema_name: str = "public" + table_class: type[Table], tablename: str, schema_name: str = "public" ) -> TableConstraints: """ Get all of the constraints for a table. @@ -572,8 +572,8 @@ async def get_constraints( async def get_tablenames( - table_class: t.Type[Table], schema_name: str = "public" -) -> t.List[str]: + table_class: type[Table], schema_name: str = "public" +) -> list[str]: """ Get the tablenames for the schema. @@ -598,8 +598,8 @@ async def get_tablenames( async def get_table_schema( - table_class: t.Type[Table], tablename: str, schema_name: str = "public" -) -> t.List[RowMeta]: + table_class: type[Table], tablename: str, schema_name: str = "public" +) -> list[RowMeta]: """ Get the schema from the database. @@ -629,7 +629,7 @@ async def get_table_schema( async def get_foreign_key_reference( - table_class: t.Type[Table], constraint_name: str, constraint_schema: str + table_class: type[Table], constraint_name: str, constraint_schema: str ) -> ConstraintTable: """ Retrieve the name of the table that a foreign key is referencing. @@ -652,7 +652,7 @@ async def get_foreign_key_reference( async def create_table_class_from_db( - table_class: t.Type[Table], + table_class: type[Table], tablename: str, schema_name: str, engine_type: str, @@ -674,7 +674,7 @@ async def create_table_class_from_db( table_class=table_class, tablename=tablename, schema_name=schema_name ) - columns: t.Dict[str, Column] = {} + columns: dict[str, Column] = {} for pg_row_meta in table_schema: data_type = pg_row_meta.data_type @@ -692,7 +692,7 @@ async def create_table_class_from_db( ) column_type = Column - kwargs: t.Dict[str, t.Any] = { + kwargs: dict[str, Any] = { "null": pg_row_meta.is_nullable == "YES", "unique": constraints.is_unique(column_name=column_name), } @@ -721,7 +721,7 @@ async def create_table_class_from_db( constraint_schema=fk_constraint_table.schema, ) if constraint_table.name: - referenced_table: t.Union[str, t.Optional[t.Type[Table]]] + referenced_table: Union[str, Optional[type[Table]]] if constraint_table.name == tablename: referenced_output_schema = output_schema @@ -805,9 +805,9 @@ async def create_table_class_from_db( async def get_output_schema( schema_name: str = "public", - include: t.Optional[t.List[str]] = None, - exclude: t.Optional[t.List[str]] = None, - engine: t.Optional[Engine] = None, + include: Optional[list[str]] = None, + exclude: Optional[list[str]] = None, + engine: Optional[Engine] = None, ) -> OutputSchema: """ :param schema_name: diff --git a/piccolo/apps/schema/commands/graph.py b/piccolo/apps/schema/commands/graph.py index 4a19e4b23..ffb09e761 100644 --- a/piccolo/apps/schema/commands/graph.py +++ b/piccolo/apps/schema/commands/graph.py @@ -5,7 +5,7 @@ import dataclasses import os import sys -import typing as t +from typing import Optional import jinja2 @@ -29,7 +29,7 @@ class GraphColumn: @dataclasses.dataclass class GraphTable: name: str - columns: t.List[GraphColumn] + columns: list[GraphColumn] @dataclasses.dataclass @@ -45,7 +45,7 @@ def render_template(**kwargs): def graph( - apps: str = "all", direction: str = "LR", output: t.Optional[str] = None + apps: str = "all", direction: str = "LR", output: Optional[str] = None ): """ Prints out a graphviz .dot file for your schema. @@ -73,8 +73,8 @@ def graph( sys.exit(f"These apps aren't recognised: {', '.join(delta)}.") app_names = given_app_names - tables: t.List[GraphTable] = [] - relations: t.List[GraphRelation] = [] + tables: list[GraphTable] = [] + relations: list[GraphRelation] = [] for app_name in app_names: app_config = finder.get_app_config(app_name=app_name) diff --git a/piccolo/apps/shell/commands/run.py b/piccolo/apps/shell/commands/run.py index 4f86cc23c..4b24f7fd6 100644 --- a/piccolo/apps/shell/commands/run.py +++ b/piccolo/apps/shell/commands/run.py @@ -1,5 +1,4 @@ import sys -import typing as t from piccolo.conf.apps import Finder from piccolo.table import Table @@ -13,7 +12,7 @@ IPYTHON = False -def start_ipython_shell(**tables: t.Type[Table]): # pragma: no cover +def start_ipython_shell(**tables: type[Table]): # pragma: no cover if not IPYTHON: sys.exit( "Install iPython using `pip install ipython` to use this feature." diff --git a/piccolo/apps/sql_shell/commands/run.py b/piccolo/apps/sql_shell/commands/run.py index 8286b7cce..a666321a4 100644 --- a/piccolo/apps/sql_shell/commands/run.py +++ b/piccolo/apps/sql_shell/commands/run.py @@ -2,7 +2,7 @@ import signal import subprocess import sys -import typing as t +from typing import cast from piccolo.engine.finder import engine_finder from piccolo.engine.postgres import PostgresEngine @@ -24,7 +24,7 @@ def run() -> None: # Heavily inspired by Django's dbshell command if isinstance(engine, PostgresEngine): - engine = t.cast(PostgresEngine, engine) + engine = cast(PostgresEngine, engine) args = ["psql"] @@ -56,9 +56,9 @@ def run() -> None: signal.signal(signal.SIGINT, sigint_handler) elif isinstance(engine, SQLiteEngine): - engine = t.cast(SQLiteEngine, engine) + engine = cast(SQLiteEngine, engine) - database = t.cast(str, engine.connection_kwargs.get("database")) + database = cast(str, engine.connection_kwargs.get("database")) if not database: sys.exit("Unable to determine which database to connect to.") diff --git a/piccolo/apps/tester/commands/run.py b/piccolo/apps/tester/commands/run.py index 652911269..8882fd645 100644 --- a/piccolo/apps/tester/commands/run.py +++ b/piccolo/apps/tester/commands/run.py @@ -2,7 +2,7 @@ import os import sys -import typing as t +from typing import Optional from piccolo.table import TABLE_REGISTRY @@ -25,7 +25,7 @@ def __init__(self, var_name: str, temp_value: str): def set_var(self, value: str): os.environ[self.var_name] = value - def get_var(self) -> t.Optional[str]: + def get_var(self) -> Optional[str]: return os.environ.get(self.var_name) def __enter__(self): @@ -39,7 +39,7 @@ def __exit__(self, *args): self.set_var(self.existing_value) -def run_pytest(pytest_args: t.List[str]) -> int: # pragma: no cover +def run_pytest(pytest_args: list[str]) -> int: # pragma: no cover try: import pytest except ImportError: diff --git a/piccolo/apps/user/commands/change_permissions.py b/piccolo/apps/user/commands/change_permissions.py index edf0dc443..63ae90991 100644 --- a/piccolo/apps/user/commands/change_permissions.py +++ b/piccolo/apps/user/commands/change_permissions.py @@ -1,17 +1,17 @@ -import typing as t +from typing import TYPE_CHECKING, Optional, Union from piccolo.apps.user.tables import BaseUser from piccolo.utils.warnings import Level, colored_string -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from piccolo.columns import Column async def change_permissions( username: str, - admin: t.Optional[bool] = None, - superuser: t.Optional[bool] = None, - active: t.Optional[bool] = None, + admin: Optional[bool] = None, + superuser: Optional[bool] = None, + active: Optional[bool] = None, ): """ Change a user's permissions. @@ -34,7 +34,7 @@ async def change_permissions( ) return - params: t.Dict[t.Union[Column, str], bool] = {} + params: dict[Union[Column, str], bool] = {} if admin is not None: params[BaseUser.admin] = admin diff --git a/piccolo/apps/user/commands/create.py b/piccolo/apps/user/commands/create.py index caec89c5a..decea97c1 100644 --- a/piccolo/apps/user/commands/create.py +++ b/piccolo/apps/user/commands/create.py @@ -1,6 +1,6 @@ import sys -import typing as t from getpass import getpass, getuser +from typing import Optional from piccolo.apps.user.tables import BaseUser @@ -57,12 +57,12 @@ def get_is_active() -> bool: def create( - username: t.Optional[str] = None, - email: t.Optional[str] = None, - password: t.Optional[str] = None, - is_admin: t.Optional[bool] = None, - is_superuser: t.Optional[bool] = None, - is_active: t.Optional[bool] = None, + username: Optional[str] = None, + email: Optional[str] = None, + password: Optional[str] = None, + is_admin: Optional[bool] = None, + is_superuser: Optional[bool] = None, + is_active: Optional[bool] = None, ): """ Create a new user. diff --git a/piccolo/apps/user/commands/list.py b/piccolo/apps/user/commands/list.py index 5a39f6961..88f8cb294 100644 --- a/piccolo/apps/user/commands/list.py +++ b/piccolo/apps/user/commands/list.py @@ -1,4 +1,4 @@ -import typing as t +from typing import Any from piccolo.apps.user.tables import BaseUser from piccolo.columns import Column @@ -11,7 +11,7 @@ async def get_users( order_by: Column, ascending: bool, limit: int, page: int -) -> t.List[t.Dict[str, t.Any]]: +) -> list[dict[str, Any]]: return ( await BaseUser.select( *BaseUser.all_columns(exclude=[BaseUser.password]) diff --git a/piccolo/apps/user/tables.py b/piccolo/apps/user/tables.py index 18f4915fa..7990b9e07 100644 --- a/piccolo/apps/user/tables.py +++ b/piccolo/apps/user/tables.py @@ -8,7 +8,7 @@ import hashlib import logging import secrets -import typing as t +from typing import Any, Optional, Union from piccolo.columns import Boolean, Secret, Timestamp, Varchar from piccolo.columns.column_types import Serial @@ -109,14 +109,14 @@ def _validate_password(cls, password: str): ########################################################################### @classmethod - def update_password_sync(cls, user: t.Union[str, int], password: str): + def update_password_sync(cls, user: Union[str, int], password: str): """ A sync equivalent of :meth:`update_password`. """ return run_sync(cls.update_password(user, password)) @classmethod - async def update_password(cls, user: t.Union[str, int], password: str): + async def update_password(cls, user: Union[str, int], password: str): """ The password is the raw password string e.g. ``'password123'``. The user can be a user ID, or a username. @@ -139,7 +139,7 @@ async def update_password(cls, user: t.Union[str, int], password: str): @classmethod def hash_password( - cls, password: str, salt: str = "", iterations: t.Optional[int] = None + cls, password: str, salt: str = "", iterations: Optional[int] = None ) -> str: """ Hashes the password, ready for storage, and for comparing during @@ -167,7 +167,7 @@ def hash_password( ).hex() return f"pbkdf2_sha256${iterations}${salt}${hashed}" - def __setattr__(self, name: str, value: t.Any): + def __setattr__(self, name: str, value: Any): """ Make sure that if the password is set, it's stored in a hashed form. """ @@ -177,7 +177,7 @@ def __setattr__(self, name: str, value: t.Any): super().__setattr__(name, value) @classmethod - def split_stored_password(cls, password: str) -> t.List[str]: + def split_stored_password(cls, password: str) -> list[str]: elements = password.split("$") if len(elements) != 4: raise ValueError("Unable to split hashed password") @@ -186,14 +186,14 @@ def split_stored_password(cls, password: str) -> t.List[str]: ########################################################################### @classmethod - def login_sync(cls, username: str, password: str) -> t.Optional[int]: + def login_sync(cls, username: str, password: str) -> Optional[int]: """ A sync equivalent of :meth:`login`. """ return run_sync(cls.login(username, password)) @classmethod - async def login(cls, username: str, password: str) -> t.Optional[int]: + async def login(cls, username: str, password: str) -> Optional[int]: """ Make sure the user exists and the password is valid. If so, the ``last_login`` value is updated in the database. diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 4b80a9ff6..b1994a272 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -4,10 +4,20 @@ import datetime import decimal import inspect -import typing as t import uuid +from collections.abc import Iterable from dataclasses import dataclass, field, fields from enum import Enum +from typing import ( + TYPE_CHECKING, + Any, + Generic, + Optional, + TypedDict, + TypeVar, + Union, + cast, +) from piccolo.columns.choices import Choice from piccolo.columns.combination import Where @@ -34,7 +44,7 @@ from piccolo.querystring import QueryString, Selectable from piccolo.utils.warnings import colored_warning -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from piccolo.columns.column_types import ForeignKey from piccolo.table import Table @@ -77,19 +87,19 @@ def __repr__(self): return self.__str__() -ReferencedTable = t.TypeVar("ReferencedTable", bound="Table") +ReferencedTable = TypeVar("ReferencedTable", bound="Table") @dataclass -class ForeignKeyMeta(t.Generic[ReferencedTable]): - references: t.Union[t.Type[ReferencedTable], LazyTableReference] +class ForeignKeyMeta(Generic[ReferencedTable]): + references: Union[type[ReferencedTable], LazyTableReference] on_delete: OnDelete on_update: OnUpdate - target_column: t.Union[Column, str, None] - proxy_columns: t.List[Column] = field(default_factory=list) + target_column: Union[Column, str, None] + proxy_columns: list[Column] = field(default_factory=list) @property - def resolved_references(self) -> t.Type[Table]: + def resolved_references(self) -> type[Table]: """ Evaluates the ``references`` attribute if it's a ``LazyTableReference``, raising a ``ValueError`` if it fails, otherwise returns a ``Table`` @@ -154,18 +164,18 @@ class ColumnMeta: index: bool = False index_method: IndexMethod = IndexMethod.btree required: bool = False - help_text: t.Optional[str] = None - choices: t.Optional[t.Type[Enum]] = None + help_text: Optional[str] = None + choices: Optional[type[Enum]] = None secret: bool = False - auto_update: t.Any = ... + auto_update: Any = ... # Used for representing the table in migrations and the playground. - params: t.Dict[str, t.Any] = field(default_factory=dict) + params: dict[str, Any] = field(default_factory=dict) ########################################################################### # Lets you to map a column to a database column with a different name. - _db_column_name: t.Optional[str] = None + _db_column_name: Optional[str] = None @property def db_column_name(self) -> str: @@ -178,8 +188,8 @@ def db_column_name(self, value: str): ########################################################################### # Set by the Table Metaclass: - _name: t.Optional[str] = None - _table: t.Optional[t.Type[Table]] = None + _name: Optional[str] = None + _table: Optional[type[Table]] = None @property def name(self) -> str: @@ -194,7 +204,7 @@ def name(self, value: str): self._name = value @property - def table(self) -> t.Type[Table]: + def table(self) -> type[Table]: if not self._table: raise ValueError( "`_table` isn't defined - the Table Metaclass should set it." @@ -202,13 +212,13 @@ def table(self) -> t.Type[Table]: return self._table @table.setter - def table(self, value: t.Type[Table]): + def table(self, value: type[Table]): self._table = value ########################################################################### # Used by Foreign Keys: - call_chain: t.List["ForeignKey"] = field(default_factory=list) + call_chain: list["ForeignKey"] = field(default_factory=list) ########################################################################### @@ -220,7 +230,7 @@ def engine_type(self) -> str: else: raise ValueError("The table has no engine defined.") - def get_choices_dict(self) -> t.Optional[t.Dict[str, t.Any]]: + def get_choices_dict(self) -> Optional[dict[str, Any]]: """ Return the choices Enum as a dict. It maps the attribute name to a dict containing the display name, and value. @@ -251,8 +261,7 @@ def get_default_alias(self): if self.call_chain: column_name = ( ".".join( - t.cast(str, i._meta.db_column_name) - for i in self.call_chain + cast(str, i._meta.db_column_name) for i in self.call_chain ) + f".{column_name}" ) @@ -350,18 +359,18 @@ def __deepcopy__(self, memo) -> ColumnMeta: return self.copy() -class ColumnKwargs(t.TypedDict, total=False): +class ColumnKwargs(TypedDict, total=False): null: bool primary_key: bool unique: bool index: bool index_method: IndexMethod required: bool - help_text: t.Optional[str] - choices: t.Optional[t.Type[Enum]] - db_column_name: t.Optional[str] + help_text: Optional[str] + choices: Optional[type[Enum]] + db_column_name: Optional[str] secret: bool - auto_update: t.Any + auto_update: Any class Column(Selectable): @@ -463,8 +472,8 @@ class Band(Table): """ - value_type: t.Type = int - default: t.Any + value_type: type = int + default: Any def __init__( self, @@ -474,11 +483,11 @@ def __init__( index: bool = False, index_method: IndexMethod = IndexMethod.btree, required: bool = False, - help_text: t.Optional[str] = None, - choices: t.Optional[t.Type[Enum]] = None, - db_column_name: t.Optional[str] = None, + help_text: Optional[str] = None, + choices: Optional[type[Enum]] = None, + db_column_name: Optional[str] = None, secret: bool = False, - auto_update: t.Any = ..., + auto_update: Any = ..., **kwargs, ) -> None: # This is for backwards compatibility - originally the `primary_key` @@ -522,12 +531,12 @@ def __init__( auto_update=auto_update, ) - self._alias: t.Optional[str] = None + self._alias: Optional[str] = None def _validate_default( self, - default: t.Any, - allowed_types: t.Iterable[t.Union[None, t.Type[t.Any]]], + default: Any, + allowed_types: Iterable[Union[None, type[Any]]], allow_recursion: bool = True, ) -> bool: """ @@ -564,7 +573,7 @@ def _validate_default( ) def _validate_choices( - self, choices: t.Type[Enum], allowed_type: t.Type[t.Any] + self, choices: type[Enum], allowed_type: type[Any] ) -> bool: """ Make sure the choices value has values of the allowed_type. @@ -590,14 +599,14 @@ def _validate_choices( return True - def is_in(self, values: t.List[t.Any]) -> Where: + def is_in(self, values: list[Any]) -> Where: if len(values) == 0: raise ValueError( "The `values` list argument must contain at least one value." ) return Where(column=self, values=values, operator=In) - def not_in(self, values: t.List[t.Any]) -> Where: + def not_in(self, values: list[Any]) -> Where: if len(values) == 0: raise ValueError( "The `values` list argument must contain at least one value." @@ -628,7 +637,7 @@ def ilike(self, value: str) -> Where: """ if self._meta.engine_type in ("postgres", "cockroach"): - operator: t.Type[ComparisonOperator] = ILike + operator: type[ComparisonOperator] = ILike else: colored_warning( "SQLite doesn't support ILIKE, falling back to LIKE." @@ -799,7 +808,7 @@ class Band(Table): virtual_foreign_key.set_proxy_columns() return virtual_foreign_key - def get_default_value(self) -> t.Any: + def get_default_value(self) -> Any: """ If the column has a default attribute, return it. If it's callable, return the response instead. @@ -845,7 +854,7 @@ def get_where_string(self, engine_type: str) -> QueryString: def get_sql_value( self, - value: t.Any, + value: Any, delimiter: str = "'", ) -> str: """ @@ -946,8 +955,8 @@ def ddl(self) -> str: if not self._meta.null: query += " NOT NULL" - foreign_key_meta = t.cast( - t.Optional[ForeignKeyMeta], + foreign_key_meta = cast( + Optional[ForeignKeyMeta], getattr(self, "_foreign_key_meta", None), ) if foreign_key_meta: @@ -1004,4 +1013,4 @@ def __repr__(self): ) -Self = t.TypeVar("Self", bound=Column) +Self = TypeVar("Self", bound=Column) diff --git a/piccolo/columns/choices.py b/piccolo/columns/choices.py index d3facf4d2..2886ad94f 100644 --- a/piccolo/columns/choices.py +++ b/piccolo/columns/choices.py @@ -1,7 +1,7 @@ from __future__ import annotations -import typing as t from dataclasses import dataclass +from typing import Any @dataclass @@ -29,5 +29,5 @@ class Title(Enum): """ - value: t.Any + value: Any display_name: str diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 2cef6625d..572aac102 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -29,11 +29,21 @@ class Band(Table): import copy import decimal import inspect -import typing as t import uuid +from collections.abc import Callable from dataclasses import dataclass from datetime import date, datetime, time, timedelta from enum import Enum +from typing import ( + TYPE_CHECKING, + Any, + Generic, + Literal, + Optional, + Union, + cast, + overload, +) from typing_extensions import Unpack @@ -71,7 +81,7 @@ class Band(Table): from piccolo.utils.encoding import dump_json from piccolo.utils.warnings import colored_warning -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from piccolo.columns.base import ColumnMeta from piccolo.query.operators.json import ( GetChildElement, @@ -94,7 +104,7 @@ class ConcatDelegate: def get_querystring( self, column: Column, - value: t.Union[str, Column, QueryString], + value: Union[str, Column, QueryString], reverse: bool = False, ) -> QueryString: """ @@ -137,8 +147,8 @@ class MathDelegate: def get_querystring( self, column_name: str, - operator: t.Literal["+", "-", "/", "*"], - value: t.Union[int, float, Integer], + operator: Literal["+", "-", "/", "*"], + value: Union[int, float, Integer], reverse: bool = False, ) -> QueryString: if isinstance(value, Integer): @@ -184,7 +194,7 @@ class Concert(Table): # Maps the attribute name in Python's timedelta to what it's called in # Postgres. - postgres_attr_map: t.Dict[str, str] = { + postgres_attr_map: dict[str, str] = { "days": "DAYS", "seconds": "SECONDS", "microseconds": "MICROSECONDS", @@ -233,7 +243,7 @@ def get_sqlite_interval_string(self, interval: timedelta) -> str: def get_querystring( self, column: Column, - operator: t.Literal["+", "-"], + operator: Literal["+", "-"], value: timedelta, engine_type: str, ) -> QueryString: @@ -318,8 +328,8 @@ class Band(Table): def __init__( self, - length: t.Optional[int] = 255, - default: t.Union[str, Enum, t.Callable[[], str], None] = "", + length: Optional[int] = 255, + default: Union[str, Enum, Callable[[], str], None] = "", **kwargs: Unpack[ColumnKwargs], ) -> None: self._validate_default(default, (str, None)) @@ -335,13 +345,13 @@ def column_type(self): ########################################################################### # For update queries - def __add__(self, value: t.Union[str, Varchar, Text]) -> QueryString: + def __add__(self, value: Union[str, Varchar, Text]) -> QueryString: return self.concat_delegate.get_querystring( column=self, value=value, ) - def __radd__(self, value: t.Union[str, Varchar, Text]) -> QueryString: + def __radd__(self, value: Union[str, Varchar, Text]) -> QueryString: return self.concat_delegate.get_querystring( column=self, value=value, @@ -351,16 +361,16 @@ def __radd__(self, value: t.Union[str, Varchar, Text]) -> QueryString: ########################################################################### # Descriptors - @t.overload + @overload def __get__(self, obj: Table, objtype=None) -> str: ... - @t.overload + @overload def __get__(self, obj: None, objtype=None) -> Varchar: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self - def __set__(self, obj, value: t.Union[str, None]): + def __set__(self, obj, value: Union[str, None]): obj.__dict__[self._meta.name] = value @@ -388,16 +398,16 @@ def __init__(self, *args, **kwargs): ########################################################################### # Descriptors - @t.overload + @overload def __get__(self, obj: Table, objtype=None) -> str: ... - @t.overload + @overload def __get__(self, obj: None, objtype=None) -> Secret: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self - def __set__(self, obj, value: t.Union[str, None]): + def __set__(self, obj, value: Union[str, None]): obj.__dict__[self._meta.name] = value @@ -427,7 +437,7 @@ class Band(Table): def __init__( self, - default: t.Union[str, Enum, None, t.Callable[[], str]] = "", + default: Union[str, Enum, None, Callable[[], str]] = "", **kwargs: Unpack[ColumnKwargs], ) -> None: self._validate_default(default, (str, None)) @@ -437,13 +447,13 @@ def __init__( ########################################################################### # For update queries - def __add__(self, value: t.Union[str, Varchar, Text]) -> QueryString: + def __add__(self, value: Union[str, Varchar, Text]) -> QueryString: return self.concat_delegate.get_querystring( column=self, value=value, ) - def __radd__(self, value: t.Union[str, Varchar, Text]) -> QueryString: + def __radd__(self, value: Union[str, Varchar, Text]) -> QueryString: return self.concat_delegate.get_querystring( column=self, value=value, @@ -453,16 +463,16 @@ def __radd__(self, value: t.Union[str, Varchar, Text]) -> QueryString: ########################################################################### # Descriptors - @t.overload + @overload def __get__(self, obj: Table, objtype=None) -> str: ... - @t.overload + @overload def __get__(self, obj: None, objtype=None) -> Text: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self - def __set__(self, obj, value: t.Union[str, None]): + def __set__(self, obj, value: Union[str, None]): obj.__dict__[self._meta.name] = value @@ -519,16 +529,16 @@ def __init__( ########################################################################### # Descriptors - @t.overload + @overload def __get__(self, obj: Table, objtype=None) -> uuid.UUID: ... - @t.overload + @overload def __get__(self, obj: None, objtype=None) -> UUID: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self - def __set__(self, obj, value: t.Union[uuid.UUID, None]): + def __set__(self, obj, value: Union[uuid.UUID, None]): obj.__dict__[self._meta.name] = value @@ -556,7 +566,7 @@ class Band(Table): def __init__( self, - default: t.Union[int, Enum, t.Callable[[], int], None] = 0, + default: Union[int, Enum, Callable[[], int], None] = 0, **kwargs: Unpack[ColumnKwargs], ) -> None: self._validate_default(default, (int, None)) @@ -566,12 +576,12 @@ def __init__( ########################################################################### # For update queries - def __add__(self, value: t.Union[int, float, Integer]) -> QueryString: + def __add__(self, value: Union[int, float, Integer]) -> QueryString: return self.math_delegate.get_querystring( column_name=self._meta.db_column_name, operator="+", value=value ) - def __radd__(self, value: t.Union[int, float, Integer]) -> QueryString: + def __radd__(self, value: Union[int, float, Integer]) -> QueryString: return self.math_delegate.get_querystring( column_name=self._meta.db_column_name, operator="+", @@ -579,12 +589,12 @@ def __radd__(self, value: t.Union[int, float, Integer]) -> QueryString: reverse=True, ) - def __sub__(self, value: t.Union[int, float, Integer]) -> QueryString: + def __sub__(self, value: Union[int, float, Integer]) -> QueryString: return self.math_delegate.get_querystring( column_name=self._meta.db_column_name, operator="-", value=value ) - def __rsub__(self, value: t.Union[int, float, Integer]) -> QueryString: + def __rsub__(self, value: Union[int, float, Integer]) -> QueryString: return self.math_delegate.get_querystring( column_name=self._meta.db_column_name, operator="-", @@ -592,12 +602,12 @@ def __rsub__(self, value: t.Union[int, float, Integer]) -> QueryString: reverse=True, ) - def __mul__(self, value: t.Union[int, float, Integer]) -> QueryString: + def __mul__(self, value: Union[int, float, Integer]) -> QueryString: return self.math_delegate.get_querystring( column_name=self._meta.db_column_name, operator="*", value=value ) - def __rmul__(self, value: t.Union[int, float, Integer]) -> QueryString: + def __rmul__(self, value: Union[int, float, Integer]) -> QueryString: return self.math_delegate.get_querystring( column_name=self._meta.db_column_name, operator="*", @@ -605,12 +615,12 @@ def __rmul__(self, value: t.Union[int, float, Integer]) -> QueryString: reverse=True, ) - def __truediv__(self, value: t.Union[int, float, Integer]) -> QueryString: + def __truediv__(self, value: Union[int, float, Integer]) -> QueryString: return self.math_delegate.get_querystring( column_name=self._meta.db_column_name, operator="/", value=value ) - def __rtruediv__(self, value: t.Union[int, float, Integer]) -> QueryString: + def __rtruediv__(self, value: Union[int, float, Integer]) -> QueryString: return self.math_delegate.get_querystring( column_name=self._meta.db_column_name, operator="/", @@ -618,14 +628,12 @@ def __rtruediv__(self, value: t.Union[int, float, Integer]) -> QueryString: reverse=True, ) - def __floordiv__(self, value: t.Union[int, float, Integer]) -> QueryString: + def __floordiv__(self, value: Union[int, float, Integer]) -> QueryString: return self.math_delegate.get_querystring( column_name=self._meta.db_column_name, operator="/", value=value ) - def __rfloordiv__( - self, value: t.Union[int, float, Integer] - ) -> QueryString: + def __rfloordiv__(self, value: Union[int, float, Integer]) -> QueryString: return self.math_delegate.get_querystring( column_name=self._meta.db_column_name, operator="/", @@ -636,16 +644,16 @@ def __rfloordiv__( ########################################################################### # Descriptors - @t.overload + @overload def __get__(self, obj: Table, objtype=None) -> int: ... - @t.overload + @overload def __get__(self, obj: None, objtype=None) -> Integer: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self - def __set__(self, obj, value: t.Union[int, None]): + def __set__(self, obj, value: Union[int, None]): obj.__dict__[self._meta.name] = value @@ -692,16 +700,16 @@ def column_type(self): ########################################################################### # Descriptors - @t.overload + @overload def __get__(self, obj: Table, objtype=None) -> int: ... - @t.overload + @overload def __get__(self, obj: None, objtype=None) -> BigInt: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self - def __set__(self, obj, value: t.Union[int, None]): + def __set__(self, obj, value: Union[int, None]): obj.__dict__[self._meta.name] = value @@ -740,16 +748,16 @@ def column_type(self): ########################################################################### # Descriptors - @t.overload + @overload def __get__(self, obj: Table, objtype=None) -> int: ... - @t.overload + @overload def __get__(self, obj: None, objtype=None) -> SmallInt: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self - def __set__(self, obj, value: t.Union[int, None]): + def __set__(self, obj, value: Union[int, None]): obj.__dict__[self._meta.name] = value @@ -790,16 +798,16 @@ def default(self): ########################################################################### # Descriptors - @t.overload + @overload def __get__(self, obj: Table, objtype=None) -> int: ... - @t.overload + @overload def __get__(self, obj: None, objtype=None) -> Serial: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self - def __set__(self, obj, value: t.Union[int, None]): + def __set__(self, obj, value: Union[int, None]): obj.__dict__[self._meta.name] = value @@ -822,16 +830,16 @@ def column_type(self): ########################################################################### # Descriptors - @t.overload + @overload def __get__(self, obj: Table, objtype=None) -> int: ... - @t.overload + @overload def __get__(self, obj: None, objtype=None) -> BigSerial: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self - def __set__(self, obj, value: t.Union[int, None]): + def __set__(self, obj, value: Union[int, None]): obj.__dict__[self._meta.name] = value @@ -858,16 +866,16 @@ def __init__( ########################################################################### # Descriptors - @t.overload + @overload def __get__(self, obj: Table, objtype=None) -> int: ... - @t.overload + @overload def __get__(self, obj: None, objtype=None) -> PrimaryKey: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self - def __set__(self, obj, value: t.Union[int, None]): + def __set__(self, obj, value: Union[int, None]): obj.__dict__[self._meta.name] = value @@ -947,16 +955,16 @@ def __sub__(self, value: timedelta) -> QueryString: ########################################################################### # Descriptors - @t.overload + @overload def __get__(self, obj: Table, objtype=None) -> datetime: ... - @t.overload + @overload def __get__(self, obj: None, objtype=None) -> Timestamp: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self - def __set__(self, obj, value: t.Union[datetime, None]): + def __set__(self, obj, value: Union[datetime, None]): obj.__dict__[self._meta.name] = value @@ -1043,16 +1051,16 @@ def __sub__(self, value: timedelta) -> QueryString: ########################################################################### # Descriptors - @t.overload + @overload def __get__(self, obj: Table, objtype=None) -> datetime: ... - @t.overload + @overload def __get__(self, obj: None, objtype=None) -> Timestamptz: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self - def __set__(self, obj, value: t.Union[datetime, None]): + def __set__(self, obj, value: Union[datetime, None]): obj.__dict__[self._meta.name] = value @@ -1124,16 +1132,16 @@ def __sub__(self, value: timedelta) -> QueryString: ########################################################################### # Descriptors - @t.overload + @overload def __get__(self, obj: Table, objtype=None) -> date: ... - @t.overload + @overload def __get__(self, obj: None, objtype=None) -> Date: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self - def __set__(self, obj, value: t.Union[date, None]): + def __set__(self, obj, value: Union[date, None]): obj.__dict__[self._meta.name] = value @@ -1202,16 +1210,16 @@ def __sub__(self, value: timedelta) -> QueryString: ########################################################################### # Descriptors - @t.overload + @overload def __get__(self, obj: Table, objtype=None) -> time: ... - @t.overload + @overload def __get__(self, obj: None, objtype=None) -> Time: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self - def __set__(self, obj, value: t.Union[time, None]): + def __set__(self, obj, value: Union[time, None]): obj.__dict__[self._meta.name] = value @@ -1292,16 +1300,16 @@ def __sub__(self, value: timedelta) -> QueryString: ########################################################################### # Descriptors - @t.overload + @overload def __get__(self, obj: Table, objtype=None) -> timedelta: ... - @t.overload + @overload def __get__(self, obj: None, objtype=None) -> Interval: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self - def __set__(self, obj, value: t.Union[timedelta, None]): + def __set__(self, obj, value: Union[timedelta, None]): obj.__dict__[self._meta.name] = value @@ -1333,7 +1341,7 @@ class Band(Table): def __init__( self, - default: t.Union[bool, Enum, t.Callable[[], bool], None] = False, + default: Union[bool, Enum, Callable[[], bool], None] = False, **kwargs: Unpack[ColumnKwargs], ) -> None: self._validate_default(default, (bool, None)) @@ -1383,16 +1391,16 @@ def ne(self, value) -> Where: ########################################################################### # Descriptors - @t.overload + @overload def __get__(self, obj: Table, objtype=None) -> bool: ... - @t.overload + @overload def __get__(self, obj: None, objtype=None) -> Boolean: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self - def __set__(self, obj, value: t.Union[bool, None]): + def __set__(self, obj, value: Union[bool, None]): obj.__dict__[self._meta.name] = value @@ -1443,14 +1451,14 @@ def column_type(self): return "NUMERIC" @property - def precision(self) -> t.Optional[int]: + def precision(self) -> Optional[int]: """ The total number of digits allowed. """ return self.digits[0] if self.digits is not None else None @property - def scale(self) -> t.Optional[int]: + def scale(self) -> Optional[int]: """ The number of digits after the decimal point. """ @@ -1458,9 +1466,9 @@ def scale(self) -> t.Optional[int]: def __init__( self, - digits: t.Optional[t.Tuple[int, int]] = None, - default: t.Union[ - decimal.Decimal, Enum, t.Callable[[], decimal.Decimal], None + digits: Optional[tuple[int, int]] = None, + default: Union[ + decimal.Decimal, Enum, Callable[[], decimal.Decimal], None ] = decimal.Decimal(0.0), **kwargs: Unpack[ColumnKwargs], ) -> None: @@ -1483,16 +1491,16 @@ def __init__( ########################################################################### # Descriptors - @t.overload + @overload def __get__(self, obj: Table, objtype=None) -> decimal.Decimal: ... - @t.overload + @overload def __get__(self, obj: None, objtype=None) -> Numeric: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self - def __set__(self, obj, value: t.Union[decimal.Decimal, None]): + def __set__(self, obj, value: Union[decimal.Decimal, None]): obj.__dict__[self._meta.name] = value @@ -1504,16 +1512,16 @@ class Decimal(Numeric): ########################################################################### # Descriptors - @t.overload + @overload def __get__(self, obj: Table, objtype=None) -> decimal.Decimal: ... - @t.overload + @overload def __get__(self, obj: None, objtype=None) -> Decimal: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self - def __set__(self, obj, value: t.Union[decimal.Decimal, None]): + def __set__(self, obj, value: Union[decimal.Decimal, None]): obj.__dict__[self._meta.name] = value @@ -1542,7 +1550,7 @@ class Concert(Table): def __init__( self, - default: t.Union[float, Enum, t.Callable[[], float], None] = 0.0, + default: Union[float, Enum, Callable[[], float], None] = 0.0, **kwargs: Unpack[ColumnKwargs], ) -> None: self._validate_default(default, (float, None)) @@ -1552,16 +1560,16 @@ def __init__( ########################################################################### # Descriptors - @t.overload + @overload def __get__(self, obj: Table, objtype=None) -> float: ... - @t.overload + @overload def __get__(self, obj: None, objtype=None) -> Real: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self - def __set__(self, obj, value: t.Union[float, None]): + def __set__(self, obj, value: Union[float, None]): obj.__dict__[self._meta.name] = value @@ -1573,16 +1581,16 @@ class Float(Real): ########################################################################### # Descriptors - @t.overload + @overload def __get__(self, obj: Table, objtype=None) -> float: ... - @t.overload + @overload def __get__(self, obj: None, objtype=None) -> Float: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self - def __set__(self, obj, value: t.Union[float, None]): + def __set__(self, obj, value: Union[float, None]): obj.__dict__[self._meta.name] = value @@ -1598,16 +1606,16 @@ def column_type(self): ########################################################################### # Descriptors - @t.overload + @overload def __get__(self, obj: Table, objtype=None) -> float: ... - @t.overload + @overload def __get__(self, obj: None, objtype=None) -> DoublePrecision: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self - def __set__(self, obj, value: t.Union[float, None]): + def __set__(self, obj, value: Union[float, None]): obj.__dict__[self._meta.name] = value @@ -1619,7 +1627,7 @@ class ForeignKeySetupResponse: is_lazy: bool -class ForeignKey(Column, t.Generic[ReferencedTable]): +class ForeignKey(Column, Generic[ReferencedTable]): """ Used to reference another table. Uses the same type as the primary key column on the table it references. @@ -1832,50 +1840,50 @@ def value_type(self): target_column = self._foreign_key_meta.resolved_target_column return target_column.value_type - @t.overload + @overload def __init__( self, - references: t.Type[ReferencedTable], - default: t.Any = None, + references: type[ReferencedTable], + default: Any = None, null: bool = True, on_delete: OnDelete = OnDelete.cascade, on_update: OnUpdate = OnUpdate.cascade, - target_column: t.Union[str, Column, None] = None, + target_column: Union[str, Column, None] = None, **kwargs, ) -> None: ... - @t.overload + @overload def __init__( self, references: LazyTableReference, - default: t.Any = None, + default: Any = None, null: bool = True, on_delete: OnDelete = OnDelete.cascade, on_update: OnUpdate = OnUpdate.cascade, - target_column: t.Union[str, Column, None] = None, + target_column: Union[str, Column, None] = None, **kwargs, ) -> None: ... - @t.overload + @overload def __init__( self, references: str, - default: t.Any = None, + default: Any = None, null: bool = True, on_delete: OnDelete = OnDelete.cascade, on_update: OnUpdate = OnUpdate.cascade, - target_column: t.Union[str, Column, None] = None, + target_column: Union[str, Column, None] = None, **kwargs, ) -> None: ... def __init__( self, - references: t.Union[t.Type[ReferencedTable], LazyTableReference, str], - default: t.Any = None, + references: Union[type[ReferencedTable], LazyTableReference, str], + default: Any = None, null: bool = True, on_delete: OnDelete = OnDelete.cascade, on_update: OnUpdate = OnUpdate.cascade, - target_column: t.Union[str, Column, None] = None, + target_column: Union[str, Column, None] = None, **kwargs, ) -> None: from piccolo.table import Table @@ -1914,7 +1922,7 @@ def __init__( target_column=target_column, ) - def _setup(self, table_class: t.Type[Table]) -> ForeignKeySetupResponse: + def _setup(self, table_class: type[Table]) -> ForeignKeySetupResponse: """ This is called by the ``TableMetaclass``. A ``ForeignKey`` column can only be completely setup once it's parent ``Table`` is known. @@ -1965,9 +1973,9 @@ def _setup(self, table_class: t.Type[Table]) -> ForeignKeySetupResponse: if is_table_class: # Record the reverse relationship on the target table. - t.cast( - t.Type[Table], references - )._meta._foreign_key_references.append(self) + cast(type[Table], references)._meta._foreign_key_references.append( + self + ) # Allow columns on the referenced table to be accessed via # auto completion. @@ -1982,8 +1990,8 @@ def copy(self) -> ForeignKey: return column def all_columns( - self, exclude: t.Optional[t.List[t.Union[Column, str]]] = None - ) -> t.List[Column]: + self, exclude: Optional[list[Union[Column, str]]] = None + ) -> list[Column]: """ Allow a user to access all of the columns on the related table. This is intended for use with ``select`` queries, and saves the user from @@ -2088,8 +2096,8 @@ class Treasurer(Table): return foreign_key def all_related( - self, exclude: t.Optional[t.List[t.Union[ForeignKey, str]]] = None - ) -> t.List[ForeignKey]: + self, exclude: Optional[list[Union[ForeignKey, str]]] = None + ) -> list[ForeignKey]: """ Returns each ``ForeignKey`` column on the related table. This is intended for use with ``objects`` queries, where you want to return @@ -2157,7 +2165,7 @@ def set_proxy_columns(self) -> None: _fk_meta.proxy_columns.append(_column) @property - def _(self) -> t.Type[ReferencedTable]: + def _(self) -> type[ReferencedTable]: """ This allows us specify joins in a way which is friendly to static type checkers like Mypy and Pyright. @@ -2188,9 +2196,9 @@ def _(self) -> t.Type[ReferencedTable]: easily know if any of your joins contain typos. """ - return t.cast(t.Type[ReferencedTable], self) + return cast(type[ReferencedTable], self) - def __getattribute__(self, name: str) -> t.Union[Column, t.Any]: + def __getattribute__(self, name: str) -> Union[Column, Any]: """ Returns attributes unmodified unless they're Column instances, in which case a copy is returned with an updated call_chain (which records the @@ -2217,7 +2225,7 @@ def __getattribute__(self, name: str) -> t.Union[Column, t.Any]: if name.startswith("_"): return value - foreignkey_class: t.Type[ForeignKey] = object.__getattribute__( + foreignkey_class: type[ForeignKey] = object.__getattribute__( self, "__class__" ) @@ -2270,21 +2278,21 @@ def __getattribute__(self, name: str) -> t.Union[Column, t.Any]: ########################################################################### # Descriptors - @t.overload - def __get__(self, obj: Table, objtype=None) -> t.Any: ... + @overload + def __get__(self, obj: Table, objtype=None) -> Any: ... - @t.overload + @overload def __get__( self, obj: None, objtype=None ) -> ForeignKey[ReferencedTable]: ... - @t.overload - def __get__(self, obj: t.Any, objtype=None) -> t.Any: ... + @overload + def __get__(self, obj: Any, objtype=None) -> Any: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self - def __set__(self, obj, value: t.Any): + def __set__(self, obj, value: Any): obj.__dict__[self._meta.name] = value @@ -2307,11 +2315,11 @@ class JSON(Column): def __init__( self, - default: t.Union[ + default: Union[ str, - t.List, - t.Dict, - t.Callable[[], t.Union[str, t.List, t.Dict]], + list, + dict, + Callable[[], Union[str, list, dict]], None, ] = "{}", **kwargs: Unpack[ColumnKwargs], @@ -2324,7 +2332,7 @@ def __init__( self.default = default super().__init__(default=default, **kwargs) - self.json_operator: t.Optional[str] = None + self.json_operator: Optional[str] = None @property def column_type(self): @@ -2336,7 +2344,7 @@ def column_type(self): ########################################################################### - def arrow(self, key: t.Union[str, int, QueryString]) -> GetChildElement: + def arrow(self, key: Union[str, int, QueryString]) -> GetChildElement: """ Allows a child element of the JSON structure to be returned - for example:: @@ -2352,7 +2360,7 @@ def arrow(self, key: t.Union[str, int, QueryString]) -> GetChildElement: return GetChildElement(identifier=self, key=key, alias=alias) def __getitem__( - self, value: t.Union[str, int, QueryString] + self, value: Union[str, int, QueryString] ) -> GetChildElement: """ A shortcut for the ``arrow`` method, used for retrieving a child @@ -2371,7 +2379,7 @@ def __getitem__( def from_path( self, - path: t.List[t.Union[str, int]], + path: list[Union[str, int]], ) -> GetElementFromPath: """ Allows an element of the JSON structure to be returned, which can be @@ -2407,16 +2415,16 @@ def from_path( ########################################################################### # Descriptors - @t.overload + @overload def __get__(self, obj: Table, objtype=None) -> str: ... - @t.overload + @overload def __get__(self, obj: None, objtype=None) -> JSON: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self - def __set__(self, obj, value: t.Union[str, t.Dict]): + def __set__(self, obj, value: Union[str, dict]): obj.__dict__[self._meta.name] = value @@ -2440,16 +2448,16 @@ def column_type(self): ########################################################################### # Descriptors - @t.overload + @overload def __get__(self, obj: Table, objtype=None) -> str: ... - @t.overload + @overload def __get__(self, obj: None, objtype=None) -> JSONB: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self - def __set__(self, obj, value: t.Union[str, t.Dict]): + def __set__(self, obj, value: Union[str, dict]): obj.__dict__[self._meta.name] = value @@ -2489,12 +2497,12 @@ def column_type(self): def __init__( self, - default: t.Union[ + default: Union[ bytes, bytearray, Enum, - t.Callable[[], bytes], - t.Callable[[], bytearray], + Callable[[], bytes], + Callable[[], bytearray], None, ] = b"", **kwargs: Unpack[ColumnKwargs], @@ -2510,10 +2518,10 @@ def __init__( ########################################################################### # Descriptors - @t.overload + @overload def __get__(self, obj: Table, objtype=None) -> bytes: ... - @t.overload + @overload def __get__(self, obj: None, objtype=None) -> Bytea: ... def __get__(self, obj, objtype=None): @@ -2531,10 +2539,10 @@ class Blob(Bytea): ########################################################################### # Descriptors - @t.overload + @overload def __get__(self, obj: Table, objtype=None) -> bytes: ... - @t.overload + @overload def __get__(self, obj: None, objtype=None) -> Blob: ... def __get__(self, obj, objtype=None): @@ -2594,9 +2602,7 @@ class Ticket(Table): def __init__( self, base_column: Column, - default: t.Union[ - t.List, Enum, t.Callable[[], t.List], None - ] = ListProxy(), + default: Union[list, Enum, Callable[[], list], None] = ListProxy(), **kwargs: Unpack[ColumnKwargs], ) -> None: if isinstance(base_column, ForeignKey): @@ -2622,7 +2628,7 @@ def __init__( self.base_column = base_column self.default = default - self.index: t.Optional[int] = None + self.index: Optional[int] = None super().__init__(default=default, base_column=base_column, **kwargs) @property @@ -2641,7 +2647,7 @@ def column_type(self): ) raise Exception("Unrecognized engine type") - def _setup_base_column(self, table_class: t.Type[Table]): + def _setup_base_column(self, table_class: type[Table]): """ Called from the ``Table.__init_subclass__`` - makes sure that the ``base_column`` has a reference to the parent table. @@ -2687,7 +2693,7 @@ def _get_inner_column(self) -> Column: else: return self.base_column - def _get_inner_value_type(self) -> t.Type: + def _get_inner_value_type(self) -> type: """ A helper function to get the innermost value type for the array. For example:: @@ -2728,7 +2734,7 @@ def __getitem__(self, value: int) -> Array: if value < 0: raise ValueError("Only positive integers are allowed.") - instance = t.cast(Array, self.copy()) + instance = cast(Array, self.copy()) # We deliberately add 1, as Postgres treats the first array element # as index 1. @@ -2751,7 +2757,7 @@ def get_select_string( return QueryString(select_string) - def any(self, value: t.Any) -> Where: + def any(self, value: Any) -> Where: """ Check if any of the items in the array match the given value. @@ -2769,7 +2775,7 @@ def any(self, value: t.Any) -> Where: else: raise ValueError("Unrecognised engine type") - def not_any(self, value: t.Any) -> Where: + def not_any(self, value: Any) -> Where: """ Check if the given value isn't in the array. @@ -2787,7 +2793,7 @@ def not_any(self, value: t.Any) -> Where: else: raise ValueError("Unrecognised engine type") - def all(self, value: t.Any) -> Where: + def all(self, value: Any) -> Where: """ Check if all of the items in the array match the given value. @@ -2805,7 +2811,7 @@ def all(self, value: t.Any) -> Where: else: raise ValueError("Unrecognised engine type") - def cat(self, value: t.Union[t.Any, t.List[t.Any]]) -> QueryString: + def cat(self, value: Union[Any, list[Any]]) -> QueryString: """ Used in an ``update`` query to append items to an array. @@ -2836,20 +2842,20 @@ def cat(self, value: t.Union[t.Any, t.List[t.Any]]) -> QueryString: db_column_name = self._meta.db_column_name return QueryString(f'array_cat("{db_column_name}", {{}})', value) - def __add__(self, value: t.Union[t.Any, t.List[t.Any]]) -> QueryString: + def __add__(self, value: Union[Any, list[Any]]) -> QueryString: return self.cat(value) ########################################################################### # Descriptors - @t.overload - def __get__(self, obj: Table, objtype=None) -> t.List[t.Any]: ... + @overload + def __get__(self, obj: Table, objtype=None) -> list[Any]: ... - @t.overload + @overload def __get__(self, obj: None, objtype=None) -> Array: ... def __get__(self, obj, objtype=None): return obj.__dict__[self._meta.name] if obj else self - def __set__(self, obj, value: t.List[t.Any]): + def __set__(self, obj, value: list[Any]): obj.__dict__[self._meta.name] = value diff --git a/piccolo/columns/combination.py b/piccolo/columns/combination.py index ede58f09b..a8179801f 100644 --- a/piccolo/columns/combination.py +++ b/piccolo/columns/combination.py @@ -1,17 +1,17 @@ from __future__ import annotations -import typing as t +from typing import TYPE_CHECKING, Any, Union from piccolo.columns.operators.comparison import ( ComparisonOperator, Equal, IsNull, ) -from piccolo.custom_types import Combinable, Iterable +from piccolo.custom_types import Combinable, CustomIterable from piccolo.querystring import QueryString from piccolo.utils.sql_values import convert_to_sql_value -if t.TYPE_CHECKING: +if TYPE_CHECKING: from piccolo.columns.base import Column @@ -59,7 +59,7 @@ def __str__(self): class And(Combination): operator = "AND" - def get_column_values(self) -> t.Dict[Column, t.Any]: + def get_column_values(self) -> dict[Column, Any]: """ This is used by `get_or_create` to know which values to assign if the row doesn't exist in the database. @@ -109,7 +109,7 @@ class Undefined: class WhereRaw(CombinableMixin): __slots__ = ("querystring",) - def __init__(self, sql: str, *args: t.Any) -> None: + def __init__(self, sql: str, *args: Any) -> None: """ Execute raw SQL queries in your where clause. Use with caution! @@ -145,9 +145,9 @@ class Where(CombinableMixin): def __init__( self, column: Column, - value: t.Any = UNDEFINED, - values: t.Union[Iterable, Undefined] = UNDEFINED, - operator: t.Type[ComparisonOperator] = ComparisonOperator, + value: Any = UNDEFINED, + values: Union[CustomIterable, Undefined] = UNDEFINED, + operator: type[ComparisonOperator] = ComparisonOperator, ) -> None: """ We use the UNDEFINED value to show the value was deliberately @@ -163,7 +163,7 @@ def __init__( self.operator = operator - def clean_value(self, value: t.Any) -> t.Any: + def clean_value(self, value: Any) -> Any: """ If a where clause contains a ``Table`` instance, we should convert that to a column reference. For example: @@ -200,7 +200,7 @@ def values_querystring(self) -> QueryString: @property def querystring(self) -> QueryString: - args: t.List[t.Any] = [] + args: list[Any] = [] if self.value != UNDEFINED: args.append(self.value) @@ -219,7 +219,7 @@ def querystring(self) -> QueryString: @property def querystring_for_update_and_delete(self) -> QueryString: - args: t.List[t.Any] = [] + args: list[Any] = [] if self.value != UNDEFINED: args.append(self.value) diff --git a/piccolo/columns/defaults/base.py b/piccolo/columns/defaults/base.py index 062162032..fcf46bd85 100644 --- a/piccolo/columns/defaults/base.py +++ b/piccolo/columns/defaults/base.py @@ -1,7 +1,7 @@ from __future__ import annotations -import typing as t from abc import ABC, abstractmethod +from typing import Any from piccolo.utils.repr import repr_class_instance @@ -18,10 +18,10 @@ def sqlite(self) -> str: pass @abstractmethod - def python(self) -> t.Any: + def python(self) -> Any: pass - def get_postgres_interval_string(self, attributes: t.List[str]) -> str: + def get_postgres_interval_string(self, attributes: list[str]) -> str: """ Returns a string usable as an interval argument in Postgres e.g. "1 day 2 hour". @@ -39,7 +39,7 @@ def get_postgres_interval_string(self, attributes: t.List[str]) -> str: return " ".join(interval_components) - def get_sqlite_interval_string(self, attributes: t.List[str]) -> str: + def get_sqlite_interval_string(self, attributes: list[str]) -> str: """ Returns a string usable as an interval argument in SQLite e.g. "'-2 hours', '1 days'". diff --git a/piccolo/columns/defaults/date.py b/piccolo/columns/defaults/date.py index 423f112ca..b802c6764 100644 --- a/piccolo/columns/defaults/date.py +++ b/piccolo/columns/defaults/date.py @@ -1,8 +1,9 @@ from __future__ import annotations import datetime -import typing as t +from collections.abc import Callable from enum import Enum +from typing import Union from .base import Default @@ -102,14 +103,14 @@ def from_date(cls, instance: datetime.date): # Might add an enum back which encapsulates all of the options. -DateArg = t.Union[ +DateArg = Union[ DateOffset, DateCustom, DateNow, Enum, None, datetime.date, - t.Callable[[], datetime.date], + Callable[[], datetime.date], ] diff --git a/piccolo/columns/defaults/interval.py b/piccolo/columns/defaults/interval.py index 4d5f72ae8..798a4a050 100644 --- a/piccolo/columns/defaults/interval.py +++ b/piccolo/columns/defaults/interval.py @@ -1,8 +1,9 @@ from __future__ import annotations import datetime -import typing as t +from collections.abc import Callable from enum import Enum +from typing import Union from .base import Default @@ -75,12 +76,12 @@ def from_timedelta(cls, instance: datetime.timedelta): ############################################################################### -IntervalArg = t.Union[ +IntervalArg = Union[ IntervalCustom, Enum, None, datetime.timedelta, - t.Callable[[], datetime.timedelta], + Callable[[], datetime.timedelta], ] diff --git a/piccolo/columns/defaults/time.py b/piccolo/columns/defaults/time.py index 9b72416ea..a32dcdf47 100644 --- a/piccolo/columns/defaults/time.py +++ b/piccolo/columns/defaults/time.py @@ -1,8 +1,9 @@ from __future__ import annotations import datetime -import typing as t +from collections.abc import Callable from enum import Enum +from typing import Union from .base import Default @@ -89,14 +90,14 @@ def from_time(cls, instance: datetime.time): ) -TimeArg = t.Union[ +TimeArg = Union[ TimeCustom, TimeNow, TimeOffset, Enum, None, datetime.time, - t.Callable[[], datetime.time], + Callable[[], datetime.time], ] diff --git a/piccolo/columns/defaults/timestamp.py b/piccolo/columns/defaults/timestamp.py index 8fb984f77..11388c694 100644 --- a/piccolo/columns/defaults/timestamp.py +++ b/piccolo/columns/defaults/timestamp.py @@ -1,8 +1,9 @@ from __future__ import annotations import datetime -import typing as t +from collections.abc import Callable from enum import Enum +from typing import Union from .base import Default @@ -134,7 +135,7 @@ class DatetimeDefault: ############################################################################### -TimestampArg = t.Union[ +TimestampArg = Union[ TimestampCustom, TimestampNow, TimestampOffset, @@ -142,7 +143,7 @@ class DatetimeDefault: None, datetime.datetime, DatetimeDefault, - t.Callable[[], datetime.datetime], + Callable[[], datetime.datetime], ] diff --git a/piccolo/columns/defaults/timestamptz.py b/piccolo/columns/defaults/timestamptz.py index f0e792973..1cb6d32ff 100644 --- a/piccolo/columns/defaults/timestamptz.py +++ b/piccolo/columns/defaults/timestamptz.py @@ -1,8 +1,9 @@ from __future__ import annotations import datetime -import typing as t +from collections.abc import Callable from enum import Enum +from typing import Union from .timestamp import TimestampCustom, TimestampNow, TimestampOffset @@ -68,14 +69,14 @@ def from_datetime(cls, instance: datetime.datetime): # type: ignore ) -TimestamptzArg = t.Union[ +TimestamptzArg = Union[ TimestamptzCustom, TimestamptzNow, TimestamptzOffset, Enum, None, datetime.datetime, - t.Callable[[], datetime.datetime], + Callable[[], datetime.datetime], ] diff --git a/piccolo/columns/defaults/uuid.py b/piccolo/columns/defaults/uuid.py index 17b07021c..5f2289612 100644 --- a/piccolo/columns/defaults/uuid.py +++ b/piccolo/columns/defaults/uuid.py @@ -1,6 +1,7 @@ -import typing as t import uuid +from collections.abc import Callable from enum import Enum +from typing import Union from .base import Default @@ -22,7 +23,7 @@ def python(self): return uuid.uuid4() -UUIDArg = t.Union[UUID4, uuid.UUID, str, Enum, None, t.Callable[[], uuid.UUID]] +UUIDArg = Union[UUID4, uuid.UUID, str, Enum, None, Callable[[], uuid.UUID]] __all__ = ["UUIDArg", "UUID4"] diff --git a/piccolo/columns/m2m.py b/piccolo/columns/m2m.py index 0a75f9eb7..25d93f731 100644 --- a/piccolo/columns/m2m.py +++ b/piccolo/columns/m2m.py @@ -1,8 +1,9 @@ from __future__ import annotations import inspect -import typing as t +from collections.abc import Sequence from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Optional, Union from piccolo.columns.column_types import ( JSON, @@ -15,7 +16,7 @@ from piccolo.utils.list import flatten from piccolo.utils.sync import run_sync -if t.TYPE_CHECKING: +if TYPE_CHECKING: from piccolo.table import Table @@ -151,12 +152,12 @@ def get_select_string( @dataclass class M2MMeta: - joining_table: t.Union[t.Type[Table], LazyTableReference] - _foreign_key_columns: t.Optional[t.List[ForeignKey]] = None + joining_table: Union[type[Table], LazyTableReference] + _foreign_key_columns: Optional[list[ForeignKey]] = None # Set by the Table Metaclass: - _name: t.Optional[str] = None - _table: t.Optional[t.Type[Table]] = None + _name: Optional[str] = None + _table: Optional[type[Table]] = None @property def name(self) -> str: @@ -167,7 +168,7 @@ def name(self) -> str: return self._name @property - def table(self) -> t.Type[Table]: + def table(self) -> type[Table]: if not self._table: raise ValueError( "`_table` isn't defined - the Table Metaclass should set it." @@ -175,7 +176,7 @@ def table(self) -> t.Type[Table]: return self._table @property - def resolved_joining_table(self) -> t.Type[Table]: + def resolved_joining_table(self) -> type[Table]: """ Evaluates the ``joining_table`` attribute if it's a ``LazyTableReference``, raising a ``ValueError`` if it fails, otherwise @@ -196,7 +197,7 @@ def resolved_joining_table(self) -> t.Type[Table]: ) @property - def foreign_key_columns(self) -> t.List[ForeignKey]: + def foreign_key_columns(self) -> list[ForeignKey]: if not self._foreign_key_columns: self._foreign_key_columns = ( self.resolved_joining_table._meta.foreign_key_columns[:2] @@ -236,7 +237,7 @@ class GenreToBand(Table): raise ValueError("No matching foreign key column found!") @property - def primary_table(self) -> t.Type[Table]: + def primary_table(self) -> type[Table]: return self.primary_foreign_key._foreign_key_meta.resolved_references @property @@ -251,7 +252,7 @@ def secondary_foreign_key(self) -> ForeignKey: raise ValueError("No matching foreign key column found!") @property - def secondary_table(self) -> t.Type[Table]: + def secondary_table(self) -> type[Table]: return self.secondary_foreign_key._foreign_key_meta.resolved_references @@ -259,11 +260,11 @@ def secondary_table(self) -> t.Type[Table]: class M2MAddRelated: target_row: Table m2m: M2M - rows: t.Sequence[Table] - extra_column_values: t.Dict[t.Union[Column, str], t.Any] + rows: Sequence[Table] + extra_column_values: dict[Union[Column, str], Any] @property - def resolved_extra_column_values(self) -> t.Dict[str, t.Any]: + def resolved_extra_column_values(self) -> dict[str, Any]: return { i._meta.name if isinstance(i, Column) else i: j for i, j in self.extra_column_values.items() @@ -327,7 +328,7 @@ def __await__(self): class M2MRemoveRelated: target_row: Table m2m: M2M - rows: t.Sequence[Table] + rows: Sequence[Table] async def run(self): fk = self.m2m._meta.secondary_foreign_key @@ -404,8 +405,8 @@ def __await__(self): class M2M: def __init__( self, - joining_table: t.Union[t.Type[Table], LazyTableReference], - foreign_key_columns: t.Optional[t.List[ForeignKey]] = None, + joining_table: Union[type[Table], LazyTableReference], + foreign_key_columns: Optional[list[ForeignKey]] = None, ): """ :param joining_table: @@ -428,7 +429,7 @@ def __init__( def __call__( self, - *columns: t.Union[Column, t.List[Column]], + *columns: Union[Column, list[Column]], as_list: bool = False, load_json: bool = False, ) -> M2MSelect: diff --git a/piccolo/columns/readable.py b/piccolo/columns/readable.py index ebd32bf51..cd02c5c91 100644 --- a/piccolo/columns/readable.py +++ b/piccolo/columns/readable.py @@ -1,11 +1,12 @@ from __future__ import annotations -import typing as t +from collections.abc import Sequence from dataclasses import dataclass +from typing import TYPE_CHECKING from piccolo.querystring import QueryString, Selectable -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from piccolo.columns.base import Column @@ -18,7 +19,7 @@ class Readable(Selectable): """ template: str - columns: t.Sequence[Column] + columns: Sequence[Column] output_name: str = "readable" @property diff --git a/piccolo/columns/reference.py b/piccolo/columns/reference.py index 9870f119a..532cf309d 100644 --- a/piccolo/columns/reference.py +++ b/piccolo/columns/reference.py @@ -6,10 +6,10 @@ import importlib import inspect -import typing as t from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Optional -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from piccolo.columns.column_types import ForeignKey from piccolo.table import Table @@ -36,8 +36,8 @@ class LazyTableReference: """ table_class_name: str - app_name: t.Optional[str] = None - module_path: t.Optional[str] = None + app_name: Optional[str] = None + module_path: Optional[str] = None def __post_init__(self): if self.app_name is None and self.module_path is None: @@ -49,7 +49,7 @@ def __post_init__(self): "Specify either app_name or module_path - not both." ) - def resolve(self) -> t.Type[Table]: + def resolve(self) -> type[Table]: if self.app_name is not None: from piccolo.conf.apps import Finder @@ -60,7 +60,7 @@ def resolve(self) -> t.Type[Table]: if self.module_path: module = importlib.import_module(self.module_path) - table: t.Optional[t.Type[Table]] = getattr( + table: Optional[type[Table]] = getattr( module, self.table_class_name, None ) @@ -91,9 +91,9 @@ def __str__(self): @dataclass class LazyColumnReferenceStore: - foreign_key_columns: t.List[ForeignKey] = field(default_factory=list) + foreign_key_columns: list[ForeignKey] = field(default_factory=list) - def for_table(self, table: t.Type[Table]) -> t.List[ForeignKey]: + def for_table(self, table: type[Table]) -> list[ForeignKey]: return [ i for i in self.foreign_key_columns @@ -101,7 +101,7 @@ def for_table(self, table: t.Type[Table]) -> t.List[ForeignKey]: and i._foreign_key_meta.references.resolve() is table ] - def for_tablename(self, tablename: str) -> t.List[ForeignKey]: + def for_tablename(self, tablename: str) -> list[ForeignKey]: return [ i for i in self.foreign_key_columns diff --git a/piccolo/conf/apps.py b/piccolo/conf/apps.py index c5a1fd7dc..d000a52cf 100644 --- a/piccolo/conf/apps.py +++ b/piccolo/conf/apps.py @@ -6,11 +6,12 @@ import os import pathlib import traceback -import typing as t from abc import abstractmethod +from collections.abc import Callable, Sequence from dataclasses import dataclass, field from importlib import import_module from types import ModuleType +from typing import Optional, Union, cast import black @@ -45,12 +46,12 @@ def get_package(name: str) -> str: def table_finder( - modules: t.Sequence[str], - package: t.Optional[str] = None, - include_tags: t.Optional[t.Sequence[str]] = None, - exclude_tags: t.Optional[t.Sequence[str]] = None, + modules: Sequence[str], + package: Optional[str] = None, + include_tags: Optional[Sequence[str]] = None, + exclude_tags: Optional[Sequence[str]] = None, exclude_imported: bool = False, -) -> t.List[t.Type[Table]]: +) -> list[type[Table]]: """ Rather than explicitly importing and registering table classes with the ``AppConfig``, ``table_finder`` can be used instead. It imports any ``Table`` @@ -95,7 +96,7 @@ class Task(Table): # included # 'blog.tables', instead of ['blog.tables']. modules = [modules] - table_subclasses: t.List[t.Type[Table]] = [] + table_subclasses: list[type[Table]] = [] for module_path in modules: full_module_path = ( @@ -151,9 +152,9 @@ class Command: """ - callable: t.Callable - command_name: t.Optional[str] = None - aliases: t.List[str] = field(default_factory=list) + callable: Callable + command_name: Optional[str] = None + aliases: list[str] = field(default_factory=list) @dataclass @@ -181,12 +182,10 @@ class AppConfig: """ app_name: str - migrations_folder_path: t.Union[str, pathlib.Path] - table_classes: t.List[t.Type[Table]] = field(default_factory=list) - migration_dependencies: t.List[str] = field(default_factory=list) - commands: t.List[t.Union[t.Callable, Command]] = field( - default_factory=list - ) + migrations_folder_path: Union[str, pathlib.Path] + table_classes: list[type[Table]] = field(default_factory=list) + migration_dependencies: list[str] = field(default_factory=list) + commands: list[Union[Callable, Command]] = field(default_factory=list) @property def resolved_migrations_folder_path(self) -> str: @@ -197,21 +196,21 @@ def resolved_migrations_folder_path(self) -> str: ) def __post_init__(self) -> None: - self._migration_dependency_app_configs: t.Optional[ - t.List[AppConfig] - ] = None + self._migration_dependency_app_configs: Optional[list[AppConfig]] = ( + None + ) - def register_table(self, table_class: t.Type[Table]): + def register_table(self, table_class: type[Table]): self.table_classes.append(table_class) return table_class - def get_commands(self) -> t.List[Command]: + def get_commands(self) -> list[Command]: return [ i if isinstance(i, Command) else Command(i) for i in self.commands ] @property - def migration_dependency_app_configs(self) -> t.List[AppConfig]: + def migration_dependency_app_configs(self) -> list[AppConfig]: """ Get all of the ``AppConfig`` instances from this app's migration dependencies. @@ -219,8 +218,8 @@ def migration_dependency_app_configs(self) -> t.List[AppConfig]: # We cache the value so it's more efficient, and also so we can set the # underlying value in unit tests for easier mocking. if self._migration_dependency_app_configs is None: - modules: t.List[PiccoloAppModule] = [ - t.cast(PiccoloAppModule, import_module(module_path)) + modules: list[PiccoloAppModule] = [ + cast(PiccoloAppModule, import_module(module_path)) for module_path in self.migration_dependencies ] self._migration_dependency_app_configs = [ @@ -229,7 +228,7 @@ def migration_dependency_app_configs(self) -> t.List[AppConfig]: return self._migration_dependency_app_configs - def get_table_with_name(self, table_class_name: str) -> t.Type[Table]: + def get_table_with_name(self, table_class_name: str) -> type[Table]: """ Returns a ``Table`` subclass with the given name from this app, if it exists. Otherwise raises a ``ValueError``. @@ -256,9 +255,9 @@ class AppRegistry: """ - def __init__(self, apps: t.Optional[t.List[str]] = None): + def __init__(self, apps: Optional[list[str]] = None): self.apps = apps or [] - self.app_configs: t.Dict[str, AppConfig] = {} + self.app_configs: dict[str, AppConfig] = {} app_names = [] for app in self.apps: @@ -282,7 +281,7 @@ def __init__(self, apps: t.Optional[t.List[str]] = None): self._validate_app_names(app_names) @staticmethod - def _validate_app_names(app_names: t.List[str]): + def _validate_app_names(app_names: list[str]): """ Raise a ValueError if an app_name is repeated. """ @@ -298,10 +297,10 @@ def _validate_app_names(app_names: t.List[str]): "multiple times." ) - def get_app_config(self, app_name: str) -> t.Optional[AppConfig]: + def get_app_config(self, app_name: str) -> Optional[AppConfig]: return self.app_configs.get(app_name) - def get_table_classes(self, app_name: str) -> t.List[t.Type[Table]]: + def get_table_classes(self, app_name: str) -> list[type[Table]]: """ Returns each Table subclass defined in the given app if it exists. Otherwise raises a ValueError. @@ -317,7 +316,7 @@ def get_table_classes(self, app_name: str) -> t.List[t.Type[Table]]: def get_table_with_name( self, app_name: str, table_class_name: str - ) -> t.Optional[t.Type[Table]]: + ) -> Optional[type[Table]]: """ Returns a Table subclass registered with the given app if it exists. Otherwise raises a ValueError. @@ -357,8 +356,8 @@ def __init__(self, diagnose: bool = False): self.diagnose = diagnose def _deduplicate( - self, config_modules: t.List[PiccoloAppModule] - ) -> t.List[PiccoloAppModule]: + self, config_modules: list[PiccoloAppModule] + ) -> list[PiccoloAppModule]: """ Remove all duplicates - just leaving the first instance. """ @@ -366,8 +365,8 @@ def _deduplicate( return list({c: None for c in config_modules}.keys()) def _import_app_modules( - self, config_module_paths: t.List[str] - ) -> t.List[PiccoloAppModule]: + self, config_module_paths: list[str] + ) -> list[PiccoloAppModule]: """ Import all piccolo_app.py modules within your apps, and all dependencies. @@ -376,7 +375,7 @@ def _import_app_modules( for config_module_path in config_module_paths: try: - config_module = t.cast( + config_module = cast( PiccoloAppModule, import_module(config_module_path) ) except ImportError as e: @@ -392,8 +391,8 @@ def _import_app_modules( return config_modules def get_piccolo_conf_module( - self, module_name: t.Optional[str] = None - ) -> t.Optional[PiccoloConfModule]: + self, module_name: Optional[str] = None + ) -> Optional[PiccoloConfModule]: """ Searches the path for a 'piccolo_conf.py' module to import. The location searched can be overriden by: @@ -413,7 +412,7 @@ def get_piccolo_conf_module( module_name = DEFAULT_MODULE_NAME try: - module = t.cast(PiccoloConfModule, import_module(module_name)) + module = cast(PiccoloConfModule, import_module(module_name)) except ModuleNotFoundError as exc: if self.diagnose: colored_warning( @@ -459,10 +458,10 @@ def get_app_registry(self) -> AppRegistry: return getattr(piccolo_conf_module, "APP_REGISTRY") def get_engine( - self, module_name: t.Optional[str] = None - ) -> t.Optional[Engine]: + self, module_name: Optional[str] = None + ) -> Optional[Engine]: piccolo_conf = self.get_piccolo_conf_module(module_name=module_name) - engine: t.Optional[Engine] = getattr(piccolo_conf, ENGINE_VAR, None) + engine: Optional[Engine] = getattr(piccolo_conf, ENGINE_VAR, None) if not engine: colored_warning( @@ -478,7 +477,7 @@ def get_engine( return engine - def get_app_modules(self) -> t.List[PiccoloAppModule]: + def get_app_modules(self) -> list[PiccoloAppModule]: """ Returns the ``piccolo_app.py`` modules for each registered Piccolo app in your project. @@ -493,7 +492,7 @@ def get_app_modules(self) -> t.List[PiccoloAppModule]: def get_app_names( self, sort_by_migration_dependencies: bool = True - ) -> t.List[str]: + ) -> list[str]: """ Return all of the app names. @@ -509,15 +508,15 @@ def get_app_names( ) ] - def get_sorted_app_names(self) -> t.List[str]: + def get_sorted_app_names(self) -> list[str]: """ Just here for backwards compatibility - use ``get_app_names`` directly. """ return self.get_app_names(sort_by_migration_dependencies=True) def sort_app_configs( - self, app_configs: t.List[AppConfig] - ) -> t.List[AppConfig]: + self, app_configs: list[AppConfig] + ) -> list[AppConfig]: app_config_map = { app_config.app_name: app_config for app_config in app_configs } @@ -536,7 +535,7 @@ def sort_app_configs( def get_app_configs( self, sort_by_migration_dependencies: bool = True - ) -> t.List[AppConfig]: + ) -> list[AppConfig]: """ Returns a list of ``AppConfig``, optionally sorted by migration dependencies. @@ -560,7 +559,7 @@ def get_app_config(self, app_name: str) -> AppConfig: def get_table_with_name( self, app_name: str, table_class_name: str - ) -> t.Type[Table]: + ) -> type[Table]: """ Returns a ``Table`` class registered with the given app if it exists. Otherwise it raises an ``ValueError``. @@ -572,9 +571,9 @@ def get_table_with_name( def get_table_classes( self, - include_apps: t.Optional[t.List[str]] = None, - exclude_apps: t.Optional[t.List[str]] = None, - ) -> t.List[t.Type[Table]]: + include_apps: Optional[list[str]] = None, + exclude_apps: Optional[list[str]] = None, + ) -> list[type[Table]]: """ Returns all ``Table`` classes registered with the given apps. If ``include_apps`` is ``None``, then ``Table`` classes will be returned @@ -590,7 +589,7 @@ def get_table_classes( if exclude_apps: app_names = [i for i in app_names if i not in exclude_apps] - tables: t.List[t.Type[Table]] = [] + tables: list[type[Table]] = [] for app_name in app_names: app_config = self.get_app_config(app_name=app_name) @@ -604,7 +603,7 @@ def get_table_classes( class PiccoloConfUpdater: - def __init__(self, piccolo_conf_path: t.Optional[str] = None): + def __init__(self, piccolo_conf_path: Optional[str] = None): """ :param piccolo_conf_path: The path to the piccolo_conf.py (e.g. `./piccolo_conf.py`). If not diff --git a/piccolo/custom_types.py b/piccolo/custom_types.py index 2101bc689..ea8ee07f6 100644 --- a/piccolo/custom_types.py +++ b/piccolo/custom_types.py @@ -1,18 +1,19 @@ from __future__ import annotations -import typing as t +from collections.abc import Iterable +from typing import TYPE_CHECKING, Any, TypeVar, Union -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from piccolo.columns.combination import And, Or, Where, WhereRaw # noqa from piccolo.table import Table -Combinable = t.Union["Where", "WhereRaw", "And", "Or"] -Iterable = t.Iterable[t.Any] +Combinable = Union["Where", "WhereRaw", "And", "Or"] +CustomIterable = Iterable[Any] -TableInstance = t.TypeVar("TableInstance", bound="Table") -QueryResponseType = t.TypeVar("QueryResponseType", bound=t.Any) +TableInstance = TypeVar("TableInstance", bound="Table") +QueryResponseType = TypeVar("QueryResponseType", bound=Any) ############################################################################### diff --git a/piccolo/engine/base.py b/piccolo/engine/base.py index 0181a29b5..b86af9dd2 100644 --- a/piccolo/engine/base.py +++ b/piccolo/engine/base.py @@ -4,8 +4,8 @@ import logging import pprint import string -import typing as t from abc import ABCMeta, abstractmethod +from typing import TYPE_CHECKING, Final, Generic, Optional, TypeVar, Union from typing_extensions import Self @@ -13,14 +13,14 @@ from piccolo.utils.sync import run_sync from piccolo.utils.warnings import Level, colored_string, colored_warning -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from piccolo.query.base import DDL, Query logger = logging.getLogger(__name__) # This is a set to speed up lookups from O(n) when # using str vs O(1) when using set[str] -VALID_SAVEPOINT_CHARACTERS: t.Final[set[str]] = set( +VALID_SAVEPOINT_CHARACTERS: Final[set[str]] = set( string.ascii_letters + string.digits + "-" + "_" ) @@ -45,12 +45,12 @@ async def __aexit__(self, *args, **kwargs): ... def __aiter__(self: Self) -> Self: ... @abstractmethod - async def __anext__(self) -> t.List[t.Dict]: ... + async def __anext__(self) -> list[dict]: ... class BaseTransaction(metaclass=ABCMeta): - __slots__: t.Tuple[str, ...] = tuple() + __slots__: tuple[str, ...] = tuple() @abstractmethod async def __aenter__(self, *args, **kwargs): ... @@ -61,10 +61,10 @@ async def __aexit__(self, *args, **kwargs) -> bool: ... class BaseAtomic(metaclass=ABCMeta): - __slots__: t.Tuple[str, ...] = tuple() + __slots__: tuple[str, ...] = tuple() @abstractmethod - def add(self, *query: t.Union[Query, DDL]): ... + def add(self, *query: Union[Query, DDL]): ... @abstractmethod async def run(self): ... @@ -76,10 +76,10 @@ def run_sync(self): ... def __await__(self): ... -TransactionClass = t.TypeVar("TransactionClass", bound=BaseTransaction) +TransactionClass = TypeVar("TransactionClass", bound=BaseTransaction) -class Engine(t.Generic[TransactionClass], metaclass=ABCMeta): +class Engine(Generic[TransactionClass], metaclass=ABCMeta): __slots__ = ( "query_id", "log_queries", @@ -92,7 +92,7 @@ class Engine(t.Generic[TransactionClass], metaclass=ABCMeta): def __init__( self, engine_type: str, - min_version_number: t.Union[int, float], + min_version_number: Union[int, float], log_queries: bool = False, log_responses: bool = False, ): @@ -122,7 +122,7 @@ async def batch( self, query: Query, batch_size: int = 100, - node: t.Optional[str] = None, + node: Optional[str] = None, ) -> BaseBatch: pass @@ -132,7 +132,7 @@ async def run_querystring( ): pass - def transform_response_to_dicts(self, results) -> t.List[t.Dict]: + def transform_response_to_dicts(self, results) -> list[dict]: """ If the database adapter returns something other than a list of dictionaries, it should perform the transformation here. @@ -196,7 +196,7 @@ async def close_connection_pool(self): ########################################################################### - current_transaction: contextvars.ContextVar[t.Optional[TransactionClass]] + current_transaction: contextvars.ContextVar[Optional[TransactionClass]] def transaction_exists(self) -> bool: """ @@ -222,7 +222,7 @@ def print_query(self, query_id: int, query: str): print(colored_string(f"\nQuery {query_id}:")) print(query) - def print_response(self, query_id: int, response: t.List): + def print_response(self, query_id: int, response: list): print( colored_string(f"\nQuery {query_id} response:", level=Level.high) ) diff --git a/piccolo/engine/cockroach.py b/piccolo/engine/cockroach.py index d091527bb..e823ced5a 100644 --- a/piccolo/engine/cockroach.py +++ b/piccolo/engine/cockroach.py @@ -1,6 +1,7 @@ from __future__ import annotations -import typing as t +from collections.abc import Sequence +from typing import Any, Optional from piccolo.utils.lazy_loader import LazyLoader from piccolo.utils.warnings import Level, colored_warning @@ -18,11 +19,11 @@ class CockroachEngine(PostgresEngine): def __init__( self, - config: t.Dict[str, t.Any], - extensions: t.Sequence[str] = (), + config: dict[str, Any], + extensions: Sequence[str] = (), log_queries: bool = False, log_responses: bool = False, - extra_nodes: t.Optional[t.Dict[str, CockroachEngine]] = None, + extra_nodes: Optional[dict[str, CockroachEngine]] = None, ) -> None: super().__init__( config=config, diff --git a/piccolo/engine/finder.py b/piccolo/engine/finder.py index e67accc93..a72db979a 100644 --- a/piccolo/engine/finder.py +++ b/piccolo/engine/finder.py @@ -1,11 +1,11 @@ from __future__ import annotations -import typing as t +from typing import Optional from piccolo.engine.base import Engine -def engine_finder(module_name: t.Optional[str] = None) -> t.Optional[Engine]: +def engine_finder(module_name: Optional[str] = None) -> Optional[Engine]: """ An example module name is `my_piccolo_conf`. diff --git a/piccolo/engine/postgres.py b/piccolo/engine/postgres.py index 117bf9a53..8d09d350b 100644 --- a/piccolo/engine/postgres.py +++ b/piccolo/engine/postgres.py @@ -1,8 +1,9 @@ from __future__ import annotations import contextvars -import typing as t +from collections.abc import Sequence from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Mapping, Optional, Union from typing_extensions import Self @@ -22,7 +23,7 @@ asyncpg = LazyLoader("asyncpg", globals(), "asyncpg") -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from asyncpg.connection import Connection from asyncpg.cursor import Cursor from asyncpg.pool import Pool @@ -36,8 +37,8 @@ class AsyncBatch(BaseBatch): batch_size: int # Set internally - _transaction: t.Optional[Transaction] = None - _cursor: t.Optional[Cursor] = None + _transaction: Optional[Transaction] = None + _cursor: Optional[Cursor] = None @property def cursor(self) -> Cursor: @@ -51,14 +52,14 @@ def transaction(self) -> Transaction: raise ValueError("The transaction can't be found.") return self._transaction - async def next(self) -> t.List[t.Dict]: + async def next(self) -> list[dict]: data = await self.cursor.fetch(self.batch_size) return await self.query._process_results(data) def __aiter__(self: Self) -> Self: return self - async def __anext__(self) -> t.List[t.Dict]: + async def __anext__(self) -> list[dict]: response = await self.next() if response == []: raise StopAsyncIteration() @@ -107,9 +108,9 @@ class Atomic(BaseAtomic): def __init__(self, engine: PostgresEngine): self.engine = engine - self.queries: t.List[t.Union[Query, DDL]] = [] + self.queries: list[Union[Query, DDL]] = [] - def add(self, *query: t.Union[Query, DDL]): + def add(self, *query: Union[Query, DDL]): self.queries += list(query) async def run(self): @@ -250,7 +251,7 @@ def get_savepoint_id(self) -> int: self._savepoint_id += 1 return self._savepoint_id - async def savepoint(self, name: t.Optional[str] = None) -> Savepoint: + async def savepoint(self, name: Optional[str] = None) -> Savepoint: name = name or f"savepoint_{self.get_savepoint_id()}" validate_savepoint_name(name) await self.connection.execute(f"SAVEPOINT {name}") @@ -352,11 +353,11 @@ class PostgresEngine(Engine[PostgresTransaction]): def __init__( self, - config: t.Dict[str, t.Any], - extensions: t.Sequence[str] = ("uuid-ossp",), + config: dict[str, Any], + extensions: Sequence[str] = ("uuid-ossp",), log_queries: bool = False, log_responses: bool = False, - extra_nodes: t.Optional[t.Mapping[str, PostgresEngine]] = None, + extra_nodes: Optional[Mapping[str, PostgresEngine]] = None, ) -> None: if extra_nodes is None: extra_nodes = {} @@ -366,7 +367,7 @@ def __init__( self.log_queries = log_queries self.log_responses = log_responses self.extra_nodes = extra_nodes - self.pool: t.Optional[Pool] = None + self.pool: Optional[Pool] = None database_name = config.get("database", "Unknown") self.current_transaction = contextvars.ContextVar( f"pg_current_transaction_{database_name}", default=None @@ -396,7 +397,7 @@ async def get_version(self) -> float: Returns the version of Postgres being run. """ try: - response: t.Sequence[t.Dict] = await self._run_in_new_connection( + response: Sequence[dict] = await self._run_in_new_connection( "SHOW server_version" ) except ConnectionRefusedError as exception: @@ -483,7 +484,7 @@ async def batch( self, query: Query, batch_size: int = 100, - node: t.Optional[str] = None, + node: Optional[str] = None, ) -> AsyncBatch: """ :param query: @@ -494,7 +495,7 @@ async def batch( Which node to run the query on (see ``extra_nodes``). If not specified, it runs on the main Postgres node. """ - engine: t.Any = self.extra_nodes.get(node) if node else self + engine: Any = self.extra_nodes.get(node) if node else self connection = await engine.get_new_connection() return AsyncBatch( connection=connection, query=query, batch_size=batch_size @@ -503,7 +504,7 @@ async def batch( ########################################################################### async def _run_in_pool( - self, query: str, args: t.Optional[t.Sequence[t.Any]] = None + self, query: str, args: Optional[Sequence[Any]] = None ): if args is None: args = [] @@ -516,7 +517,7 @@ async def _run_in_pool( return response async def _run_in_new_connection( - self, query: str, args: t.Optional[t.Sequence[t.Any]] = None + self, query: str, args: Optional[Sequence[Any]] = None ): if args is None: args = [] @@ -579,7 +580,7 @@ async def run_ddl(self, ddl: str, in_pool: bool = True): return response - def transform_response_to_dicts(self, results) -> t.List[t.Dict]: + def transform_response_to_dicts(self, results) -> list[dict]: """ asyncpg returns a special Record object, so we need to convert it to a dict. diff --git a/piccolo/engine/sqlite.py b/piccolo/engine/sqlite.py index c8183e336..86de912f3 100644 --- a/piccolo/engine/sqlite.py +++ b/piccolo/engine/sqlite.py @@ -5,11 +5,12 @@ import enum import os import sqlite3 -import typing as t import uuid +from collections.abc import Callable from dataclasses import dataclass from decimal import Decimal from functools import partial, wraps +from typing import TYPE_CHECKING, Any, Optional, Union from typing_extensions import Self @@ -30,7 +31,7 @@ aiosqlite = LazyLoader("aiosqlite", globals(), "aiosqlite") -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from aiosqlite import Connection, Cursor # type: ignore from piccolo.table import Table @@ -125,7 +126,7 @@ def serialise(data: list): # Register adapters -ADAPTERS: t.Dict[t.Type, t.Callable[[t.Any], t.Any]] = { +ADAPTERS: dict[type, Callable[[Any], Any]] = { Decimal: convert_numeric_in, uuid.UUID: convert_uuid_in, datetime.time: convert_time_in, @@ -143,7 +144,7 @@ def serialise(data: list): # Out -def decode_to_string(converter: t.Callable[[str], t.Any]): +def decode_to_string(converter: Callable[[str], Any]): """ This means we can use our converters with string and bytes. They are passed bytes when used directly via SQLite, and are passed strings when @@ -151,7 +152,7 @@ def decode_to_string(converter: t.Callable[[str], t.Any]): """ @wraps(converter) - def wrapper(value: t.Union[str, bytes]) -> t.Any: + def wrapper(value: Union[str, bytes]) -> Any: if isinstance(value, bytes): return converter(value.decode("utf8")) elif isinstance(value, str): @@ -247,7 +248,7 @@ def convert_timestamptz_out(value: str) -> datetime.datetime: @decode_to_string -def convert_array_out(value: str) -> t.List: +def convert_array_out(value: str) -> list: """ If the value if from an array column, deserialise the string back into a list. @@ -255,7 +256,7 @@ def convert_array_out(value: str) -> t.List: return load_json(value) -def convert_complex_array_out(value: bytes, converter: t.Callable): +def convert_complex_array_out(value: bytes, converter: Callable): """ This is used to handle arrays of things like timestamps, which we can't just load from JSON without doing additional work to convert the elements @@ -263,7 +264,7 @@ def convert_complex_array_out(value: bytes, converter: t.Callable): """ parsed = load_json(value.decode("utf8")) - def convert_list(list_value: t.List): + def convert_list(list_value: list): output = [] for value in list_value: @@ -284,7 +285,7 @@ def convert_list(list_value: t.List): @decode_to_string -def convert_M2M_out(value: str) -> t.List: +def convert_M2M_out(value: str) -> list: return value.split(",") @@ -335,7 +336,7 @@ class AsyncBatch(BaseBatch): batch_size: int # Set internally - _cursor: t.Optional[Cursor] = None + _cursor: Optional[Cursor] = None @property def cursor(self) -> Cursor: @@ -343,14 +344,14 @@ def cursor(self) -> Cursor: raise ValueError("_cursor not set") return self._cursor - async def next(self) -> t.List[t.Dict]: + async def next(self) -> list[dict]: data = await self.cursor.fetchmany(self.batch_size) return await self.query._process_results(data) def __aiter__(self: Self) -> Self: return self - async def __anext__(self) -> t.List[t.Dict]: + async def __anext__(self) -> list[dict]: response = await self.next() if response == []: raise StopAsyncIteration() @@ -404,9 +405,9 @@ def __init__( ): self.engine = engine self.transaction_type = transaction_type - self.queries: t.List[t.Union[Query, DDL]] = [] + self.queries: list[Union[Query, DDL]] = [] - def add(self, *query: t.Union[Query, DDL]): + def add(self, *query: Union[Query, DDL]): self.queries += list(query) async def run(self): @@ -546,7 +547,7 @@ def get_savepoint_id(self) -> int: self._savepoint_id += 1 return self._savepoint_id - async def savepoint(self, name: t.Optional[str] = None) -> Savepoint: + async def savepoint(self, name: Optional[str] = None) -> Savepoint: name = name or f"savepoint_{self.get_savepoint_id()}" validate_savepoint_name(name) await self.connection.execute(f"SAVEPOINT {name}") @@ -576,7 +577,7 @@ async def __aexit__(self, exception_type, exception, traceback) -> bool: ############################################################################### -def dict_factory(cursor, row) -> t.Dict: +def dict_factory(cursor, row) -> dict: return {col[0]: row[idx] for idx, col in enumerate(cursor.description)} @@ -672,7 +673,7 @@ def create_db_file(self): ########################################################################### async def batch( - self, query: Query, batch_size: int = 100, node: t.Optional[str] = None + self, query: Query, batch_size: int = 100, node: Optional[str] = None ) -> AsyncBatch: """ :param query: @@ -698,7 +699,7 @@ async def get_connection(self) -> Connection: ########################################################################### - async def _get_inserted_pk(self, cursor, table: t.Type[Table]) -> t.Any: + async def _get_inserted_pk(self, cursor, table: type[Table]) -> Any: """ If the `pk` column is a non-integer then `ROWID` and `pk` will return different types. Need to query by `lastrowid` to get `pk`s in SQLite @@ -714,9 +715,9 @@ async def _get_inserted_pk(self, cursor, table: t.Type[Table]) -> t.Any: async def _run_in_new_connection( self, query: str, - args: t.Optional[t.List[t.Any]] = None, + args: Optional[list[Any]] = None, query_type: str = "generic", - table: t.Optional[t.Type[Table]] = None, + table: Optional[type[Table]] = None, ): if args is None: args = [] @@ -740,9 +741,9 @@ async def _run_in_existing_connection( self, connection, query: str, - args: t.Optional[t.List[t.Any]] = None, + args: Optional[list[Any]] = None, query_type: str = "generic", - table: t.Optional[t.Type[Table]] = None, + table: Optional[type[Table]] = None, ): """ This is used when a transaction is currently active. diff --git a/piccolo/query/base.py b/piccolo/query/base.py index 45049e1e1..d45d885dc 100644 --- a/piccolo/query/base.py +++ b/piccolo/query/base.py @@ -1,7 +1,8 @@ from __future__ import annotations -import typing as t +from collections.abc import Generator, Sequence from time import time +from typing import TYPE_CHECKING, Any, Generic, Optional, Union, cast from piccolo.columns.column_types import JSON, JSONB from piccolo.custom_types import QueryResponseType, TableInstance @@ -12,7 +13,7 @@ from piccolo.utils.objects import make_nested_object from piccolo.utils.sync import run_sync -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from piccolo.query.mixins import OutputDelegate from piccolo.table import Table # noqa @@ -26,13 +27,13 @@ def __exit__(self, exception_type, exception, traceback): print(f"Duration: {self.end - self.start}s") -class Query(t.Generic[TableInstance, QueryResponseType]): +class Query(Generic[TableInstance, QueryResponseType]): __slots__ = ("table", "_frozen_querystrings") def __init__( self, - table: t.Type[TableInstance], - frozen_querystrings: t.Optional[t.Sequence[QueryString]] = None, + table: type[TableInstance], + frozen_querystrings: Optional[Sequence[QueryString]] = None, ): self.table = table self._frozen_querystrings = frozen_querystrings @@ -55,21 +56,21 @@ async def _process_results(self, results) -> QueryResponseType: if hasattr(self, "_raw_response_callback"): self._raw_response_callback(raw) - output: t.Optional[OutputDelegate] = getattr( + output: Optional[OutputDelegate] = getattr( self, "output_delegate", None ) ####################################################################### if output and output._output.load_json: - columns_delegate: t.Optional[ColumnsDelegate] = getattr( + columns_delegate: Optional[ColumnsDelegate] = getattr( self, "columns_delegate", None ) - json_column_names: t.List[str] = [] + json_column_names: list[str] = [] if columns_delegate is not None: - json_columns: t.List[t.Union[JSON, JSONB]] = [] + json_columns: list[Union[JSON, JSONB]] = [] for column in columns_delegate.selected_columns: if isinstance(column, (JSON, JSONB)): @@ -107,12 +108,12 @@ async def _process_results(self, results) -> QueryResponseType: if output: if output._output.as_objects: if output._output.nested: - return t.cast( + return cast( QueryResponseType, [make_nested_object(row, self.table) for row in raw], ) else: - return t.cast( + return cast( QueryResponseType, [ self.table(**columns, _exists_in_db=True) @@ -120,7 +121,7 @@ async def _process_results(self, results) -> QueryResponseType: ], ) - return t.cast(QueryResponseType, raw) + return cast(QueryResponseType, raw) def _validate(self): """ @@ -130,7 +131,7 @@ def _validate(self): """ pass - def __await__(self) -> t.Generator[None, None, QueryResponseType]: + def __await__(self) -> Generator[None, None, QueryResponseType]: """ If the user doesn't explicity call .run(), proxy to it as a convenience. @@ -138,7 +139,7 @@ def __await__(self) -> t.Generator[None, None, QueryResponseType]: return self.run().__await__() async def _run( - self, node: t.Optional[str] = None, in_pool: bool = True + self, node: Optional[str] = None, in_pool: bool = True ) -> QueryResponseType: """ Run the query on the database. @@ -184,16 +185,16 @@ async def _run( processed_results = await self._process_results(results) responses.append(processed_results) - return t.cast(QueryResponseType, responses) + return cast(QueryResponseType, responses) async def run( - self, node: t.Optional[str] = None, in_pool: bool = True + self, node: Optional[str] = None, in_pool: bool = True ) -> QueryResponseType: return await self._run(node=node, in_pool=in_pool) def run_sync( self, - node: t.Optional[str] = None, + node: Optional[str] = None, timed: bool = False, in_pool: bool = False, ) -> QueryResponseType: @@ -217,7 +218,7 @@ def run_sync( with Timer(): return run_sync(coroutine) - async def response_handler(self, response: t.List) -> t.Any: + async def response_handler(self, response: list) -> Any: """ Subclasses can override this to modify the raw response returned by the database driver. @@ -227,23 +228,23 @@ async def response_handler(self, response: t.List) -> t.Any: ########################################################################### @property - def sqlite_querystrings(self) -> t.Sequence[QueryString]: + def sqlite_querystrings(self) -> Sequence[QueryString]: raise NotImplementedError @property - def postgres_querystrings(self) -> t.Sequence[QueryString]: + def postgres_querystrings(self) -> Sequence[QueryString]: raise NotImplementedError @property - def cockroach_querystrings(self) -> t.Sequence[QueryString]: + def cockroach_querystrings(self) -> Sequence[QueryString]: raise NotImplementedError @property - def default_querystrings(self) -> t.Sequence[QueryString]: + def default_querystrings(self) -> Sequence[QueryString]: raise NotImplementedError @property - def querystrings(self) -> t.Sequence[QueryString]: + def querystrings(self) -> Sequence[QueryString]: """ Calls the correct underlying method, depending on the current engine. """ @@ -367,7 +368,7 @@ def __str__(self) -> str: class DDL: __slots__ = ("table",) - def __init__(self, table: t.Type[Table], **kwargs): + def __init__(self, table: type[Table], **kwargs): self.table = table @property @@ -379,23 +380,23 @@ def engine_type(self) -> str: raise ValueError("Engine isn't defined.") @property - def sqlite_ddl(self) -> t.Sequence[str]: + def sqlite_ddl(self) -> Sequence[str]: raise NotImplementedError @property - def postgres_ddl(self) -> t.Sequence[str]: + def postgres_ddl(self) -> Sequence[str]: raise NotImplementedError @property - def cockroach_ddl(self) -> t.Sequence[str]: + def cockroach_ddl(self) -> Sequence[str]: raise NotImplementedError @property - def default_ddl(self) -> t.Sequence[str]: + def default_ddl(self) -> Sequence[str]: raise NotImplementedError @property - def ddl(self) -> t.Sequence[str]: + def ddl(self) -> Sequence[str]: """ Calls the correct underlying method, depending on the current engine. """ diff --git a/piccolo/query/functions/aggregate.py b/piccolo/query/functions/aggregate.py index c50e557db..499d56007 100644 --- a/piccolo/query/functions/aggregate.py +++ b/piccolo/query/functions/aggregate.py @@ -1,4 +1,5 @@ -import typing as t +from collections.abc import Sequence +from typing import Optional from piccolo.columns.base import Column from piccolo.querystring import QueryString @@ -51,8 +52,8 @@ class Count(QueryString): def __init__( self, - column: t.Optional[Column] = None, - distinct: t.Optional[t.Sequence[Column]] = None, + column: Optional[Column] = None, + distinct: Optional[Sequence[Column]] = None, alias: str = "count", ): """ diff --git a/piccolo/query/functions/base.py b/piccolo/query/functions/base.py index c4181aca6..807de8365 100644 --- a/piccolo/query/functions/base.py +++ b/piccolo/query/functions/base.py @@ -1,4 +1,4 @@ -import typing as t +from typing import Optional, Union from piccolo.columns.base import Column from piccolo.querystring import QueryString @@ -9,8 +9,8 @@ class Function(QueryString): def __init__( self, - identifier: t.Union[Column, QueryString, str], - alias: t.Optional[str] = None, + identifier: Union[Column, QueryString, str], + alias: Optional[str] = None, ): alias = alias or self.__class__.__name__.lower() diff --git a/piccolo/query/functions/datetime.py b/piccolo/query/functions/datetime.py index 678355164..146fc5c42 100644 --- a/piccolo/query/functions/datetime.py +++ b/piccolo/query/functions/datetime.py @@ -1,4 +1,4 @@ -import typing as t +from typing import Literal, Optional, Union, get_args from piccolo.columns.base import Column from piccolo.columns.column_types import ( @@ -15,7 +15,7 @@ ############################################################################### # Postgres / Cockroach -ExtractComponent = t.Literal[ +ExtractComponent = Literal[ "century", "day", "decade", @@ -44,9 +44,9 @@ class Extract(QueryString): def __init__( self, - identifier: t.Union[Date, Time, Timestamp, Timestamptz, QueryString], + identifier: Union[Date, Time, Timestamp, Timestamptz, QueryString], datetime_component: ExtractComponent, - alias: t.Optional[str] = None, + alias: Optional[str] = None, ): """ .. note:: This is for Postgres / Cockroach only. @@ -69,7 +69,7 @@ def __init__( The date or time component to extract from the column. """ - if datetime_component.lower() not in t.get_args(ExtractComponent): + if datetime_component.lower() not in get_args(ExtractComponent): raise ValueError("The date time component isn't recognised.") super().__init__( @@ -86,9 +86,9 @@ def __init__( class Strftime(QueryString): def __init__( self, - identifier: t.Union[Date, Time, Timestamp, Timestamptz, QueryString], + identifier: Union[Date, Time, Timestamp, Timestamptz, QueryString], datetime_format: str, - alias: t.Optional[str] = None, + alias: Optional[str] = None, ): """ .. note:: This is for SQLite only. @@ -122,7 +122,7 @@ def __init__( # Database agnostic -def _get_engine_type(identifier: t.Union[Column, QueryString]) -> str: +def _get_engine_type(identifier: Union[Column, QueryString]) -> str: if isinstance(identifier, Column): return identifier._meta.engine_type elif isinstance(identifier, QueryString) and ( @@ -134,10 +134,10 @@ def _get_engine_type(identifier: t.Union[Column, QueryString]) -> str: def _extract_component( - identifier: t.Union[Date, Time, Timestamp, Timestamptz, QueryString], + identifier: Union[Date, Time, Timestamp, Timestamptz, QueryString], sqlite_format: str, postgres_format: ExtractComponent, - alias: t.Optional[str], + alias: Optional[str], ) -> QueryString: engine_type = _get_engine_type(identifier=identifier) @@ -159,8 +159,8 @@ def _extract_component( def Year( - identifier: t.Union[Date, Timestamp, Timestamptz, QueryString], - alias: t.Optional[str] = None, + identifier: Union[Date, Timestamp, Timestamptz, QueryString], + alias: Optional[str] = None, ) -> QueryString: """ Extract the year as an integer. @@ -174,8 +174,8 @@ def Year( def Month( - identifier: t.Union[Date, Timestamp, Timestamptz, QueryString], - alias: t.Optional[str] = None, + identifier: Union[Date, Timestamp, Timestamptz, QueryString], + alias: Optional[str] = None, ) -> QueryString: """ Extract the month as an integer. @@ -189,8 +189,8 @@ def Month( def Day( - identifier: t.Union[Date, Timestamp, Timestamptz, QueryString], - alias: t.Optional[str] = None, + identifier: Union[Date, Timestamp, Timestamptz, QueryString], + alias: Optional[str] = None, ) -> QueryString: """ Extract the day as an integer. @@ -204,8 +204,8 @@ def Day( def Hour( - identifier: t.Union[Time, Timestamp, Timestamptz, QueryString], - alias: t.Optional[str] = None, + identifier: Union[Time, Timestamp, Timestamptz, QueryString], + alias: Optional[str] = None, ) -> QueryString: """ Extract the hour as an integer. @@ -219,8 +219,8 @@ def Hour( def Minute( - identifier: t.Union[Time, Timestamp, Timestamptz, QueryString], - alias: t.Optional[str] = None, + identifier: Union[Time, Timestamp, Timestamptz, QueryString], + alias: Optional[str] = None, ) -> QueryString: """ Extract the minute as an integer. @@ -234,8 +234,8 @@ def Minute( def Second( - identifier: t.Union[Time, Timestamp, Timestamptz, QueryString], - alias: t.Optional[str] = None, + identifier: Union[Time, Timestamp, Timestamptz, QueryString], + alias: Optional[str] = None, ) -> QueryString: """ Extract the second as an integer. diff --git a/piccolo/query/functions/string.py b/piccolo/query/functions/string.py index 68b78219f..3aa4a5d45 100644 --- a/piccolo/query/functions/string.py +++ b/piccolo/query/functions/string.py @@ -5,7 +5,7 @@ """ -import typing as t +from typing import Optional, Union from piccolo.columns.base import Column from piccolo.columns.column_types import Text, Varchar @@ -72,8 +72,8 @@ class Upper(Function): class Concat(QueryString): def __init__( self, - *args: t.Union[Column, QueryString, str], - alias: t.Optional[str] = None, + *args: Union[Column, QueryString, str], + alias: Optional[str] = None, ): """ Concatenate multiple values into a single string. @@ -91,7 +91,7 @@ def __init__( placeholders = ", ".join("{}" for _ in args) - processed_args: t.List[t.Union[QueryString, Column]] = [] + processed_args: list[Union[QueryString, Column]] = [] for arg in args: if isinstance(arg, str) or ( diff --git a/piccolo/query/functions/type_conversion.py b/piccolo/query/functions/type_conversion.py index e8f1dfee5..cc4bd8b39 100644 --- a/piccolo/query/functions/type_conversion.py +++ b/piccolo/query/functions/type_conversion.py @@ -1,4 +1,4 @@ -import typing as t +from typing import Optional, Union from piccolo.columns.base import Column from piccolo.querystring import QueryString @@ -7,9 +7,9 @@ class Cast(QueryString): def __init__( self, - identifier: t.Union[Column, QueryString], + identifier: Union[Column, QueryString], as_type: Column, - alias: t.Optional[str] = None, + alias: Optional[str] = None, ): """ Cast a value to a different type. For example:: @@ -46,7 +46,7 @@ def __init__( # on which database is being used. from piccolo.table import Table, create_table_class - table: t.Optional[t.Type[Table]] = None + table: Optional[type[Table]] = None if isinstance(identifier, Column): table = identifier._meta.table diff --git a/piccolo/query/methods/alter.py b/piccolo/query/methods/alter.py index b0adba2c4..35774acd3 100644 --- a/piccolo/query/methods/alter.py +++ b/piccolo/query/methods/alter.py @@ -1,15 +1,16 @@ from __future__ import annotations import itertools -import typing as t +from collections.abc import Sequence from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union from piccolo.columns.base import Column from piccolo.columns.column_types import ForeignKey, Numeric, Varchar from piccolo.query.base import DDL from piccolo.utils.warnings import Level, colored_warning -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from piccolo.columns.base import OnDelete, OnUpdate from piccolo.table import Table @@ -52,7 +53,7 @@ def ddl(self) -> str: class AlterColumnStatement(AlterStatement): __slots__ = ("column",) - column: t.Union[Column, str] + column: Union[Column, str] @property def column_name(self) -> str: @@ -114,7 +115,7 @@ class SetColumnType(AlterStatement): old_column: Column new_column: Column - using_expression: t.Optional[str] = None + using_expression: Optional[str] = None @property def ddl(self) -> str: @@ -135,7 +136,7 @@ class SetDefault(AlterColumnStatement): __slots__ = ("value",) column: Column - value: t.Any + value: Any @property def ddl(self) -> str: @@ -215,8 +216,8 @@ class AddForeignKeyConstraint(AlterStatement): foreign_key_column_name: str referenced_table_name: str referenced_column_name: str - on_delete: t.Optional[OnDelete] - on_update: t.Optional[OnUpdate] + on_delete: Optional[OnDelete] + on_update: Optional[OnUpdate] @property def ddl(self) -> str: @@ -236,7 +237,7 @@ def ddl(self) -> str: class SetDigits(AlterColumnStatement): __slots__ = ("digits", "column_type") - digits: t.Optional[t.Tuple[int, int]] + digits: Optional[tuple[int, int]] column_type: str @property @@ -265,7 +266,7 @@ def ddl(self) -> str: @dataclass class DropTable: - table: t.Type[Table] + table: type[Table] cascade: bool if_exists: bool @@ -304,24 +305,24 @@ class Alter(DDL): "_rename_constraint", ) - def __init__(self, table: t.Type[Table], **kwargs): + def __init__(self, table: type[Table], **kwargs): super().__init__(table, **kwargs) - self._add_foreign_key_constraint: t.List[AddForeignKeyConstraint] = [] - self._add: t.List[AddColumn] = [] - self._drop_constraint: t.List[DropConstraint] = [] - self._drop_default: t.List[DropDefault] = [] - self._drop_table: t.Optional[DropTable] = None - self._drop: t.List[DropColumn] = [] - self._rename_columns: t.List[RenameColumn] = [] - self._rename_table: t.List[RenameTable] = [] - self._set_column_type: t.List[SetColumnType] = [] - self._set_default: t.List[SetDefault] = [] - self._set_digits: t.List[SetDigits] = [] - self._set_length: t.List[SetLength] = [] - self._set_null: t.List[SetNull] = [] - self._set_schema: t.List[SetSchema] = [] - self._set_unique: t.List[SetUnique] = [] - self._rename_constraint: t.List[RenameConstraint] = [] + self._add_foreign_key_constraint: list[AddForeignKeyConstraint] = [] + self._add: list[AddColumn] = [] + self._drop_constraint: list[DropConstraint] = [] + self._drop_default: list[DropDefault] = [] + self._drop_table: Optional[DropTable] = None + self._drop: list[DropColumn] = [] + self._rename_columns: list[RenameColumn] = [] + self._rename_table: list[RenameTable] = [] + self._set_column_type: list[SetColumnType] = [] + self._set_default: list[SetDefault] = [] + self._set_digits: list[SetDigits] = [] + self._set_length: list[SetLength] = [] + self._set_null: list[SetNull] = [] + self._set_schema: list[SetSchema] = [] + self._set_unique: list[SetUnique] = [] + self._rename_constraint: list[RenameConstraint] = [] def add_column(self: Self, name: str, column: Column) -> Self: """ @@ -340,7 +341,7 @@ def add_column(self: Self, name: str, column: Column) -> Self: self._add.append(AddColumn(column, name)) return self - def drop_column(self, column: t.Union[str, Column]) -> Alter: + def drop_column(self, column: Union[str, Column]) -> Alter: """ Drop a column from the table:: @@ -350,7 +351,7 @@ def drop_column(self, column: t.Union[str, Column]) -> Alter: self._drop.append(DropColumn(column)) return self - def drop_default(self, column: t.Union[str, Column]) -> Alter: + def drop_default(self, column: Union[str, Column]) -> Alter: """ Drop the default from a column:: @@ -406,7 +407,7 @@ def rename_constraint(self, old_name: str, new_name: str) -> Alter: return self def rename_column( - self, column: t.Union[str, Column], new_name: str + self, column: Union[str, Column], new_name: str ) -> Alter: """ Rename a column on the table:: @@ -425,7 +426,7 @@ def set_column_type( self, old_column: Column, new_column: Column, - using_expression: t.Optional[str] = None, + using_expression: Optional[str] = None, ) -> Alter: """ Change the type of a column:: @@ -448,7 +449,7 @@ def set_column_type( ) return self - def set_default(self, column: Column, value: t.Any) -> Alter: + def set_default(self, column: Column, value: Any) -> Alter: """ Set the default for a column:: @@ -459,7 +460,7 @@ def set_default(self, column: Column, value: t.Any) -> Alter: return self def set_null( - self, column: t.Union[str, Column], boolean: bool = True + self, column: Union[str, Column], boolean: bool = True ) -> Alter: """ Change a column to be nullable or not:: @@ -475,7 +476,7 @@ def set_null( return self def set_unique( - self, column: t.Union[str, Column], boolean: bool = True + self, column: Union[str, Column], boolean: bool = True ) -> Alter: """ Make a column unique or not:: @@ -490,7 +491,7 @@ def set_unique( self._set_unique.append(SetUnique(column, boolean)) return self - def set_length(self, column: t.Union[str, Varchar], length: int) -> Alter: + def set_length(self, column: Union[str, Varchar], length: int) -> Alter: """ Change the max length of a varchar column. Unfortunately, this isn't supported by SQLite, but SQLite also doesn't enforce any length limits @@ -518,7 +519,7 @@ def set_length(self, column: t.Union[str, Varchar], length: int) -> Alter: self._set_length.append(SetLength(column, length)) return self - def _get_constraint_name(self, column: t.Union[str, ForeignKey]) -> str: + def _get_constraint_name(self, column: Union[str, ForeignKey]) -> str: column_name = AlterColumnStatement(column=column).column_name tablename = self.table._meta.tablename return f"{tablename}_{column_name}_fkey" @@ -530,7 +531,7 @@ def drop_constraint(self, constraint_name: str) -> Alter: return self def drop_foreign_key_constraint( - self, column: t.Union[str, ForeignKey] + self, column: Union[str, ForeignKey] ) -> Alter: constraint_name = self._get_constraint_name(column=column) self._drop_constraint.append( @@ -540,12 +541,12 @@ def drop_foreign_key_constraint( def add_foreign_key_constraint( self, - column: t.Union[str, ForeignKey], - referenced_table_name: t.Optional[str] = None, - referenced_column_name: t.Optional[str] = None, - constraint_name: t.Optional[str] = None, - on_delete: t.Optional[OnDelete] = None, - on_update: t.Optional[OnUpdate] = None, + column: Union[str, ForeignKey], + referenced_table_name: Optional[str] = None, + referenced_column_name: Optional[str] = None, + constraint_name: Optional[str] = None, + on_delete: Optional[OnDelete] = None, + on_update: Optional[OnUpdate] = None, ) -> Alter: """ Add a new foreign key constraint:: @@ -591,8 +592,8 @@ def add_foreign_key_constraint( def set_digits( self, - column: t.Union[str, Numeric], - digits: t.Optional[t.Tuple[int, int]], + column: Union[str, Numeric], + digits: Optional[tuple[int, int]], ) -> Alter: """ Alter the precision and scale for a ``Numeric`` column. @@ -623,7 +624,7 @@ def set_schema(self, schema_name: str) -> Alter: return self @property - def default_ddl(self) -> t.Sequence[str]: + def default_ddl(self) -> Sequence[str]: if self._drop_table is not None: return [self._drop_table.ddl] @@ -660,4 +661,4 @@ def default_ddl(self) -> t.Sequence[str]: return [query] -Self = t.TypeVar("Self", bound=Alter) +Self = TypeVar("Self", bound=Alter) diff --git a/piccolo/query/methods/count.py b/piccolo/query/methods/count.py index 99d46c39b..3c61ae9b6 100644 --- a/piccolo/query/methods/count.py +++ b/piccolo/query/methods/count.py @@ -1,6 +1,7 @@ from __future__ import annotations -import typing as t +from collections.abc import Sequence +from typing import TYPE_CHECKING, Optional, TypeVar, Union from piccolo.custom_types import Combinable from piccolo.query.base import Query @@ -8,7 +9,7 @@ from piccolo.query.mixins import WhereDelegate from piccolo.querystring import QueryString -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from piccolo.columns import Column from piccolo.table import Table @@ -19,9 +20,9 @@ class Count(Query): def __init__( self, - table: t.Type[Table], - column: t.Optional[Column] = None, - distinct: t.Optional[t.Sequence[Column]] = None, + table: type[Table], + column: Optional[Column] = None, + distinct: Optional[Sequence[Column]] = None, **kwargs, ): super().__init__(table, **kwargs) @@ -32,11 +33,11 @@ def __init__( ########################################################################### # Clauses - def where(self: Self, *where: t.Union[Combinable, QueryString]) -> Self: + def where(self: Self, *where: Union[Combinable, QueryString]) -> Self: self.where_delegate.where(*where) return self - def distinct(self: Self, columns: t.Optional[t.Sequence[Column]]) -> Self: + def distinct(self: Self, columns: Optional[Sequence[Column]]) -> Self: self._distinct = columns return self @@ -46,8 +47,8 @@ async def response_handler(self, response) -> bool: return response[0]["count"] @property - def default_querystrings(self) -> t.Sequence[QueryString]: - table: t.Type[Table] = self.table + def default_querystrings(self) -> Sequence[QueryString]: + table: type[Table] = self.table query = table.select( CountFunction(column=self.column, distinct=self._distinct) @@ -58,4 +59,4 @@ def default_querystrings(self) -> t.Sequence[QueryString]: return query.querystrings -Self = t.TypeVar("Self", bound=Count) +Self = TypeVar("Self", bound=Count) diff --git a/piccolo/query/methods/create.py b/piccolo/query/methods/create.py index 68cccf6b2..592cc9f05 100644 --- a/piccolo/query/methods/create.py +++ b/piccolo/query/methods/create.py @@ -1,11 +1,12 @@ from __future__ import annotations -import typing as t +from collections.abc import Sequence +from typing import TYPE_CHECKING from piccolo.query.base import DDL from piccolo.query.methods.create_index import CreateIndex -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from piccolo.table import Table @@ -18,7 +19,7 @@ class Create(DDL): def __init__( self, - table: t.Type[Table], + table: type[Table], if_not_exists: bool = False, only_default_columns: bool = False, auto_create_schema: bool = True, @@ -43,8 +44,8 @@ def __init__( self.auto_create_schema = auto_create_schema @property - def default_ddl(self) -> t.Sequence[str]: - ddl: t.List[str] = [] + def default_ddl(self) -> Sequence[str]: + ddl: list[str] = [] schema_name = self.table._meta.schema if ( diff --git a/piccolo/query/methods/create_index.py b/piccolo/query/methods/create_index.py index c81d38b9d..64ae4b4d8 100644 --- a/piccolo/query/methods/create_index.py +++ b/piccolo/query/methods/create_index.py @@ -1,20 +1,21 @@ from __future__ import annotations -import typing as t +from collections.abc import Sequence +from typing import TYPE_CHECKING, Union from piccolo.columns import Column from piccolo.columns.indexes import IndexMethod from piccolo.query.base import DDL -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from piccolo.table import Table class CreateIndex(DDL): def __init__( self, - table: t.Type[Table], - columns: t.Union[t.List[Column], t.List[str]], + table: type[Table], + columns: Union[list[Column], list[str]], method: IndexMethod = IndexMethod.btree, if_not_exists: bool = False, **kwargs, @@ -25,7 +26,7 @@ def __init__( super().__init__(table, **kwargs) @property - def column_names(self) -> t.List[str]: + def column_names(self) -> list[str]: return [ i._meta.db_column_name if isinstance(i, Column) else i for i in self.columns @@ -39,7 +40,7 @@ def prefix(self) -> str: return prefix @property - def postgres_ddl(self) -> t.Sequence[str]: + def postgres_ddl(self) -> Sequence[str]: column_names = self.column_names index_name = self.table._get_index_name(column_names) tablename = self.table._meta.get_formatted_tablename() @@ -53,11 +54,11 @@ def postgres_ddl(self) -> t.Sequence[str]: ] @property - def cockroach_ddl(self) -> t.Sequence[str]: + def cockroach_ddl(self) -> Sequence[str]: return self.postgres_ddl @property - def sqlite_ddl(self) -> t.Sequence[str]: + def sqlite_ddl(self) -> Sequence[str]: column_names = self.column_names index_name = self.table._get_index_name(column_names) tablename = self.table._meta.get_formatted_tablename() diff --git a/piccolo/query/methods/delete.py b/piccolo/query/methods/delete.py index 5570ddde9..ea7d56a7e 100644 --- a/piccolo/query/methods/delete.py +++ b/piccolo/query/methods/delete.py @@ -1,13 +1,14 @@ from __future__ import annotations -import typing as t +from collections.abc import Sequence +from typing import TYPE_CHECKING, TypeVar, Union from piccolo.custom_types import Combinable from piccolo.query.base import Query from piccolo.query.mixins import ReturningDelegate, WhereDelegate from piccolo.querystring import QueryString -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from piccolo.columns import Column from piccolo.table import Table @@ -24,13 +25,13 @@ class Delete(Query): "where_delegate", ) - def __init__(self, table: t.Type[Table], force: bool = False, **kwargs): + def __init__(self, table: type[Table], force: bool = False, **kwargs): super().__init__(table, **kwargs) self.force = force self.returning_delegate = ReturningDelegate() self.where_delegate = WhereDelegate() - def where(self: Self, *where: t.Union[Combinable, QueryString]) -> Self: + def where(self: Self, *where: Union[Combinable, QueryString]) -> Self: self.where_delegate.where(*where) return self @@ -52,7 +53,7 @@ def _validate(self): ) @property - def default_querystrings(self) -> t.Sequence[QueryString]: + def default_querystrings(self) -> Sequence[QueryString]: query = f"DELETE FROM {self.table._meta.get_formatted_tablename()}" querystring = QueryString(query) @@ -74,4 +75,4 @@ def default_querystrings(self) -> t.Sequence[QueryString]: return [querystring] -Self = t.TypeVar("Self", bound=Delete) +Self = TypeVar("Self", bound=Delete) diff --git a/piccolo/query/methods/drop_index.py b/piccolo/query/methods/drop_index.py index 049a066dd..1b2d9f082 100644 --- a/piccolo/query/methods/drop_index.py +++ b/piccolo/query/methods/drop_index.py @@ -1,20 +1,21 @@ from __future__ import annotations -import typing as t +from collections.abc import Sequence +from typing import TYPE_CHECKING, Union from piccolo.columns.base import Column from piccolo.query.base import Query from piccolo.querystring import QueryString -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from piccolo.table import Table class DropIndex(Query): def __init__( self, - table: t.Type[Table], - columns: t.Union[t.List[Column], t.List[str]], + table: type[Table], + columns: Union[list[Column], list[str]], if_exists: bool = True, **kwargs, ): @@ -23,13 +24,13 @@ def __init__( super().__init__(table, **kwargs) @property - def column_names(self) -> t.List[str]: + def column_names(self) -> list[str]: return [ i._meta.name if isinstance(i, Column) else i for i in self.columns ] @property - def default_querystrings(self) -> t.Sequence[QueryString]: + def default_querystrings(self) -> Sequence[QueryString]: column_names = self.column_names index_name = self.table._get_index_name(column_names) query = "DROP INDEX" diff --git a/piccolo/query/methods/exists.py b/piccolo/query/methods/exists.py index 7fac83a75..d6a346ac9 100644 --- a/piccolo/query/methods/exists.py +++ b/piccolo/query/methods/exists.py @@ -1,6 +1,7 @@ from __future__ import annotations -import typing as t +from collections.abc import Sequence +from typing import TypeVar, Union from piccolo.custom_types import Combinable, TableInstance from piccolo.query.base import Query @@ -12,11 +13,11 @@ class Exists(Query[TableInstance, bool]): __slots__ = ("where_delegate",) - def __init__(self, table: t.Type[TableInstance], **kwargs): + def __init__(self, table: type[TableInstance], **kwargs): super().__init__(table, **kwargs) self.where_delegate = WhereDelegate() - def where(self: Self, *where: t.Union[Combinable, QueryString]) -> Self: + def where(self: Self, *where: Union[Combinable, QueryString]) -> Self: self.where_delegate.where(*where) return self @@ -25,7 +26,7 @@ async def response_handler(self, response) -> bool: return bool(response[0]["exists"]) @property - def default_querystrings(self) -> t.Sequence[QueryString]: + def default_querystrings(self) -> Sequence[QueryString]: select = Select(table=self.table) select.where_delegate._where = self.where_delegate._where return [ @@ -35,4 +36,4 @@ def default_querystrings(self) -> t.Sequence[QueryString]: ] -Self = t.TypeVar("Self", bound=Exists) +Self = TypeVar("Self", bound=Exists) diff --git a/piccolo/query/methods/indexes.py b/piccolo/query/methods/indexes.py index 14ab8e552..c5c8b8be7 100644 --- a/piccolo/query/methods/indexes.py +++ b/piccolo/query/methods/indexes.py @@ -1,6 +1,6 @@ from __future__ import annotations -import typing as t +from collections.abc import Sequence from piccolo.query.base import Query from piccolo.querystring import QueryString @@ -12,7 +12,7 @@ class Indexes(Query): """ @property - def postgres_querystrings(self) -> t.Sequence[QueryString]: + def postgres_querystrings(self) -> Sequence[QueryString]: return [ QueryString( "SELECT indexname AS name FROM pg_indexes " @@ -22,11 +22,11 @@ def postgres_querystrings(self) -> t.Sequence[QueryString]: ] @property - def cockroach_querystrings(self) -> t.Sequence[QueryString]: + def cockroach_querystrings(self) -> Sequence[QueryString]: return self.postgres_querystrings @property - def sqlite_querystrings(self) -> t.Sequence[QueryString]: + def sqlite_querystrings(self) -> Sequence[QueryString]: tablename = self.table._meta.tablename return [QueryString(f"PRAGMA index_list({tablename})")] diff --git a/piccolo/query/methods/insert.py b/piccolo/query/methods/insert.py index d7c655b80..f9bce9516 100644 --- a/piccolo/query/methods/insert.py +++ b/piccolo/query/methods/insert.py @@ -1,6 +1,15 @@ from __future__ import annotations -import typing as t +from collections.abc import Sequence +from typing import ( + TYPE_CHECKING, + Any, + Generic, + Literal, + Optional, + TypeVar, + Union, +) from piccolo.custom_types import Combinable, TableInstance from piccolo.query.base import Query @@ -12,18 +21,18 @@ ) from piccolo.querystring import QueryString -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from piccolo.columns.base import Column from piccolo.table import Table class Insert( - t.Generic[TableInstance], Query[TableInstance, t.List[t.Dict[str, t.Any]]] + Generic[TableInstance], Query[TableInstance, list[dict[str, Any]]] ): __slots__ = ("add_delegate", "on_conflict_delegate", "returning_delegate") def __init__( - self, table: t.Type[TableInstance], *instances: TableInstance, **kwargs + self, table: type[TableInstance], *instances: TableInstance, **kwargs ): super().__init__(table, **kwargs) self.add_delegate = AddDelegate() @@ -44,14 +53,12 @@ def returning(self: Self, *columns: Column) -> Self: def on_conflict( self: Self, - target: t.Optional[t.Union[str, Column, t.Tuple[Column, ...]]] = None, - action: t.Union[ - OnConflictAction, t.Literal["DO NOTHING", "DO UPDATE"] + target: Optional[Union[str, Column, tuple[Column, ...]]] = None, + action: Union[ + OnConflictAction, Literal["DO NOTHING", "DO UPDATE"] ] = OnConflictAction.do_nothing, - values: t.Optional[ - t.Sequence[t.Union[Column, t.Tuple[Column, t.Any]]] - ] = None, - where: t.Optional[Combinable] = None, + values: Optional[Sequence[Union[Column, tuple[Column, Any]]]] = None, + where: Optional[Combinable] = None, ) -> Self: if ( self.engine_type == "sqlite" @@ -81,7 +88,7 @@ def on_conflict( ########################################################################### - def _raw_response_callback(self, results: t.List): + def _raw_response_callback(self, results: list): """ Assign the ids of the created rows to the model instances. """ @@ -97,7 +104,7 @@ def _raw_response_callback(self, results: t.List): table_instance._exists_in_db = True @property - def default_querystrings(self) -> t.Sequence[QueryString]: + def default_querystrings(self) -> Sequence[QueryString]: base = f"INSERT INTO {self.table._meta.get_formatted_tablename()}" columns = ",".join( f'"{i._meta.db_column_name}"' for i in self.table._meta.columns @@ -142,4 +149,4 @@ def default_querystrings(self) -> t.Sequence[QueryString]: return [querystring] -Self = t.TypeVar("Self", bound=Insert) +Self = TypeVar("Self", bound=Insert) diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index a870a14e3..1bc0d711f 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -1,6 +1,16 @@ from __future__ import annotations -import typing as t +from collections.abc import Callable, Generator, Sequence +from typing import ( + TYPE_CHECKING, + Any, + Generic, + Literal, + Optional, + TypeVar, + Union, + cast, +) from piccolo.columns.column_types import ForeignKey, ReferencedTable from piccolo.columns.combination import And, Where @@ -27,7 +37,7 @@ from piccolo.utils.dictionary import make_nested from piccolo.utils.sync import run_sync -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from piccolo.columns import Column from piccolo.table import Table @@ -36,14 +46,14 @@ class GetOrCreate( - Proxy["Objects[TableInstance]", TableInstance], t.Generic[TableInstance] + Proxy["Objects[TableInstance]", TableInstance], Generic[TableInstance] ): def __init__( self, query: Objects[TableInstance], - table_class: t.Type[TableInstance], + table_class: type[TableInstance], where: Combinable, - defaults: t.Dict[Column, t.Any], + defaults: dict[Column, Any], ): self.query = query self.table_class = table_class @@ -51,7 +61,7 @@ def __init__( self.defaults = defaults async def run( - self, node: t.Optional[str] = None, in_pool: bool = True + self, node: Optional[str] = None, in_pool: bool = True ) -> TableInstance: """ :raises ValueError: @@ -100,32 +110,32 @@ async def run( .run() ) - instance = t.cast(TableInstance, instance) + instance = cast(TableInstance, instance) instance._was_created = True return instance class Get( - Proxy["First[TableInstance]", t.Optional[TableInstance]], - t.Generic[TableInstance], + Proxy["First[TableInstance]", Optional[TableInstance]], + Generic[TableInstance], ): pass class First( - Proxy["Objects[TableInstance]", t.Optional[TableInstance]], - t.Generic[TableInstance], + Proxy["Objects[TableInstance]", Optional[TableInstance]], + Generic[TableInstance], ): async def run( - self, node: t.Optional[str] = None, in_pool: bool = True - ) -> t.Optional[TableInstance]: + self, node: Optional[str] = None, in_pool: bool = True + ) -> Optional[TableInstance]: objects = await self.query.run( node=node, in_pool=in_pool, use_callbacks=False ) results = objects[0] if objects else None - modified_response: t.Optional[TableInstance] = ( + modified_response: Optional[TableInstance] = ( await self.query.callback_delegate.invoke( results=results, kind=CallbackType.success ) @@ -133,7 +143,7 @@ async def run( return modified_response -class Create(t.Generic[TableInstance]): +class Create(Generic[TableInstance]): """ This is provided as a simple convenience. Rather than running:: @@ -148,22 +158,22 @@ class Create(t.Generic[TableInstance]): def __init__( self, - table_class: t.Type[TableInstance], - columns: t.Dict[str, t.Any], + table_class: type[TableInstance], + columns: dict[str, Any], ): self.table_class = table_class self.columns = columns async def run( self, - node: t.Optional[str] = None, + node: Optional[str] = None, in_pool: bool = True, ) -> TableInstance: instance = self.table_class(**self.columns) await instance.save().run(node=node, in_pool=in_pool) return instance - def __await__(self) -> t.Generator[None, None, TableInstance]: + def __await__(self) -> Generator[None, None, TableInstance]: """ If the user doesn't explicity call .run(), proxy to it as a convenience. @@ -179,14 +189,14 @@ class UpdateSelf: def __init__( self, row: Table, - values: t.Dict[t.Union[Column, str], t.Any], + values: dict[Union[Column, str], Any], ): self.row = row self.values = values async def run( self, - node: t.Optional[str] = None, + node: Optional[str] = None, in_pool: bool = True, ) -> None: if not self.row._exists_in_db: @@ -218,7 +228,7 @@ async def run( for key, value in response[0].items(): setattr(self.row, key, value) - def __await__(self) -> t.Generator[None, None, None]: + def __await__(self) -> Generator[None, None, None]: """ If the user doesn't explicity call .run(), proxy to it as a convenience. @@ -229,7 +239,7 @@ def run_sync(self, *args, **kwargs) -> None: return run_sync(self.run(*args, **kwargs)) -class GetRelated(t.Generic[ReferencedTable]): +class GetRelated(Generic[ReferencedTable]): def __init__(self, row: Table, foreign_key: ForeignKey[ReferencedTable]): self.row = row @@ -237,9 +247,9 @@ def __init__(self, row: Table, foreign_key: ForeignKey[ReferencedTable]): async def run( self, - node: t.Optional[str] = None, + node: Optional[str] = None, in_pool: bool = True, - ) -> t.Optional[ReferencedTable]: + ) -> Optional[ReferencedTable]: if not self.row._exists_in_db: raise ValueError("The object doesn't exist in the database.") @@ -264,8 +274,8 @@ async def run( if data is None or not any(data.values()): return None - references = t.cast( - t.Type[ReferencedTable], + references = cast( + type[ReferencedTable], self.foreign_key._foreign_key_meta.resolved_references, ) @@ -275,14 +285,14 @@ async def run( def __await__( self, - ) -> t.Generator[None, None, t.Optional[ReferencedTable]]: + ) -> Generator[None, None, Optional[ReferencedTable]]: """ If the user doesn't explicity call .run(), proxy to it as a convenience. """ return self.run().__await__() - def run_sync(self, *args, **kwargs) -> t.Optional[ReferencedTable]: + def run_sync(self, *args, **kwargs) -> Optional[ReferencedTable]: return run_sync(self.run(*args, **kwargs)) @@ -290,7 +300,7 @@ def run_sync(self, *args, **kwargs) -> t.Optional[ReferencedTable]: class Objects( - Query[TableInstance, t.List[TableInstance]], t.Generic[TableInstance] + Query[TableInstance, list[TableInstance]], Generic[TableInstance] ): """ Almost identical to select, except you have to select all fields, and @@ -312,8 +322,8 @@ class Objects( def __init__( self, - table: t.Type[TableInstance], - prefetch: t.Sequence[t.Union[ForeignKey, t.List[ForeignKey]]] = (), + table: type[TableInstance], + prefetch: Sequence[Union[ForeignKey, list[ForeignKey]]] = (), **kwargs, ): super().__init__(table, **kwargs) @@ -337,7 +347,7 @@ def output(self: Self, load_json: bool = False) -> Self: def callback( self: Self, - callbacks: t.Union[t.Callable, t.List[t.Callable]], + callbacks: Union[Callable, list[Callable]], *, on: CallbackType = CallbackType.success, ) -> Self: @@ -355,7 +365,7 @@ def limit(self: Self, number: int) -> Self: return self def prefetch( - self: Self, *fk_columns: t.Union[ForeignKey, t.List[ForeignKey]] + self: Self, *fk_columns: Union[ForeignKey, list[ForeignKey]] ) -> Self: self.prefetch_delegate.prefetch(*fk_columns) return self @@ -365,9 +375,9 @@ def offset(self: Self, number: int) -> Self: return self def order_by( - self: Self, *columns: t.Union[Column, str, OrderByRaw], ascending=True + self: Self, *columns: Union[Column, str, OrderByRaw], ascending=True ) -> Self: - _columns: t.List[t.Union[Column, OrderByRaw]] = [] + _columns: list[Union[Column, OrderByRaw]] = [] for column in columns: if isinstance(column, str): _columns.append(self.table._meta.get_column_by_name(column)) @@ -377,7 +387,7 @@ def order_by( self.order_by_delegate.order_by(*_columns, ascending=ascending) return self - def where(self: Self, *where: t.Union[Combinable, QueryString]) -> Self: + def where(self: Self, *where: Union[Combinable, QueryString]) -> Self: self.where_delegate.where(*where) return self @@ -389,9 +399,9 @@ def first(self) -> First[TableInstance]: def lock_rows( self: Self, - lock_strength: t.Union[ + lock_strength: Union[ LockStrength, - t.Literal[ + Literal[ "UPDATE", "NO KEY UPDATE", "KEY SHARE", @@ -400,7 +410,7 @@ def lock_rows( ] = LockStrength.update, nowait: bool = False, skip_locked: bool = False, - of: t.Tuple[type[Table], ...] = (), + of: tuple[type[Table], ...] = (), ) -> Self: self.lock_rows_delegate.lock_rows( lock_strength, nowait, skip_locked, of @@ -415,7 +425,7 @@ def get(self, where: Combinable) -> Get[TableInstance]: def get_or_create( self, where: Combinable, - defaults: t.Optional[t.Dict[Column, t.Any]] = None, + defaults: Optional[dict[Column, Any]] = None, ) -> GetOrCreate[TableInstance]: if defaults is None: defaults = {} @@ -423,15 +433,15 @@ def get_or_create( query=self, table_class=self.table, where=where, defaults=defaults ) - def create(self, **columns: t.Any) -> Create[TableInstance]: + def create(self, **columns: Any) -> Create[TableInstance]: return Create[TableInstance](table_class=self.table, columns=columns) ########################################################################### async def batch( self, - batch_size: t.Optional[int] = None, - node: t.Optional[str] = None, + batch_size: Optional[int] = None, + node: Optional[str] = None, **kwargs, ) -> BaseBatch: if batch_size: @@ -447,7 +457,7 @@ async def response_handler(self, response): return response @property - def default_querystrings(self) -> t.Sequence[QueryString]: + def default_querystrings(self) -> Sequence[QueryString]: select = Select(table=self.table) for attr in ( @@ -481,17 +491,17 @@ def default_querystrings(self) -> t.Sequence[QueryString]: async def run( self, - node: t.Optional[str] = None, + node: Optional[str] = None, in_pool: bool = True, use_callbacks: bool = True, - ) -> t.List[TableInstance]: + ) -> list[TableInstance]: results = await super().run(node=node, in_pool=in_pool) if use_callbacks: # With callbacks, the user can return any data that they want. # Assume that most of the time they will still return a list of # Table instances. - modified: t.List[TableInstance] = ( + modified: list[TableInstance] = ( await self.callback_delegate.invoke( results, kind=CallbackType.success ) @@ -502,8 +512,8 @@ async def run( def __await__( self, - ) -> t.Generator[None, None, t.List[TableInstance]]: + ) -> Generator[None, None, list[TableInstance]]: return super().__await__() -Self = t.TypeVar("Self", bound=Objects) +Self = TypeVar("Self", bound=Objects) diff --git a/piccolo/query/methods/raw.py b/piccolo/query/methods/raw.py index 4a4ea215f..9c35ba53d 100644 --- a/piccolo/query/methods/raw.py +++ b/piccolo/query/methods/raw.py @@ -1,12 +1,13 @@ from __future__ import annotations -import typing as t +from collections.abc import Sequence +from typing import TYPE_CHECKING, Optional from piccolo.engine.base import BaseBatch from piccolo.query.base import Query from piccolo.querystring import QueryString -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from piccolo.table import Table @@ -15,7 +16,7 @@ class Raw(Query): def __init__( self, - table: t.Type[Table], + table: type[Table], querystring: QueryString = QueryString(""), **kwargs, ): @@ -24,8 +25,8 @@ def __init__( async def batch( self, - batch_size: t.Optional[int] = None, - node: t.Optional[str] = None, + batch_size: Optional[int] = None, + node: Optional[str] = None, **kwargs, ) -> BaseBatch: if batch_size: @@ -35,5 +36,5 @@ async def batch( return await self.table._meta.db.batch(self, **kwargs) @property - def default_querystrings(self) -> t.Sequence[QueryString]: + def default_querystrings(self) -> Sequence[QueryString]: return [self.querystring] diff --git a/piccolo/query/methods/refresh.py b/piccolo/query/methods/refresh.py index 5807285be..792e14df3 100644 --- a/piccolo/query/methods/refresh.py +++ b/piccolo/query/methods/refresh.py @@ -1,11 +1,12 @@ from __future__ import annotations -import typing as t +from collections.abc import Sequence +from typing import TYPE_CHECKING, Optional from piccolo.utils.encoding import JSONDict from piccolo.utils.sync import run_sync -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from piccolo.columns import Column from piccolo.table import Table @@ -30,7 +31,7 @@ class Refresh: def __init__( self, instance: Table, - columns: t.Optional[t.Sequence[Column]] = None, + columns: Optional[Sequence[Column]] = None, load_json: bool = False, ): self.instance = instance @@ -50,7 +51,7 @@ def __init__( self.load_json = load_json @property - def _columns(self) -> t.Sequence[Column]: + def _columns(self) -> Sequence[Column]: """ Works out which columns the user wants to refresh. """ @@ -61,7 +62,7 @@ def _columns(self) -> t.Sequence[Column]: i for i in self.instance._meta.columns if not i._meta.primary_key ] - def _get_columns(self, instance: Table, columns: t.Sequence[Column]): + def _get_columns(self, instance: Table, columns: Sequence[Column]): """ If `prefetch` was used on the object, for example:: @@ -100,7 +101,7 @@ def _get_columns(self, instance: Table, columns: t.Sequence[Column]): return select_columns - def _update_instance(self, instance: Table, data_dict: t.Dict): + def _update_instance(self, instance: Table, data_dict: dict): """ Update the table instance. It is called recursively, if the instance has child instances. @@ -119,7 +120,7 @@ def _update_instance(self, instance: Table, data_dict: t.Dict): setattr(instance, key, value) async def run( - self, in_pool: bool = True, node: t.Optional[str] = None + self, in_pool: bool = True, node: Optional[str] = None ) -> Table: """ Run it asynchronously. For example:: diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index c45fdbd46..4ba3a2977 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -1,8 +1,17 @@ from __future__ import annotations import itertools -import typing as t from collections import OrderedDict +from collections.abc import Callable, Sequence +from typing import ( + TYPE_CHECKING, + Any, + Literal, + Optional, + TypeVar, + Union, + overload, +) from piccolo.columns import Column, Selectable from piccolo.columns.column_types import JSON, JSONB @@ -33,7 +42,7 @@ from piccolo.utils.encoding import dump_json, load_json from piccolo.utils.warnings import colored_warning -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from piccolo.custom_types import Combinable from piccolo.table import Table # noqa @@ -48,7 +57,7 @@ class SelectRaw(Selectable): - def __init__(self, sql: str, *args: t.Any) -> None: + def __init__(self, sql: str, *args: Any) -> None: """ Execute raw SQL in your select query. @@ -69,7 +78,7 @@ def get_select_string( return self.querystring -OptionalDict = t.Optional[t.Dict[str, t.Any]] +OptionalDict = Optional[dict[str, Any]] class First(Proxy["Select", OptionalDict]): @@ -82,7 +91,7 @@ def __init__(self, query: Select): async def run( self, - node: t.Optional[str] = None, + node: Optional[str] = None, in_pool: bool = True, ) -> OptionalDict: rows = await self.query.run( @@ -96,16 +105,16 @@ async def run( return modified_response -class SelectList(Proxy["Select", t.List]): +class SelectList(Proxy["Select", list]): """ This is for static typing purposes. """ async def run( self, - node: t.Optional[str] = None, + node: Optional[str] = None, in_pool: bool = True, - ) -> t.List: + ) -> list: rows = await self.query.run( node=node, in_pool=in_pool, use_callbacks=False ) @@ -131,14 +140,14 @@ class SelectJSON(Proxy["Select", str]): async def run( self, - node: t.Optional[str] = None, + node: Optional[str] = None, in_pool: bool = True, ) -> str: rows = await self.query.run(node=node, in_pool=in_pool) return dump_json(rows) -class Select(Query[TableInstance, t.List[t.Dict[str, t.Any]]]): +class Select(Query[TableInstance, list[dict[str, Any]]]): __slots__ = ( "columns_list", "exclude_secrets", @@ -157,8 +166,8 @@ class Select(Query[TableInstance, t.List[t.Dict[str, t.Any]]]): def __init__( self, - table: t.Type[TableInstance], - columns_list: t.Optional[t.Sequence[t.Union[Selectable, str]]] = None, + table: type[TableInstance], + columns_list: Optional[Sequence[Union[Selectable, str]]] = None, exclude_secrets: bool = False, **kwargs, ): @@ -181,22 +190,20 @@ def __init__( self.columns(*columns_list) - def columns(self: Self, *columns: t.Union[Selectable, str]) -> Self: + def columns(self: Self, *columns: Union[Selectable, str]) -> Self: _columns = self.table._process_column_args(*columns) self.columns_delegate.columns(*_columns) return self - def distinct( - self: Self, *, on: t.Optional[t.Sequence[Column]] = None - ) -> Self: + def distinct(self: Self, *, on: Optional[Sequence[Column]] = None) -> Self: if on is not None and self.engine_type == "sqlite": raise NotImplementedError("SQLite doesn't support DISTINCT ON") self.distinct_delegate.distinct(enabled=True, on=on) return self - def group_by(self: Self, *columns: t.Union[Column, str]) -> Self: - _columns: t.List[Column] = [ + def group_by(self: Self, *columns: Union[Column, str]) -> Self: + _columns: list[Column] = [ i for i in self.table._process_column_args(*columns) if isinstance(i, Column) @@ -225,9 +232,9 @@ def offset(self: Self, number: int) -> Self: def lock_rows( self: Self, - lock_strength: t.Union[ + lock_strength: Union[ LockStrength, - t.Literal[ + Literal[ "UPDATE", "NO KEY UPDATE", "KEY SHARE", @@ -236,7 +243,7 @@ def lock_rows( ] = LockStrength.update, nowait: bool = False, skip_locked: bool = False, - of: t.Tuple[type[Table], ...] = (), + of: tuple[type[Table], ...] = (), ) -> Self: self.lock_rows_delegate.lock_rows( lock_strength, nowait, skip_locked, of @@ -245,8 +252,8 @@ def lock_rows( async def _splice_m2m_rows( self, - response: t.List[t.Dict[str, t.Any]], - secondary_table: t.Type[Table], + response: list[dict[str, Any]], + secondary_table: type[Table], secondary_table_pk: Column, m2m_name: str, m2m_select: M2MSelect, @@ -396,7 +403,7 @@ async def response_handler(self, response): return response def order_by( - self: Self, *columns: t.Union[Column, str, OrderByRaw], ascending=True + self: Self, *columns: Union[Column, str, OrderByRaw], ascending=True ) -> Self: """ :param columns: @@ -405,7 +412,7 @@ def order_by( which allows you for complex use cases like ``OrderByRaw('random()')``. """ - _columns: t.List[t.Union[Column, OrderByRaw]] = [] + _columns: list[Union[Column, OrderByRaw]] = [] for column in columns: if isinstance(column, str): _columns.append(self.table._meta.get_column_by_name(column)) @@ -415,25 +422,25 @@ def order_by( self.order_by_delegate.order_by(*_columns, ascending=ascending) return self - @t.overload + @overload def output(self: Self, *, as_list: bool) -> SelectList: # type: ignore ... - @t.overload + @overload def output(self: Self, *, as_json: bool) -> SelectJSON: # type: ignore ... - @t.overload + @overload def output(self: Self, *, load_json: bool) -> Self: ... - @t.overload + @overload def output(self: Self, *, load_json: bool, as_list: bool) -> SelectJSON: # type: ignore # noqa: E501 ... - @t.overload + @overload def output(self: Self, *, load_json: bool, nested: bool) -> Self: ... - @t.overload + @overload def output(self: Self, *, nested: bool) -> Self: ... def output( @@ -443,7 +450,7 @@ def output( as_json: bool = False, load_json: bool = False, nested: bool = False, - ) -> t.Union[Self, SelectJSON, SelectList]: + ) -> Union[Self, SelectJSON, SelectList]: self.output_delegate.output( as_list=as_list, as_json=as_json, @@ -459,21 +466,21 @@ def output( def callback( self: Self, - callbacks: t.Union[t.Callable, t.List[t.Callable]], + callbacks: Union[Callable, list[Callable]], *, on: CallbackType = CallbackType.success, ) -> Self: self.callback_delegate.callback(callbacks, on=on) return self - def where(self: Self, *where: t.Union[Combinable, QueryString]) -> Self: + def where(self: Self, *where: Union[Combinable, QueryString]) -> Self: self.where_delegate.where(*where) return self async def batch( self, - batch_size: t.Optional[int] = None, - node: t.Optional[str] = None, + batch_size: Optional[int] = None, + node: Optional[str] = None, **kwargs, ) -> BaseBatch: if batch_size: @@ -484,14 +491,14 @@ async def batch( ########################################################################### - def _get_joins(self, columns: t.Sequence[Selectable]) -> t.List[str]: + def _get_joins(self, columns: Sequence[Selectable]) -> list[str]: """ A call chain is a sequence of foreign keys representing joins which need to be made to retrieve a column in another table. """ - joins: t.List[str] = [] + joins: list[str] = [] - readables: t.List[Readable] = [ + readables: list[Readable] = [ i for i in columns if isinstance(i, Readable) ] @@ -499,7 +506,7 @@ def _get_joins(self, columns: t.Sequence[Selectable]) -> t.List[str]: for readable in readables: columns += readable.columns - querystrings: t.List[QueryString] = [ + querystrings: list[QueryString] = [ i for i in columns if isinstance(i, QueryString) ] for querystring in querystrings: @@ -510,7 +517,7 @@ def _get_joins(self, columns: t.Sequence[Selectable]) -> t.List[str]: if not isinstance(column, Column): continue - _joins: t.List[str] = [] + _joins: list[str] = [] for index, key in enumerate(column._meta.call_chain, 0): table_alias = key.table_alias @@ -542,7 +549,7 @@ def _get_joins(self, columns: t.Sequence[Selectable]) -> t.List[str]: # Remove duplicates return list(OrderedDict.fromkeys(joins)) - def _check_valid_call_chain(self, keys: t.Sequence[Selectable]) -> bool: + def _check_valid_call_chain(self, keys: Sequence[Selectable]) -> bool: for column in keys: if not isinstance(column, Column): continue @@ -556,7 +563,7 @@ def _check_valid_call_chain(self, keys: t.Sequence[Selectable]) -> bool: return True @property - def default_querystrings(self) -> t.Sequence[QueryString]: + def default_querystrings(self) -> Sequence[QueryString]: # JOIN self._check_valid_call_chain(self.columns_delegate.selected_columns) @@ -567,7 +574,7 @@ def default_querystrings(self) -> t.Sequence[QueryString]: ) # Combine all joins, and remove duplicates - joins: t.List[str] = list( + joins: list[str] = list( OrderedDict.fromkeys(select_joins + where_joins + order_by_joins) ) @@ -584,14 +591,14 @@ def default_querystrings(self) -> t.Sequence[QueryString]: engine_type = self.table._meta.db.engine_type - select_strings: t.List[QueryString] = [ + select_strings: list[QueryString] = [ c.get_select_string(engine_type=engine_type) for c in self.columns_delegate.selected_columns ] ####################################################################### - args: t.List[t.Any] = [] + args: list[Any] = [] query = "SELECT" @@ -658,11 +665,11 @@ def default_querystrings(self) -> t.Sequence[QueryString]: async def run( self, - node: t.Optional[str] = None, + node: Optional[str] = None, in_pool: bool = True, use_callbacks: bool = True, **kwargs, - ) -> t.List[t.Dict[str, t.Any]]: + ) -> list[dict[str, Any]]: results = await super().run(node=node, in_pool=in_pool) if use_callbacks: return await self.callback_delegate.invoke( @@ -672,4 +679,4 @@ async def run( return results -Self = t.TypeVar("Self", bound=Select) +Self = TypeVar("Self", bound=Select) diff --git a/piccolo/query/methods/table_exists.py b/piccolo/query/methods/table_exists.py index dc801bb59..2d90059cd 100644 --- a/piccolo/query/methods/table_exists.py +++ b/piccolo/query/methods/table_exists.py @@ -1,6 +1,6 @@ from __future__ import annotations -import typing as t +from collections.abc import Sequence from piccolo.custom_types import TableInstance from piccolo.query.base import Query @@ -9,13 +9,13 @@ class TableExists(Query[TableInstance, bool]): - __slots__: t.Tuple = () + __slots__: tuple = () async def response_handler(self, response): return bool(response[0]["exists"]) @property - def sqlite_querystrings(self) -> t.Sequence[QueryString]: + def sqlite_querystrings(self) -> Sequence[QueryString]: return [ QueryString( "SELECT EXISTS(SELECT * FROM sqlite_master WHERE " @@ -25,7 +25,7 @@ def sqlite_querystrings(self) -> t.Sequence[QueryString]: ] @property - def postgres_querystrings(self) -> t.Sequence[QueryString]: + def postgres_querystrings(self) -> Sequence[QueryString]: subquery = QueryString( "SELECT * FROM information_schema.tables WHERE table_name = {}", self.table._meta.tablename, @@ -41,5 +41,5 @@ def postgres_querystrings(self) -> t.Sequence[QueryString]: return [query] @property - def cockroach_querystrings(self) -> t.Sequence[QueryString]: + def cockroach_querystrings(self) -> Sequence[QueryString]: return self.postgres_querystrings diff --git a/piccolo/query/methods/update.py b/piccolo/query/methods/update.py index 5cd2e5073..97715b7d9 100644 --- a/piccolo/query/methods/update.py +++ b/piccolo/query/methods/update.py @@ -1,6 +1,7 @@ from __future__ import annotations -import typing as t +from collections.abc import Sequence +from typing import TYPE_CHECKING, Any, Optional, Union from piccolo.custom_types import Combinable, TableInstance from piccolo.query.base import Query @@ -11,7 +12,7 @@ ) from piccolo.querystring import QueryString -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from piccolo.columns import Column @@ -19,7 +20,7 @@ class UpdateError(Exception): pass -class Update(Query[TableInstance, t.List[t.Any]]): +class Update(Query[TableInstance, list[Any]]): __slots__ = ( "force", "returning_delegate", @@ -28,7 +29,7 @@ class Update(Query[TableInstance, t.List[t.Any]]): ) def __init__( - self, table: t.Type[TableInstance], force: bool = False, **kwargs + self, table: type[TableInstance], force: bool = False, **kwargs ): super().__init__(table, **kwargs) self.force = force @@ -41,7 +42,7 @@ def __init__( def values( self, - values: t.Optional[t.Dict[t.Union[Column, str], t.Any]] = None, + values: Optional[dict[Union[Column, str], Any]] = None, **kwargs, ) -> Update: if values is None: @@ -50,7 +51,7 @@ def values( self.values_delegate.values(values) return self - def where(self, *where: t.Union[Combinable, QueryString]) -> Update: + def where(self, *where: Union[Combinable, QueryString]) -> Update: self.where_delegate.where(*where) return self @@ -85,7 +86,7 @@ def _validate(self): ########################################################################### @property - def default_querystrings(self) -> t.Sequence[QueryString]: + def default_querystrings(self) -> Sequence[QueryString]: columns_str = ", ".join( f'"{col._meta.db_column_name}" = {{}}' for col, _ in self.values_delegate._values.items() diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index b1b726cf3..16b69e8b7 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -3,9 +3,10 @@ import asyncio import collections.abc import itertools -import typing as t +from collections.abc import Callable, Sequence from dataclasses import dataclass, field from enum import Enum, auto +from typing import TYPE_CHECKING, Any, Literal, Optional, Union from piccolo.columns import And, Column, Or, Where from piccolo.columns.column_types import ForeignKey @@ -15,7 +16,7 @@ from piccolo.utils.list import flatten from piccolo.utils.sql_values import convert_to_sql_value -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from piccolo.querystring import Selectable from piccolo.table import Table # noqa @@ -33,7 +34,7 @@ class Distinct: __slots__ = ("enabled", "on") enabled: bool - on: t.Optional[t.Sequence[Column]] + on: Optional[Sequence[Column]] @property def querystring(self) -> QueryString: @@ -152,17 +153,17 @@ class OrderByRaw: class OrderByItem: __slots__ = ("columns", "ascending") - columns: t.Sequence[t.Union[Column, OrderByRaw]] + columns: Sequence[Union[Column, OrderByRaw]] ascending: bool @dataclass class OrderBy: - order_by_items: t.List[OrderByItem] = field(default_factory=list) + order_by_items: list[OrderByItem] = field(default_factory=list) @property def querystring(self) -> QueryString: - order_by_strings: t.List[str] = [] + order_by_strings: list[str] = [] for order_by_item in self.order_by_items: order = "ASC" if order_by_item.ascending else "DESC" for column in order_by_item.columns: @@ -185,7 +186,7 @@ def __str__(self): class Returning: __slots__ = ("columns",) - columns: t.List[Column] + columns: list[Column] @property def querystring(self) -> QueryString: @@ -230,13 +231,13 @@ class CallbackType(Enum): @dataclass class Callback: kind: CallbackType - target: t.Callable + target: Callable @dataclass class WhereDelegate: - _where: t.Optional[Combinable] = None - _where_columns: t.List[Column] = field(default_factory=list) + _where: Optional[Combinable] = None + _where_columns: list[Column] = field(default_factory=list) def get_where_columns(self): """ @@ -257,7 +258,7 @@ def _extract_columns(self, combinable: Combinable): elif isinstance(combinable, WhereRaw): self._where_columns.extend(combinable.querystring.columns) - def where(self, *where: t.Union[Combinable, QueryString]): + def where(self, *where: Union[Combinable, QueryString]): for arg in where: if isinstance(arg, bool): raise ValueError( @@ -278,7 +279,7 @@ def where(self, *where: t.Union[Combinable, QueryString]): class OrderByDelegate: _order_by: OrderBy = field(default_factory=OrderBy) - def get_order_by_columns(self) -> t.List[Column]: + def get_order_by_columns(self) -> list[Column]: """ Used to work out which columns are needed for joins. """ @@ -290,7 +291,7 @@ def get_order_by_columns(self) -> t.List[Column]: if isinstance(i, Column) ] - def order_by(self, *columns: t.Union[Column, OrderByRaw], ascending=True): + def order_by(self, *columns: Union[Column, OrderByRaw], ascending=True): if len(columns) < 1: raise ValueError("At least one column must be passed to order_by.") @@ -301,7 +302,7 @@ def order_by(self, *columns: t.Union[Column, OrderByRaw], ascending=True): @dataclass class LimitDelegate: - _limit: t.Optional[Limit] = None + _limit: Optional[Limit] = None _first: bool = False def limit(self, number: int): @@ -319,7 +320,7 @@ class AsOfDelegate: Currently supports Cockroach using AS OF SYSTEM TIME. """ - _as_of: t.Optional[AsOf] = None + _as_of: Optional[AsOf] = None def as_of(self, interval: str = "-1s"): self._as_of = AsOf(interval) @@ -331,9 +332,7 @@ class DistinctDelegate: default_factory=lambda: Distinct(enabled=False, on=None) ) - def distinct( - self, enabled: bool, on: t.Optional[t.Sequence[Column]] = None - ): + def distinct(self, enabled: bool, on: Optional[Sequence[Column]] = None): if on and not isinstance(on, collections.abc.Sequence): # Check a sequence is passed in, otherwise the user will get some # unuseful errors later on. @@ -344,9 +343,9 @@ def distinct( @dataclass class ReturningDelegate: - _returning: t.Optional[Returning] = None + _returning: Optional[Returning] = None - def returning(self, columns: t.Sequence[Column]): + def returning(self, columns: Sequence[Column]): self._returning = Returning(columns=list(columns)) @@ -360,9 +359,9 @@ def count(self): @dataclass class AddDelegate: - _add: t.List[Table] = field(default_factory=list) + _add: list[Table] = field(default_factory=list) - def add(self, *instances: Table, table_class: t.Type[Table]): + def add(self, *instances: Table, table_class: type[Table]): for instance in instances: if not isinstance(instance, table_class): raise TypeError("Incompatible type added.") @@ -384,10 +383,10 @@ class OutputDelegate: def output( self, - as_list: t.Optional[bool] = None, - as_json: t.Optional[bool] = None, - load_json: t.Optional[bool] = None, - nested: t.Optional[bool] = None, + as_list: Optional[bool] = None, + as_json: Optional[bool] = None, + load_json: Optional[bool] = None, + nested: Optional[bool] = None, ): """ :param as_list: @@ -429,13 +428,13 @@ class CallbackDelegate: .callback([handler1, handler2]) """ - _callbacks: t.Dict[CallbackType, t.List[Callback]] = field( + _callbacks: dict[CallbackType, list[Callback]] = field( default_factory=lambda: {kind: [] for kind in CallbackType} ) def callback( self, - callbacks: t.Union[t.Callable, t.List[t.Callable]], + callbacks: Union[Callable, list[Callable]], *, on: CallbackType, ): @@ -446,7 +445,7 @@ def callback( else: self._callbacks[on].append(Callback(kind=on, target=callbacks)) - async def invoke(self, results: t.Any, *, kind: CallbackType) -> t.Any: + async def invoke(self, results: Any, *, kind: CallbackType) -> Any: """ Utility function that invokes the registered callbacks in the correct way, handling both sync and async callbacks. Only callbacks of the @@ -472,9 +471,9 @@ class PrefetchDelegate: .prefetch(MyTable.column_a, MyTable.column_b) """ - fk_columns: t.List[ForeignKey] = field(default_factory=list) + fk_columns: list[ForeignKey] = field(default_factory=list) - def prefetch(self, *fk_columns: t.Union[ForeignKey, t.List[ForeignKey]]): + def prefetch(self, *fk_columns: Union[ForeignKey, list[ForeignKey]]): """ :param columns: We accept ``ForeignKey`` and ``List[ForeignKey]`` here, in case @@ -482,7 +481,7 @@ def prefetch(self, *fk_columns: t.Union[ForeignKey, t.List[ForeignKey]]): in which case we flatten the list. """ - _fk_columns: t.List[ForeignKey] = [] + _fk_columns: list[ForeignKey] = [] for column in fk_columns: if isinstance(column, list): _fk_columns.extend(column) @@ -501,9 +500,9 @@ class ColumnsDelegate: .columns(MyTable.column_a, MyTable.column_b) """ - selected_columns: t.Sequence[Selectable] = field(default_factory=list) + selected_columns: Sequence[Selectable] = field(default_factory=list) - def columns(self, *columns: t.Union[Selectable, t.List[Selectable]]): + def columns(self, *columns: Union[Selectable, list[Selectable]]): """ :param columns: We accept ``Selectable`` and ``List[Selectable]`` here, in case @@ -531,10 +530,10 @@ class ValuesDelegate: Used to specify new column values - primarily used in update queries. """ - table: t.Type[Table] - _values: t.Dict[Column, t.Any] = field(default_factory=dict) + table: type[Table] + _values: dict[Column, Any] = field(default_factory=dict) - def values(self, values: t.Dict[t.Union[Column, str], t.Any]): + def values(self, values: dict[Union[Column, str], Any]): """ Example usage: @@ -549,7 +548,7 @@ def values(self, values: t.Dict[t.Union[Column, str], t.Any]): .values(column_a=1}) """ - cleaned_values: t.Dict[Column, t.Any] = {} + cleaned_values: dict[Column, Any] = {} for key, value in values.items(): if isinstance(key, Column): column = key @@ -564,7 +563,7 @@ def values(self, values: t.Dict[t.Union[Column, str], t.Any]): self._values.update(cleaned_values) - def get_sql_values(self) -> t.List[t.Any]: + def get_sql_values(self) -> list[Any]: """ Convert any Enums into values, and serialise any JSON. """ @@ -587,7 +586,7 @@ class OffsetDelegate: """ - _offset: t.Optional[Offset] = None + _offset: Optional[Offset] = None def offset(self, number: int = 0): self._offset = Offset(number) @@ -597,7 +596,7 @@ def offset(self, number: int = 0): class GroupBy: __slots__ = ("columns",) - columns: t.Sequence[Column] + columns: Sequence[Column] @property def querystring(self) -> QueryString: @@ -620,7 +619,7 @@ class GroupByDelegate: """ - _group_by: t.Optional[GroupBy] = None + _group_by: Optional[GroupBy] = None def group_by(self, *columns: Column): self._group_by = GroupBy(columns=columns) @@ -637,12 +636,10 @@ class OnConflictAction(str, Enum): @dataclass class OnConflictItem: - target: t.Optional[t.Union[str, Column, t.Tuple[Column, ...]]] = None - action: t.Optional[OnConflictAction] = None - values: t.Optional[t.Sequence[t.Union[Column, t.Tuple[Column, t.Any]]]] = ( - None - ) - where: t.Optional[Combinable] = None + target: Optional[Union[str, Column, tuple[Column, ...]]] = None + action: Optional[OnConflictAction] = None + values: Optional[Sequence[Union[Column, tuple[Column, Any]]]] = None + where: Optional[Combinable] = None @property def target_string(self) -> str: @@ -726,7 +723,7 @@ class OnConflict: parent class. """ - on_conflict_items: t.List[OnConflictItem] = field(default_factory=list) + on_conflict_items: list[OnConflictItem] = field(default_factory=list) @property def querystring(self) -> QueryString: @@ -757,14 +754,12 @@ class OnConflictDelegate: def on_conflict( self, - target: t.Optional[t.Union[str, Column, t.Tuple[Column, ...]]] = None, - action: t.Union[ - OnConflictAction, t.Literal["DO NOTHING", "DO UPDATE"] + target: Optional[Union[str, Column, tuple[Column, ...]]] = None, + action: Union[ + OnConflictAction, Literal["DO NOTHING", "DO UPDATE"] ] = OnConflictAction.do_nothing, - values: t.Optional[ - t.Sequence[t.Union[Column, t.Tuple[Column, t.Any]]] - ] = None, - where: t.Optional[Combinable] = None, + values: Optional[Sequence[Union[Column, tuple[Column, Any]]]] = None, + where: Optional[Combinable] = None, ): action_: OnConflictAction if isinstance(action, OnConflictAction): @@ -806,7 +801,7 @@ class LockRows: lock_strength: LockStrength nowait: bool skip_locked: bool - of: t.Tuple[t.Type[Table], ...] + of: tuple[type[Table], ...] def __post_init__(self): if not isinstance(self.lock_strength, LockStrength): @@ -846,13 +841,13 @@ def __str__(self) -> str: @dataclass class LockRowsDelegate: - _lock_rows: t.Optional[LockRows] = None + _lock_rows: Optional[LockRows] = None def lock_rows( self, - lock_strength: t.Union[ + lock_strength: Union[ LockStrength, - t.Literal[ + Literal[ "UPDATE", "NO KEY UPDATE", "KEY SHARE", @@ -861,7 +856,7 @@ def lock_rows( ] = LockStrength.update, nowait=False, skip_locked=False, - of: t.Tuple[type[Table], ...] = (), + of: tuple[type[Table], ...] = (), ): lock_strength_: LockStrength if isinstance(lock_strength, LockStrength): diff --git a/piccolo/query/operators/json.py b/piccolo/query/operators/json.py index ea6d05097..be7529135 100644 --- a/piccolo/query/operators/json.py +++ b/piccolo/query/operators/json.py @@ -1,17 +1,17 @@ from __future__ import annotations -import typing as t +from typing import TYPE_CHECKING, Any, Optional, Union from piccolo.querystring import QueryString from piccolo.utils.encoding import dump_json -if t.TYPE_CHECKING: +if TYPE_CHECKING: from piccolo.columns.column_types import JSON class JSONQueryString(QueryString): - def clean_value(self, value: t.Any): + def clean_value(self, value: Any): if not isinstance(value, (str, QueryString)): value = dump_json(value) return value @@ -42,9 +42,9 @@ class GetChildElement(JSONQueryString): def __init__( self, - identifier: t.Union[JSON, QueryString], - key: t.Union[str, int, QueryString], - alias: t.Optional[str] = None, + identifier: Union[JSON, QueryString], + key: Union[str, int, QueryString], + alias: Optional[str] = None, ): if isinstance(key, int): # asyncpg only accepts integer keys if we explicitly mark it as an @@ -53,7 +53,7 @@ def __init__( super().__init__("{} -> {}", identifier, key, alias=alias) - def arrow(self, key: t.Union[str, int, QueryString]) -> GetChildElement: + def arrow(self, key: Union[str, int, QueryString]) -> GetChildElement: """ This allows you to drill multiple levels deep into a JSON object if needed. @@ -77,7 +77,7 @@ def arrow(self, key: t.Union[str, int, QueryString]) -> GetChildElement: return GetChildElement(identifier=self, key=key, alias=self._alias) def __getitem__( - self, value: t.Union[str, int, QueryString] + self, value: Union[str, int, QueryString] ) -> GetChildElement: return GetChildElement(identifier=self, key=value, alias=self._alias) @@ -94,9 +94,9 @@ class GetElementFromPath(JSONQueryString): def __init__( self, - identifier: t.Union[JSON, QueryString], - path: t.List[t.Union[str, int]], - alias: t.Optional[str] = None, + identifier: Union[JSON, QueryString], + path: list[Union[str, int]], + alias: Optional[str] = None, ): """ :param path: diff --git a/piccolo/query/proxy.py b/piccolo/query/proxy.py index 30ce06083..03b9827ca 100644 --- a/piccolo/query/proxy.py +++ b/piccolo/query/proxy.py @@ -1,5 +1,6 @@ import inspect -import typing as t +from collections.abc import Generator +from typing import Generic, Optional, TypeVar from typing_extensions import Protocol @@ -8,22 +9,20 @@ class Runnable(Protocol): - async def run( - self, node: t.Optional[str] = None, in_pool: bool = True - ): ... + async def run(self, node: Optional[str] = None, in_pool: bool = True): ... -QueryType = t.TypeVar("QueryType", bound=Runnable) -ResponseType = t.TypeVar("ResponseType") +QueryType = TypeVar("QueryType", bound=Runnable) +ResponseType = TypeVar("ResponseType") -class Proxy(t.Generic[QueryType, ResponseType]): +class Proxy(Generic[QueryType, ResponseType]): def __init__(self, query: QueryType): self.query = query async def run( self, - node: t.Optional[str] = None, + node: Optional[str] = None, in_pool: bool = True, ) -> ResponseType: return await self.query.run(node=node, in_pool=in_pool) @@ -33,7 +32,7 @@ def run_sync(self, *args, **kwargs) -> ResponseType: def __await__( self, - ) -> t.Generator[None, None, ResponseType]: + ) -> Generator[None, None, ResponseType]: """ If the user doesn't explicity call .run(), proxy to it as a convenience. diff --git a/piccolo/querystring.py b/piccolo/querystring.py index 7dec758a8..ea4b686c8 100644 --- a/piccolo/querystring.py +++ b/piccolo/querystring.py @@ -1,15 +1,16 @@ from __future__ import annotations -import typing as t from abc import ABCMeta, abstractmethod +from collections.abc import Sequence from dataclasses import dataclass from datetime import datetime from importlib.util import find_spec from string import Formatter +from typing import TYPE_CHECKING, Any, Optional -if t.TYPE_CHECKING: # pragma: no cover - from piccolo.table import Table +if TYPE_CHECKING: # pragma: no cover from piccolo.columns import Column + from piccolo.table import Table from uuid import UUID @@ -26,7 +27,7 @@ class Selectable(metaclass=ABCMeta): __slots__ = ("_alias",) - _alias: t.Optional[str] + _alias: Optional[str] @abstractmethod def get_select_string( @@ -74,10 +75,10 @@ class QueryString(Selectable): def __init__( self, template: str, - *args: t.Any, + *args: Any, query_type: str = "generic", - table: t.Optional[t.Type[Table]] = None, - alias: t.Optional[str] = None, + table: Optional[type[Table]] = None, + alias: Optional[str] = None, ) -> None: """ :param template: @@ -99,15 +100,13 @@ def __init__( self.template = template self.query_type = query_type self.table = table - self._frozen_compiled_strings: t.Optional[ - t.Tuple[str, t.List[t.Any]] - ] = None + self._frozen_compiled_strings: Optional[tuple[str, list[Any]]] = None self._alias = alias self.args, self.columns = self.process_args(args) def process_args( - self, args: t.Sequence[t.Any] - ) -> t.Tuple[t.Sequence[t.Any], t.Sequence[Column]]: + self, args: Sequence[Any] + ) -> tuple[Sequence[Any], Sequence[Column]]: """ If a Column is passed in, we convert it to the name of the column (including joins). @@ -168,8 +167,8 @@ def __str__(self): def bundle( self, start_index: int = 1, - bundled: t.Optional[t.List[Fragment]] = None, - combined_args: t.Optional[t.List] = None, + bundled: Optional[list[Fragment]] = None, + combined_args: Optional[list] = None, ): # Split up the string, separating by {}. fragments = [ @@ -206,7 +205,7 @@ def bundle( def compile_string( self, engine_type: str = "postgres" - ) -> t.Tuple[str, t.List[t.Any]]: + ) -> tuple[str, list[Any]]: """ Compiles the template ready for the engine - keeping the arguments separate from the template. diff --git a/piccolo/schema.py b/piccolo/schema.py index 01cd0bd91..8d949a700 100644 --- a/piccolo/schema.py +++ b/piccolo/schema.py @@ -1,7 +1,7 @@ from __future__ import annotations import abc -import typing as t +from typing import Optional, cast from piccolo.engine.base import Engine from piccolo.engine.finder import engine_finder @@ -110,7 +110,7 @@ def __init__( table_name: str, new_schema: str, db: Engine, - current_schema: t.Optional[str] = None, + current_schema: Optional[str] = None, ): self.table_name = table_name self.current_schema = current_schema @@ -131,9 +131,9 @@ def __init__(self, db: Engine, schema_name: str): self.db = db self.schema_name = schema_name - async def run(self) -> t.List[str]: - response = t.cast( - t.List[t.Dict], + async def run(self) -> list[str]: + response = cast( + list[dict], await self.db.run_querystring( QueryString( """ @@ -158,9 +158,9 @@ class ListSchemas: def __init__(self, db: Engine): self.db = db - async def run(self) -> t.List[str]: - response = t.cast( - t.List[t.Dict], + async def run(self) -> list[str]: + response = cast( + list[dict], await self.db.run_querystring( QueryString( "SELECT schema_name FROM information_schema.schemata" @@ -177,7 +177,7 @@ def __await__(self): class SchemaManager: - def __init__(self, db: t.Optional[Engine] = None): + def __init__(self, db: Optional[Engine] = None): """ A useful utility class for interacting with schemas. @@ -267,7 +267,7 @@ def move_table( self, table_name: str, new_schema: str, - current_schema: t.Optional[str] = None, + current_schema: Optional[str] = None, ) -> MoveTable: """ Moves a table to a different schema:: diff --git a/piccolo/table.py b/piccolo/table.py index ee98c1bfa..e4ddd7daf 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -3,9 +3,10 @@ import inspect import itertools import types -import typing as t import warnings +from collections.abc import Sequence from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Optional, Union, cast, overload from piccolo.columns import Column from piccolo.columns.column_types import ( @@ -55,7 +56,7 @@ from piccolo.utils.sync import run_sync from piccolo.utils.warnings import colored_warning -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from piccolo.querystring import Selectable PROTECTED_TABLENAMES = ("user",) @@ -65,7 +66,7 @@ ) -TABLE_REGISTRY: t.List[t.Type[Table]] = [] +TABLE_REGISTRY: list[type[Table]] = [] @dataclass @@ -75,26 +76,26 @@ class TableMeta: """ tablename: str = "" - columns: t.List[Column] = field(default_factory=list) - default_columns: t.List[Column] = field(default_factory=list) - non_default_columns: t.List[Column] = field(default_factory=list) - array_columns: t.List[Array] = field(default_factory=list) - email_columns: t.List[Email] = field(default_factory=list) - foreign_key_columns: t.List[ForeignKey] = field(default_factory=list) + columns: list[Column] = field(default_factory=list) + default_columns: list[Column] = field(default_factory=list) + non_default_columns: list[Column] = field(default_factory=list) + array_columns: list[Array] = field(default_factory=list) + email_columns: list[Email] = field(default_factory=list) + foreign_key_columns: list[ForeignKey] = field(default_factory=list) primary_key: Column = field(default_factory=Column) - json_columns: t.List[t.Union[JSON, JSONB]] = field(default_factory=list) - secret_columns: t.List[Secret] = field(default_factory=list) - auto_update_columns: t.List[Column] = field(default_factory=list) - tags: t.List[str] = field(default_factory=list) - help_text: t.Optional[str] = None - _db: t.Optional[Engine] = None - m2m_relationships: t.List[M2M] = field(default_factory=list) - schema: t.Optional[str] = None + json_columns: list[Union[JSON, JSONB]] = field(default_factory=list) + secret_columns: list[Secret] = field(default_factory=list) + auto_update_columns: list[Column] = field(default_factory=list) + tags: list[str] = field(default_factory=list) + help_text: Optional[str] = None + _db: Optional[Engine] = None + m2m_relationships: list[M2M] = field(default_factory=list) + schema: Optional[str] = None # Records reverse foreign key relationships - i.e. when the current table # is the target of a foreign key. Used by external libraries such as # Piccolo API. - _foreign_key_references: t.List[ForeignKey] = field(default_factory=list) + _foreign_key_references: list[ForeignKey] = field(default_factory=list) def get_formatted_tablename( self, include_schema: bool = True, quoted: bool = True @@ -120,8 +121,8 @@ def get_formatted_tablename( return ".".join(components) @property - def foreign_key_references(self) -> t.List[ForeignKey]: - foreign_keys: t.List[ForeignKey] = list(self._foreign_key_references) + def foreign_key_references(self) -> list[ForeignKey]: + foreign_keys: list[ForeignKey] = list(self._foreign_key_references) lazy_column_references = LAZY_COLUMN_REFERENCES.for_tablename( tablename=self.tablename ) @@ -173,11 +174,11 @@ def get_column_by_name(self, name: str) -> Column: return column_object - def get_auto_update_values(self) -> t.Dict[Column, t.Any]: + def get_auto_update_values(self) -> dict[Column, Any]: """ If columns have ``auto_update`` defined, then we retrieve these values. """ - output: t.Dict[Column, t.Any] = {} + output: dict[Column, Any] = {} for column in self.auto_update_columns: value = column._meta.auto_update if callable(value): @@ -203,7 +204,7 @@ def __repr__(cls): # `SessionsBase` is a `Table` subclass: def session_auth( - session_table: t.Type[SessionsBase] = SessionsBase + session_table: type[SessionsBase] = SessionsBase ): ... @@ -227,11 +228,11 @@ class Table(metaclass=TableMetaclass): def __init_subclass__( cls, - tablename: t.Optional[str] = None, - db: t.Optional[Engine] = None, - tags: t.Optional[t.List[str]] = None, - help_text: t.Optional[str] = None, - schema: t.Optional[str] = None, + tablename: Optional[str] = None, + db: Optional[Engine] = None, + tags: Optional[list[str]] = None, + help_text: Optional[str] = None, + schema: Optional[str] = None, ): # sourcery no-metrics """ Automatically populate the _meta, which includes the tablename, and @@ -268,17 +269,17 @@ def __init_subclass__( if tablename in PROTECTED_TABLENAMES: warnings.warn(TABLENAME_WARNING.format(tablename=tablename)) - columns: t.List[Column] = [] - default_columns: t.List[Column] = [] - non_default_columns: t.List[Column] = [] - array_columns: t.List[Array] = [] - foreign_key_columns: t.List[ForeignKey] = [] - secret_columns: t.List[Secret] = [] - json_columns: t.List[t.Union[JSON, JSONB]] = [] - email_columns: t.List[Email] = [] - auto_update_columns: t.List[Column] = [] - primary_key: t.Optional[Column] = None - m2m_relationships: t.List[M2M] = [] + columns: list[Column] = [] + default_columns: list[Column] = [] + non_default_columns: list[Column] = [] + array_columns: list[Array] = [] + foreign_key_columns: list[ForeignKey] = [] + secret_columns: list[Secret] = [] + json_columns: list[Union[JSON, JSONB]] = [] + email_columns: list[Email] = [] + auto_update_columns: list[Column] = [] + primary_key: Optional[Column] = None + m2m_relationships: list[M2M] = [] attribute_names = itertools.chain( *[i.__dict__.keys() for i in reversed(cls.__mro__)] @@ -372,7 +373,7 @@ def __init_subclass__( def __init__( self, - _data: t.Optional[t.Dict[Column, t.Any]] = None, + _data: Optional[dict[Column, Any]] = None, _ignore_missing: bool = False, _exists_in_db: bool = False, **kwargs, @@ -412,7 +413,7 @@ def __init__( # This is used by get_or_create to indicate to the user whether it # was an existing row or not. - self._was_created: t.Optional[bool] = None + self._was_created: Optional[bool] = None for column in self._meta.columns: value = _data.get(column, ...) @@ -423,7 +424,7 @@ def __init__( if value is ...: value = kwargs.pop( - t.cast(str, column._meta.db_column_name), ... + cast(str, column._meta.db_column_name), ... ) if value is ...: @@ -456,7 +457,7 @@ def _create_serial_primary_key(cls) -> Serial: @classmethod def from_dict( - cls: t.Type[TableInstance], data: t.Dict[str, t.Any] + cls: type[TableInstance], data: dict[str, Any] ) -> TableInstance: """ Used when loading fixtures. It can be overriden by subclasses in case @@ -468,8 +469,8 @@ def from_dict( ########################################################################### def save( - self, columns: t.Optional[t.Sequence[t.Union[Column, str]]] = None - ) -> t.Union[Insert, Update]: + self, columns: Optional[Sequence[Union[Column, str]]] = None + ) -> Union[Insert, Update]: """ A proxy to an insert or update query. @@ -504,7 +505,7 @@ def save( for i in columns ] - values: t.Dict[Column, t.Any] = { + values: dict[Column, Any] = { i: getattr(self, i._meta.name, None) for i in column_instances } @@ -525,9 +526,7 @@ def save( == getattr(self, self._meta.primary_key._meta.name) ) - def update_self( - self, values: t.Dict[t.Union[Column, str], t.Any] - ) -> UpdateSelf: + def update_self(self, values: dict[Union[Column, str], Any]) -> UpdateSelf: """ This allows the user to update a single object - useful when the values are derived from the database in some way. @@ -581,7 +580,7 @@ def remove(self) -> Delete: def refresh( self, - columns: t.Optional[t.Sequence[Column]] = None, + columns: Optional[Sequence[Column]] = None, load_json: bool = False, ) -> Refresh: """ @@ -611,16 +610,16 @@ def refresh( """ return Refresh(instance=self, columns=columns, load_json=load_json) - @t.overload + @overload def get_related( self, foreign_key: ForeignKey[ReferencedTable] ) -> GetRelated[ReferencedTable]: ... - @t.overload + @overload def get_related(self, foreign_key: str) -> GetRelated[Table]: ... def get_related( - self, foreign_key: t.Union[str, ForeignKey[ReferencedTable]] + self, foreign_key: Union[str, ForeignKey[ReferencedTable]] ) -> GetRelated[ReferencedTable]: """ Used to fetch a ``Table`` instance, for the target of a foreign key. @@ -666,7 +665,7 @@ def add_m2m( self, *rows: Table, m2m: M2M, - extra_column_values: t.Dict[t.Union[Column, str], t.Any] = {}, + extra_column_values: dict[Union[Column, str], Any] = {}, ) -> M2MAddRelated: """ Save the row if it doesn't already exist in the database, and insert @@ -733,7 +732,7 @@ def remove_m2m(self, *rows: Table, m2m: M2M) -> M2MRemoveRelated: m2m=m2m, ) - def to_dict(self, *columns: Column) -> t.Dict[str, t.Any]: + def to_dict(self, *columns: Column) -> dict[str, Any]: """ A convenience method which returns a dictionary, mapping column names to values for this table instance. @@ -785,7 +784,7 @@ def to_dict(self, *columns: Column) -> t.Dict[str, t.Any]: ) return output - def __setitem__(self, key: str, value: t.Any): + def __setitem__(self, key: str, value: Any): setattr(self, key, value) def __getitem__(self, key: str): @@ -858,8 +857,8 @@ def __repr__(self) -> str: @classmethod def all_related( - cls, exclude: t.Optional[t.List[t.Union[str, ForeignKey]]] = None - ) -> t.List[ForeignKey]: + cls, exclude: Optional[list[Union[str, ForeignKey]]] = None + ) -> list[ForeignKey]: """ Used in conjunction with ``objects`` queries. Just as we can use ``all_related`` on a ``ForeignKey``, you can also use it for the table @@ -908,8 +907,8 @@ def all_related( @classmethod def all_columns( - cls, exclude: t.Optional[t.Sequence[t.Union[str, Column]]] = None - ) -> t.List[Column]: + cls, exclude: Optional[Sequence[Union[str, Column]]] = None + ) -> list[Column]: """ Used in conjunction with ``select`` queries. Just as we can use ``all_columns`` to retrieve all of the columns from a related table, @@ -975,7 +974,7 @@ def ref(cls, column_name: str) -> Column: @classmethod def insert( - cls: t.Type[TableInstance], *rows: TableInstance + cls: type[TableInstance], *rows: TableInstance ) -> Insert[TableInstance]: """ Insert rows into the database. @@ -993,7 +992,7 @@ def insert( return query @classmethod - def raw(cls, sql: str, *args: t.Any) -> Raw: + def raw(cls, sql: str, *args: Any) -> Raw: """ Execute raw SQL queries on the underlying engine - use with caution! @@ -1012,8 +1011,8 @@ def raw(cls, sql: str, *args: t.Any) -> Raw: @classmethod def _process_column_args( - cls, *columns: t.Union[Selectable, str] - ) -> t.Sequence[Selectable]: + cls, *columns: Union[Selectable, str] + ) -> Sequence[Selectable]: """ Users can specify some column arguments as either Column instances, or as strings representing the column name, for convenience. @@ -1030,7 +1029,7 @@ def _process_column_args( @classmethod def select( - cls, *columns: t.Union[Selectable, str], exclude_secrets=False + cls, *columns: Union[Selectable, str], exclude_secrets=False ) -> Select: """ Get data in the form of a list of dictionaries, with each dictionary @@ -1109,8 +1108,8 @@ def alter(cls) -> Alter: @classmethod def objects( - cls: t.Type[TableInstance], - *prefetch: t.Union[ForeignKey, t.List[ForeignKey]], + cls: type[TableInstance], + *prefetch: Union[ForeignKey, list[ForeignKey]], ) -> Objects[TableInstance]: """ Returns a list of table instances (each representing a row), which you @@ -1151,8 +1150,8 @@ def objects( @classmethod def count( cls, - column: t.Optional[Column] = None, - distinct: t.Optional[t.Sequence[Column]] = None, + column: Optional[Column] = None, + distinct: Optional[Sequence[Column]] = None, ) -> Count: """ Count the number of matching rows:: @@ -1224,7 +1223,7 @@ def table_exists(cls) -> TableExists: @classmethod def update( cls, - values: t.Optional[t.Dict[t.Union[Column, str], t.Any]] = None, + values: Optional[dict[Union[Column, str], Any]] = None, force: bool = False, use_auto_update: bool = True, **kwargs, @@ -1288,7 +1287,7 @@ def indexes(cls) -> Indexes: @classmethod def create_index( cls, - columns: t.Union[t.List[Column], t.List[str]], + columns: Union[list[Column], list[str]], method: IndexMethod = IndexMethod.btree, if_not_exists: bool = False, ) -> CreateIndex: @@ -1311,7 +1310,7 @@ def create_index( @classmethod def drop_index( cls, - columns: t.Union[t.List[Column], t.List[str]], + columns: Union[list[Column], list[str]], if_exists: bool = True, ) -> DropIndex: """ @@ -1328,7 +1327,7 @@ def drop_index( ########################################################################### @classmethod - def _get_index_name(cls, column_names: t.List[str]) -> str: + def _get_index_name(cls, column_names: list[str]) -> str: """ Generates an index name from the table name and column names. """ @@ -1338,7 +1337,7 @@ def _get_index_name(cls, column_names: t.List[str]) -> str: @classmethod def _table_str( - cls, abbreviated=False, excluded_params: t.Optional[t.List[str]] = None + cls, abbreviated=False, excluded_params: Optional[list[str]] = None ): """ Returns a basic string representation of the table and its columns. @@ -1357,7 +1356,7 @@ def _table_str( spacer = "\n " columns = [] for col in cls._meta.columns: - params: t.List[str] = [] + params: list[str] = [] for key, value in col._meta.params.items(): if key in excluded_params: continue @@ -1392,10 +1391,10 @@ def _table_str( def create_table_class( class_name: str, - bases: t.Tuple[t.Type] = (Table,), - class_kwargs: t.Dict[str, t.Any] = {}, - class_members: t.Dict[str, t.Any] = {}, -) -> t.Type[Table]: + bases: tuple[type] = (Table,), + class_kwargs: dict[str, Any] = {}, + class_members: dict[str, Any] = {}, +) -> type[Table]: """ Used to dynamically create ``Table``subclasses at runtime. Most users will not require this. It's mostly used internally for Piccolo's @@ -1411,8 +1410,8 @@ def create_table_class( For example, `{'my_column': Varchar()}`. """ - return t.cast( - t.Type[Table], + return cast( + type[Table], types.new_class( name=class_name, bases=bases, @@ -1427,7 +1426,7 @@ def create_table_class( async def create_db_tables( - *tables: t.Type[Table], if_not_exists: bool = False + *tables: type[Table], if_not_exists: bool = False ) -> None: """ Creates the database table for each ``Table`` class passed in. The tables @@ -1458,7 +1457,7 @@ async def create_db_tables( def create_db_tables_sync( - *tables: t.Type[Table], if_not_exists: bool = False + *tables: type[Table], if_not_exists: bool = False ) -> None: """ A sync wrapper around :func:`create_db_tables`. @@ -1466,7 +1465,7 @@ def create_db_tables_sync( run_sync(create_db_tables(*tables, if_not_exists=if_not_exists)) -def create_tables(*tables: t.Type[Table], if_not_exists: bool = False) -> None: +def create_tables(*tables: type[Table], if_not_exists: bool = False) -> None: """ This original implementation has been replaced, because it was synchronous, and felt at odds with the rest of the Piccolo codebase which is async @@ -1485,7 +1484,7 @@ def create_tables(*tables: t.Type[Table], if_not_exists: bool = False) -> None: return create_db_tables_sync(*tables, if_not_exists=if_not_exists) -async def drop_db_tables(*tables: t.Type[Table]) -> None: +async def drop_db_tables(*tables: type[Table]) -> None: """ Drops the database table for each ``Table`` class passed in. The tables are dropped in the correct order, based on their foreign keys. @@ -1518,14 +1517,14 @@ async def drop_db_tables(*tables: t.Type[Table]) -> None: await atomic.run() -def drop_db_tables_sync(*tables: t.Type[Table]) -> None: +def drop_db_tables_sync(*tables: type[Table]) -> None: """ A sync wrapper around :func:`drop_db_tables`. """ run_sync(drop_db_tables(*tables)) -def drop_tables(*tables: t.Type[Table]) -> None: +def drop_tables(*tables: type[Table]) -> None: """ This original implementation has been replaced, because it was synchronous, and felt at odds with the rest of the Piccolo codebase which is async @@ -1548,8 +1547,8 @@ def drop_tables(*tables: t.Type[Table]) -> None: def sort_table_classes( - table_classes: t.List[t.Type[Table]], -) -> t.List[t.Type[Table]]: + table_classes: list[type[Table]], +) -> list[type[Table]]: """ Sort the table classes based on their foreign keys, so they can be created in the correct order. @@ -1564,7 +1563,7 @@ def sort_table_classes( sorter = TopologicalSorter(graph) ordered_tablenames = tuple(sorter.static_order()) - output: t.List[t.Type[Table]] = [] + output: list[type[Table]] = [] for tablename in ordered_tablenames: table_class = table_class_dict.get(tablename) if table_class is not None: @@ -1574,10 +1573,10 @@ def sort_table_classes( def _get_graph( - table_classes: t.List[t.Type[Table]], + table_classes: list[type[Table]], iterations: int = 0, max_iterations: int = 5, -) -> t.Dict[str, t.Set[str]]: +) -> dict[str, set[str]]: """ Analyses the tables based on their foreign keys, and returns a data structure like: @@ -1590,13 +1589,13 @@ def _get_graph( to it via a foreign key. """ - output: t.Dict[str, t.Set[str]] = {} + output: dict[str, set[str]] = {} if iterations >= max_iterations: return output for table_class in table_classes: - dependents: t.Set[str] = set() + dependents: set[str] = set() for fk in table_class._meta.foreign_key_columns: referenced_table = fk._foreign_key_meta.resolved_references diff --git a/piccolo/table_reflection.py b/piccolo/table_reflection.py index 3d0500990..52618761b 100644 --- a/piccolo/table_reflection.py +++ b/piccolo/table_reflection.py @@ -4,8 +4,8 @@ """ import asyncio -import typing as t from dataclasses import dataclass +from typing import Any, Optional, Union from piccolo.apps.schema.commands.generate import get_output_schema from piccolo.engine import engine_finder @@ -58,7 +58,7 @@ class Singleton(type): A metaclass that creates a Singleton base class when called. """ - _instances: t.Dict = {} + _instances: dict = {} def __call__(cls, *args, **kwargs): if cls not in cls._instances: @@ -80,7 +80,7 @@ class TableStorage(metaclass=Singleton): works with Postgres. """ - def __init__(self, engine: t.Optional[Engine] = None): + def __init__(self, engine: Optional[Engine] = None): """ :param engine: Which engine to use to make the database queries. If not specified, @@ -89,13 +89,13 @@ def __init__(self, engine: t.Optional[Engine] = None): """ self.engine = engine or engine_finder() self.tables = ImmutableDict() - self._schema_tables: t.Dict[str, t.List[str]] = {} + self._schema_tables: dict[str, list[str]] = {} async def reflect( self, schema_name: str = "public", - include: t.Union[t.List[str], str, None] = None, - exclude: t.Union[t.List[str], str, None] = None, + include: Union[list[str], str, None] = None, + exclude: Union[list[str], str, None] = None, keep_existing: bool = False, ) -> None: """ @@ -154,7 +154,7 @@ def clear(self) -> None: dict.clear(self.tables) self._schema_tables.clear() - async def get_table(self, tablename: str) -> t.Optional[t.Type[Table]]: + async def get_table(self, tablename: str) -> Optional[type[Table]]: """ Returns the ``Table`` class if it exists. If the table is not present in ``TableStorage``, it will try to reflect it. @@ -177,7 +177,7 @@ async def get_table(self, tablename: str) -> t.Optional[t.Type[Table]]: table_class = self.tables.get(tablename) return table_class - async def _add_table(self, schema_name: str, table: t.Type[Table]) -> None: + async def _add_table(self, schema_name: str, table: type[Table]) -> None: if issubclass(table, Table): table_name = self._get_table_name( table._meta.tablename, schema_name @@ -229,7 +229,7 @@ def _get_schema_and_table_name(tablename: str) -> TableNameDetail: raise ValueError("Couldn't find schema name.") @staticmethod - def _to_list(value: t.Any) -> t.List: + def _to_list(value: Any) -> list: if isinstance(value, list): return value elif isinstance(value, (tuple, set)): diff --git a/piccolo/testing/model_builder.py b/piccolo/testing/model_builder.py index 87e1c87fb..e45dc1dbe 100644 --- a/piccolo/testing/model_builder.py +++ b/piccolo/testing/model_builder.py @@ -2,8 +2,9 @@ import datetime import json -import typing as t +from collections.abc import Callable from decimal import Decimal +from typing import Any, Optional, Union, cast from uuid import UUID from piccolo.columns import JSON, JSONB, Array, Column, ForeignKey @@ -13,7 +14,7 @@ class ModelBuilder: - __DEFAULT_MAPPER: t.Dict[t.Type, t.Callable] = { + __DEFAULT_MAPPER: dict[type, Callable] = { bool: RandomBuilder.next_bool, bytes: RandomBuilder.next_bytes, datetime.date: RandomBuilder.next_date, @@ -29,8 +30,8 @@ class ModelBuilder: @classmethod async def build( cls, - table_class: t.Type[TableInstance], - defaults: t.Optional[t.Dict[t.Union[Column, str], t.Any]] = None, + table_class: type[TableInstance], + defaults: Optional[dict[Union[Column, str], Any]] = None, persist: bool = True, minimal: bool = False, ) -> TableInstance: @@ -80,8 +81,8 @@ async def build( @classmethod def build_sync( cls, - table_class: t.Type[TableInstance], - defaults: t.Optional[t.Dict[t.Union[Column, str], t.Any]] = None, + table_class: type[TableInstance], + defaults: Optional[dict[Union[Column, str], Any]] = None, persist: bool = True, minimal: bool = False, ) -> TableInstance: @@ -100,8 +101,8 @@ def build_sync( @classmethod async def _build( cls, - table_class: t.Type[TableInstance], - defaults: t.Optional[t.Dict[t.Union[Column, str], t.Any]] = None, + table_class: type[TableInstance], + defaults: Optional[dict[Union[Column, str], Any]] = None, minimal: bool = False, persist: bool = True, ) -> TableInstance: @@ -151,7 +152,7 @@ async def _build( return model @classmethod - def _randomize_attribute(cls, column: Column) -> t.Any: + def _randomize_attribute(cls, column: Column) -> Any: """ Generate a random value for a column and apply formatting. @@ -159,7 +160,7 @@ def _randomize_attribute(cls, column: Column) -> t.Any: Column class to randomize. """ - random_value: t.Any + random_value: Any if column.value_type == Decimal: precision, scale = column._meta.params["digits"] or (4, 2) random_value = RandomBuilder.next_float( @@ -170,7 +171,7 @@ def _randomize_attribute(cls, column: Column) -> t.Any: random_value = RandomBuilder.next_datetime(tz_aware=tz_aware) elif column.value_type == list: length = RandomBuilder.next_int(maximum=10) - base_type = t.cast(Array, column).base_column.value_type + base_type = cast(Array, column).base_column.value_type random_value = [ cls.__DEFAULT_MAPPER[base_type]() for _ in range(length) ] diff --git a/piccolo/testing/random_builder.py b/piccolo/testing/random_builder.py index bca29a7f2..684cd1d9f 100644 --- a/piccolo/testing/random_builder.py +++ b/piccolo/testing/random_builder.py @@ -2,8 +2,8 @@ import enum import random import string -import typing as t import uuid +from typing import Any class RandomBuilder: @@ -36,7 +36,7 @@ def next_datetime(cls, tz_aware: bool = False) -> datetime.datetime: ) @classmethod - def next_enum(cls, e: t.Type[enum.Enum]) -> t.Any: + def next_enum(cls, e: type[enum.Enum]) -> Any: return random.choice([item.value for item in e]) @classmethod diff --git a/piccolo/testing/test_case.py b/piccolo/testing/test_case.py index ba01a9cdc..08bd61a5c 100644 --- a/piccolo/testing/test_case.py +++ b/piccolo/testing/test_case.py @@ -1,6 +1,6 @@ from __future__ import annotations -import typing as t +from typing import Optional from unittest import IsolatedAsyncioTestCase, TestCase from piccolo.engine import Engine, engine_finder @@ -30,7 +30,7 @@ def test_band(self): """ # noqa: E501 - tables: t.List[t.Type[Table]] + tables: list[type[Table]] def setUp(self) -> None: create_db_tables_sync(*self.tables) @@ -54,7 +54,7 @@ async def test_band(self): """ - tables: t.List[t.Type[Table]] + tables: list[type[Table]] async def asyncSetUp(self) -> None: await create_db_tables(*self.tables) @@ -106,7 +106,7 @@ async def test_band_response(self): # # ... # - db: t.Optional[Engine] = None + db: Optional[Engine] = None async def asyncSetUp(self) -> None: db = self.db or engine_finder() diff --git a/piccolo/utils/dictionary.py b/piccolo/utils/dictionary.py index 82fa2368d..880950711 100644 --- a/piccolo/utils/dictionary.py +++ b/piccolo/utils/dictionary.py @@ -1,9 +1,9 @@ from __future__ import annotations -import typing as t +from typing import Any -def make_nested(dictionary: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]: +def make_nested(dictionary: dict[str, Any]) -> dict[str, Any]: """ Rows are returned from the database as a flat dictionary, with keys such as ``'manager.name'`` if the column belongs to a related table. @@ -20,7 +20,7 @@ def make_nested(dictionary: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]: {'name': 'Pythonistas', 'band': {'name': 'Guido'}} """ - output: t.Dict[str, t.Any] = {} + output: dict[str, Any] = {} items = list(dictionary.items()) items.sort(key=lambda x: x[0]) diff --git a/piccolo/utils/encoding.py b/piccolo/utils/encoding.py index 029ae37a3..3c637079e 100644 --- a/piccolo/utils/encoding.py +++ b/piccolo/utils/encoding.py @@ -1,6 +1,6 @@ from __future__ import annotations -import typing as t +from typing import Any try: import orjson @@ -12,9 +12,9 @@ ORJSON = False -def dump_json(data: t.Any, pretty: bool = False) -> str: +def dump_json(data: Any, pretty: bool = False) -> str: if ORJSON: - orjson_params: t.Dict[str, t.Any] = {"default": str} + orjson_params: dict[str, Any] = {"default": str} if pretty: orjson_params["option"] = ( orjson.OPT_INDENT_2 | orjson.OPT_APPEND_NEWLINE # type: ignore @@ -23,7 +23,7 @@ def dump_json(data: t.Any, pretty: bool = False) -> str: "utf8" ) else: - params: t.Dict[str, t.Any] = {"default": str} + params: dict[str, Any] = {"default": str} if pretty: params["indent"] = 2 return json.dumps(data, **params) # type: ignore @@ -63,7 +63,7 @@ class JSONDict(dict): ... -def load_json(data: str) -> t.Any: +def load_json(data: str) -> Any: response = ( orjson.loads(data) if ORJSON else json.loads(data) # type: ignore ) diff --git a/piccolo/utils/lazy_loader.py b/piccolo/utils/lazy_loader.py index 5b7211ce3..7b64a896f 100644 --- a/piccolo/utils/lazy_loader.py +++ b/piccolo/utils/lazy_loader.py @@ -3,7 +3,7 @@ import importlib import types -import typing as t +from typing import Any class LazyLoader(types.ModuleType): @@ -48,10 +48,10 @@ def _load(self) -> types.ModuleType: else: raise exc from exc - def __getattr__(self, item) -> t.Any: + def __getattr__(self, item) -> Any: module = self._load() return getattr(module, item) - def __dir__(self) -> t.List[str]: + def __dir__(self) -> list[str]: module = self._load() return dir(module) diff --git a/piccolo/utils/list.py b/piccolo/utils/list.py index dbd61b289..8ec6aa066 100644 --- a/piccolo/utils/list.py +++ b/piccolo/utils/list.py @@ -1,11 +1,12 @@ -import typing as t +from collections.abc import Sequence +from typing import TypeVar, Union -ElementType = t.TypeVar("ElementType") +ElementType = TypeVar("ElementType") def flatten( - items: t.Sequence[t.Union[ElementType, t.List[ElementType]]] -) -> t.List[ElementType]: + items: Sequence[Union[ElementType, list[ElementType]]] +) -> list[ElementType]: """ Takes a sequence of elements, and flattens it out. For example:: @@ -17,7 +18,7 @@ def flatten( await Band.select(Band.name, Band.manager.all_columns()) """ - _items: t.List[ElementType] = [] + _items: list[ElementType] = [] for item in items: if isinstance(item, list): _items.extend(item) @@ -27,9 +28,7 @@ def flatten( return _items -def batch( - data: t.List[ElementType], chunk_size: int -) -> t.List[t.List[ElementType]]: +def batch(data: list[ElementType], chunk_size: int) -> list[list[ElementType]]: """ Breaks the list down into sublists of the given ``chunk_size``. The last sublist may have fewer elements than ``chunk_size``:: diff --git a/piccolo/utils/objects.py b/piccolo/utils/objects.py index 24e6e3e32..22b48a530 100644 --- a/piccolo/utils/objects.py +++ b/piccolo/utils/objects.py @@ -1,16 +1,14 @@ from __future__ import annotations -import typing as t +from typing import TYPE_CHECKING, Any from piccolo.columns.column_types import ForeignKey -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from piccolo.table import Table -def make_nested_object( - row: t.Dict[str, t.Any], table_class: t.Type[Table] -) -> Table: +def make_nested_object(row: dict[str, Any], table_class: type[Table]) -> Table: """ Takes a nested dictionary such as this: @@ -38,7 +36,7 @@ def make_nested_object( 1 """ - table_params: t.Dict[str, t.Any] = {} + table_params: dict[str, Any] = {} for key, value in row.items(): if isinstance(value, dict): diff --git a/piccolo/utils/pydantic.py b/piccolo/utils/pydantic.py index 3c88c8764..794406f17 100644 --- a/piccolo/utils/pydantic.py +++ b/piccolo/utils/pydantic.py @@ -2,9 +2,10 @@ import itertools import json -import typing as t from collections import defaultdict +from collections.abc import Callable from functools import partial +from typing import Any, Optional, Union import pydantic @@ -30,7 +31,7 @@ JsonDict = dict # type: ignore -def pydantic_json_validator(value: t.Optional[str], required: bool = True): +def pydantic_json_validator(value: Optional[str], required: bool = True): if value is None: if required: raise ValueError("The JSON value wasn't provided.") @@ -45,7 +46,7 @@ def pydantic_json_validator(value: t.Optional[str], required: bool = True): return value -def is_table_column(column: Column, table: t.Type[Table]) -> bool: +def is_table_column(column: Column, table: type[Table]) -> bool: """ Verify that the given ``Column`` belongs to the given ``Table``. """ @@ -60,9 +61,7 @@ def is_table_column(column: Column, table: t.Type[Table]) -> bool: return False -def validate_columns( - columns: t.Tuple[Column, ...], table: t.Type[Table] -) -> bool: +def validate_columns(columns: tuple[Column, ...], table: type[Table]) -> bool: """ Verify that each column is a ``Column``` instance, and its parent is the given ``Table``. @@ -74,9 +73,7 @@ def validate_columns( ) -def get_array_value_type( - column: Array, inner: t.Optional[t.Type] = None -) -> t.Type: +def get_array_value_type(column: Array, inner: Optional[type] = None) -> type: """ Gets the correct type for an ``Array`` column (which might be multidimensional). @@ -86,14 +83,14 @@ def get_array_value_type( else: inner_type = get_pydantic_value_type(column.base_column) - return t.List[inner_type] # type: ignore + return list[inner_type] # type: ignore -def get_pydantic_value_type(column: Column) -> t.Type: +def get_pydantic_value_type(column: Column) -> type: """ Map the Piccolo ``Column`` to a Pydantic type. """ - value_type: t.Type + value_type: type if isinstance(column, (Decimal, Numeric)): value_type = pydantic.condecimal( @@ -112,20 +109,20 @@ def get_pydantic_value_type(column: Column) -> t.Type: def create_pydantic_model( - table: t.Type[Table], - nested: t.Union[bool, t.Tuple[ForeignKey, ...]] = False, - exclude_columns: t.Tuple[Column, ...] = (), - include_columns: t.Tuple[Column, ...] = (), + table: type[Table], + nested: Union[bool, tuple[ForeignKey, ...]] = False, + exclude_columns: tuple[Column, ...] = (), + include_columns: tuple[Column, ...] = (), include_default_columns: bool = False, include_readable: bool = False, all_optional: bool = False, - model_name: t.Optional[str] = None, + model_name: Optional[str] = None, deserialize_json: bool = False, recursion_depth: int = 0, max_recursion_depth: int = 5, - pydantic_config: t.Optional[pydantic.config.ConfigDict] = None, - json_schema_extra: t.Optional[t.Dict[str, t.Any]] = None, -) -> t.Type[pydantic.BaseModel]: + pydantic_config: Optional[pydantic.config.ConfigDict] = None, + json_schema_extra: Optional[dict[str, Any]] = None, +) -> type[pydantic.BaseModel]: """ Create a Pydantic model representing a table. @@ -205,8 +202,8 @@ def create_pydantic_model( ########################################################################### - columns: t.Dict[str, t.Any] = {} - validators: t.Dict[str, t.Callable] = {} + columns: dict[str, Any] = {} + validators: dict[str, Callable] = {} piccolo_columns = tuple( table._meta.columns @@ -264,11 +261,11 @@ def create_pydantic_model( else: value_type = get_pydantic_value_type(column=column) - _type = t.Optional[value_type] if is_optional else value_type + _type = Optional[value_type] if is_optional else value_type ####################################################################### - params: t.Dict[str, t.Any] = {} + params: dict[str, Any] = {} if is_optional: params["default"] = None diff --git a/piccolo/utils/sql_values.py b/piccolo/utils/sql_values.py index cc372e9e3..4d44c96f5 100644 --- a/piccolo/utils/sql_values.py +++ b/piccolo/utils/sql_values.py @@ -1,17 +1,17 @@ from __future__ import annotations import functools -import typing as t from enum import Enum +from typing import TYPE_CHECKING, Any from piccolo.utils.encoding import dump_json from piccolo.utils.warnings import colored_warning -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from piccolo.columns import Column -def convert_to_sql_value(value: t.Any, column: Column) -> t.Any: +def convert_to_sql_value(value: Any, column: Column) -> Any: """ Some values which can be passed into Piccolo queries aren't valid in the database. For example, Enums, Table instances, and dictionaries for JSON diff --git a/piccolo/utils/sync.py b/piccolo/utils/sync.py index 348829c9e..62aea2a38 100644 --- a/piccolo/utils/sync.py +++ b/piccolo/utils/sync.py @@ -1,14 +1,15 @@ from __future__ import annotations import asyncio -import typing as t +from collections.abc import Coroutine from concurrent.futures import Future, ThreadPoolExecutor +from typing import Any, TypeVar -ReturnType = t.TypeVar("ReturnType") +ReturnType = TypeVar("ReturnType") def run_sync( - coroutine: t.Coroutine[t.Any, t.Any, ReturnType], + coroutine: Coroutine[Any, Any, ReturnType], ) -> ReturnType: """ Run the coroutine synchronously - trying to accommodate as many edge cases diff --git a/piccolo/utils/warnings.py b/piccolo/utils/warnings.py index ef1fcba90..f5f36852f 100644 --- a/piccolo/utils/warnings.py +++ b/piccolo/utils/warnings.py @@ -1,6 +1,5 @@ from __future__ import annotations -import typing as t import warnings from enum import Enum @@ -21,7 +20,7 @@ def colored_string(message: str, level: Level = Level.medium) -> str: def colored_warning( message: str, - category: t.Type[Warning] = Warning, + category: type[Warning] = Warning, stacklevel: int = 3, level: Level = Level.medium, ): diff --git a/setup.py b/setup.py index 315dc5e6e..7ec55f967 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,6 @@ import itertools import os -import typing as t from setuptools import find_packages, setup @@ -18,7 +17,7 @@ LONG_DESCRIPTION = f.read() -def parse_requirement(req_path: str) -> t.List[str]: +def parse_requirement(req_path: str) -> list[str]: """ Parse requirement file. Example: @@ -32,7 +31,7 @@ def parse_requirement(req_path: str) -> t.List[str]: return [i.strip() for i in contents.strip().split("\n")] -def extras_require() -> t.Dict[str, t.List[str]]: +def extras_require() -> dict[str, list[str]]: """ Parse requirements in requirements/extras directory """ diff --git a/tests/apps/asgi/commands/files/dummy_server.py b/tests/apps/asgi/commands/files/dummy_server.py index 307fe1c80..709a98a77 100644 --- a/tests/apps/asgi/commands/files/dummy_server.py +++ b/tests/apps/asgi/commands/files/dummy_server.py @@ -1,13 +1,14 @@ import asyncio import importlib import sys -import typing as t +from collections.abc import Callable +from typing import Union, cast from httpx import ASGITransport, AsyncClient from uvicorn import Config, Server -async def dummy_server(app: t.Union[str, t.Callable] = "app:app") -> None: +async def dummy_server(app: Union[str, Callable] = "app:app") -> None: """ A very simplistic ASGI server. It's used to run the generated ASGI applications in unit tests. @@ -23,7 +24,7 @@ async def dummy_server(app: t.Union[str, t.Callable] = "app:app") -> None: if isinstance(app, str): path, app_name = app.rsplit(":") module = importlib.import_module(path) - app = t.cast(t.Callable, getattr(module, app_name)) + app = cast(Callable, getattr(module, app_name)) try: async with AsyncClient(transport=ASGITransport(app=app)) as client: diff --git a/tests/apps/fixtures/commands/test_dump_load.py b/tests/apps/fixtures/commands/test_dump_load.py index 59c4d04a2..728f2f5c0 100644 --- a/tests/apps/fixtures/commands/test_dump_load.py +++ b/tests/apps/fixtures/commands/test_dump_load.py @@ -2,7 +2,6 @@ import decimal import os import tempfile -import typing as t import uuid from unittest import TestCase @@ -65,7 +64,7 @@ def insert_rows(self): ) mega_table.save().run_sync() - def _run_comparison(self, table_class_names: t.List[str]): + def _run_comparison(self, table_class_names: list[str]): self.insert_rows() json_string = run_sync( diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index 0ba2901fd..77cf95edb 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -7,8 +7,9 @@ import shutil import tempfile import time -import typing as t import uuid +from collections.abc import Callable +from typing import TYPE_CHECKING, Optional from unittest.mock import MagicMock, patch from piccolo.apps.migrations.auto.operations import RenameTable @@ -55,7 +56,7 @@ from piccolo.utils.sync import run_sync from tests.base import DBTestCase, engines_only, engines_skip -if t.TYPE_CHECKING: +if TYPE_CHECKING: from piccolo.columns.base import Column @@ -127,8 +128,8 @@ def _get_app_config(self) -> AppConfig: def _test_migrations( self, - table_snapshots: t.List[t.List[t.Type[Table]]], - test_function: t.Optional[t.Callable[[RowMeta], bool]] = None, + table_snapshots: list[list[type[Table]]], + test_function: Optional[Callable[[RowMeta], bool]] = None, ): """ Writes a migration file to disk and runs it. @@ -1088,7 +1089,7 @@ class TableA(Table): id = UUID(primary_key=True) table_a: ForeignKey[TableA] = ForeignKey("self") - self.table_classes: t.List[t.Type[Table]] = [TableA] + self.table_classes: list[type[Table]] = [TableA] def tearDown(self): drop_db_tables_sync(Migration, *self.table_classes) diff --git a/tests/apps/migrations/auto/test_migration_manager.py b/tests/apps/migrations/auto/test_migration_manager.py index e4d1a1d41..0952e1895 100644 --- a/tests/apps/migrations/auto/test_migration_manager.py +++ b/tests/apps/migrations/auto/test_migration_manager.py @@ -1,7 +1,7 @@ import asyncio import random -import typing as t from io import StringIO +from typing import Optional from unittest import IsolatedAsyncioTestCase, TestCase from unittest.mock import MagicMock, patch @@ -307,7 +307,7 @@ def test_add_column(self) -> None: response = self.run_sync("SELECT * FROM manager;") self.assertEqual(response, [{"id": 1, "name": "Dave"}]) - row_id: t.Optional[int] = None + row_id: Optional[int] = None if engine_is("cockroach"): row_id = self.run_sync( "INSERT INTO manager VALUES (default, 'Dave', 'dave@me.com') RETURNING id;" # noqa: E501 diff --git a/tests/apps/migrations/auto/test_schema_differ.py b/tests/apps/migrations/auto/test_schema_differ.py index 9cf6d26f2..35dce6adc 100644 --- a/tests/apps/migrations/auto/test_schema_differ.py +++ b/tests/apps/migrations/auto/test_schema_differ.py @@ -1,6 +1,5 @@ from __future__ import annotations -import typing as t from unittest import TestCase from unittest.mock import MagicMock, call, patch @@ -24,12 +23,12 @@ def test_add_table(self) -> None: """ name_column = Varchar() name_column._meta.name = "name" - schema: t.List[DiffableTable] = [ + schema: list[DiffableTable] = [ DiffableTable( class_name="Band", tablename="band", columns=[name_column] ) ] - schema_snapshot: t.List[DiffableTable] = [] + schema_snapshot: list[DiffableTable] = [] schema_differ = SchemaDiffer( schema=schema, schema_snapshot=schema_snapshot, auto_input="y" ) @@ -52,8 +51,8 @@ def test_drop_table(self) -> None: """ Test dropping an existing table. """ - schema: t.List[DiffableTable] = [] - schema_snapshot: t.List[DiffableTable] = [ + schema: list[DiffableTable] = [] + schema_snapshot: list[DiffableTable] = [ DiffableTable(class_name="Band", tablename="band", columns=[]) ] schema_differ = SchemaDiffer( @@ -73,12 +72,12 @@ def test_rename_table(self) -> None: name_column = Varchar() name_column._meta.name = "name" - schema: t.List[DiffableTable] = [ + schema: list[DiffableTable] = [ DiffableTable( class_name="Act", tablename="act", columns=[name_column] ) ] - schema_snapshot: t.List[DiffableTable] = [ + schema_snapshot: list[DiffableTable] = [ DiffableTable( class_name="Band", tablename="band", columns=[name_column] ) @@ -101,7 +100,7 @@ def test_change_schema(self) -> None: """ Testing changing the schema. """ - schema: t.List[DiffableTable] = [ + schema: list[DiffableTable] = [ DiffableTable( class_name="Band", tablename="band", @@ -109,7 +108,7 @@ def test_change_schema(self) -> None: schema="schema_1", ) ] - schema_snapshot: t.List[DiffableTable] = [ + schema_snapshot: list[DiffableTable] = [ DiffableTable( class_name="Band", tablename="band", @@ -142,14 +141,14 @@ def test_add_column(self) -> None: genre_column = Varchar() genre_column._meta.name = "genre" - schema: t.List[DiffableTable] = [ + schema: list[DiffableTable] = [ DiffableTable( class_name="Band", tablename="band", columns=[name_column, genre_column], ) ] - schema_snapshot: t.List[DiffableTable] = [ + schema_snapshot: list[DiffableTable] = [ DiffableTable( class_name="Band", tablename="band", @@ -177,14 +176,14 @@ def test_drop_column(self) -> None: genre_column = Varchar() genre_column._meta.name = "genre" - schema: t.List[DiffableTable] = [ + schema: list[DiffableTable] = [ DiffableTable( class_name="Band", tablename="band", columns=[name_column], ) ] - schema_snapshot: t.List[DiffableTable] = [ + schema_snapshot: list[DiffableTable] = [ DiffableTable( class_name="Band", tablename="band", @@ -213,14 +212,14 @@ def test_rename_column(self) -> None: title_column = Varchar() title_column._meta.name = "title" - schema_snapshot: t.List[DiffableTable] = [ + schema_snapshot: list[DiffableTable] = [ DiffableTable( class_name="Band", tablename="band", columns=[title_column], ) ] - schema: t.List[DiffableTable] = [ + schema: list[DiffableTable] = [ DiffableTable( class_name="Band", tablename="band", @@ -278,14 +277,14 @@ def test_rename_multiple_columns(self, input: MagicMock) -> None: b2 = Varchar() b2._meta.name = "b2" - schema_snapshot: t.List[DiffableTable] = [ + schema_snapshot: list[DiffableTable] = [ DiffableTable( class_name="Band", tablename="band", columns=[a1, b1], ) ] - schema: t.List[DiffableTable] = [ + schema: list[DiffableTable] = [ DiffableTable( class_name="Band", tablename="band", @@ -355,14 +354,14 @@ def test_rename_some_columns(self, input: MagicMock): b2 = Varchar() b2._meta.name = "b2" - schema_snapshot: t.List[DiffableTable] = [ + schema_snapshot: list[DiffableTable] = [ DiffableTable( class_name="Band", tablename="band", columns=[a1, b1], ) ] - schema: t.List[DiffableTable] = [ + schema: list[DiffableTable] = [ DiffableTable( class_name="Band", tablename="band", @@ -425,14 +424,14 @@ def test_alter_column_precision(self) -> None: price_2 = Numeric(digits=(5, 2)) price_2._meta.name = "price" - schema: t.List[DiffableTable] = [ + schema: list[DiffableTable] = [ DiffableTable( class_name="Ticket", tablename="ticket", columns=[price_1], ) ] - schema_snapshot: t.List[DiffableTable] = [ + schema_snapshot: list[DiffableTable] = [ DiffableTable( class_name="Ticket", tablename="ticket", @@ -463,14 +462,14 @@ def test_db_column_name(self) -> None: price_2 = Numeric(digits=(5, 2), db_column_name="custom") price_2._meta.name = "price" - schema: t.List[DiffableTable] = [ + schema: list[DiffableTable] = [ DiffableTable( class_name="Ticket", tablename="ticket", columns=[price_1], ) ] - schema_snapshot: t.List[DiffableTable] = [ + schema_snapshot: list[DiffableTable] = [ DiffableTable( class_name="Ticket", tablename="ticket", diff --git a/tests/apps/migrations/commands/test_forwards_backwards.py b/tests/apps/migrations/commands/test_forwards_backwards.py index e88747eda..1ccc5bce7 100644 --- a/tests/apps/migrations/commands/test_forwards_backwards.py +++ b/tests/apps/migrations/commands/test_forwards_backwards.py @@ -1,7 +1,7 @@ from __future__ import annotations import sys -import typing as t +from typing import TYPE_CHECKING from unittest import TestCase from unittest.mock import MagicMock, call, patch @@ -22,10 +22,10 @@ Venue, ) -if t.TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from piccolo.table import Table -TABLE_CLASSES: t.List[t.Type[Table]] = [ +TABLE_CLASSES: list[type[Table]] = [ Manager, Band, Venue, diff --git a/tests/apps/schema/commands/test_generate.py b/tests/apps/schema/commands/test_generate.py index 29784fa8a..5168ab060 100644 --- a/tests/apps/schema/commands/test_generate.py +++ b/tests/apps/schema/commands/test_generate.py @@ -2,7 +2,7 @@ import ast import asyncio -import typing as t +from typing import cast from unittest import TestCase from unittest.mock import MagicMock, patch @@ -38,7 +38,7 @@ def tearDown(self): table_class.alter().drop_table().run_sync() def _compare_table_columns( - self, table_1: t.Type[Table], table_2: t.Type[Table] + self, table_1: type[Table], table_2: type[Table] ): """ Make sure that for each column in table_1, there is a corresponding @@ -166,7 +166,7 @@ def test_self_referencing_fk(self) -> None: # Make sure the 'references' value of the generated column is "self". for table in output_schema.tables: if table.__name__ == "MegaTable": - column = t.cast( + column = cast( ForeignKey, output_schema.tables[1]._meta.get_column_by_name( "self_referencing_fk" diff --git a/tests/base.py b/tests/base.py index f9f964c70..4651a4a04 100644 --- a/tests/base.py +++ b/tests/base.py @@ -2,7 +2,7 @@ import asyncio import sys -import typing as t +from typing import Optional from unittest import TestCase from unittest.mock import MagicMock @@ -169,7 +169,7 @@ def run_sync(self, query): return _Table.raw(query).run_sync() def table_exists(self, tablename: str) -> bool: - _Table: t.Type[Table] = create_table_class( + _Table: type[Table] = create_table_class( class_name=tablename.upper(), class_kwargs={"tablename": tablename} ) return _Table.table_exists().run_sync() @@ -222,7 +222,7 @@ def get_postgres_is_nullable(self, tablename, column_name: str) -> bool: def get_postgres_varchar_length( self, tablename, column_name: str - ) -> t.Optional[int]: + ) -> Optional[int]: """ Fetches whether the column is defined as nullable, from the database. """ @@ -466,7 +466,7 @@ class TableTest(TestCase): Used for tests where we need to create Piccolo tables. """ - tables: t.List[t.Type[Table]] + tables: list[type[Table]] def setUp(self) -> None: create_db_tables_sync(*self.tables) diff --git a/tests/columns/m2m/base.py b/tests/columns/m2m/base.py index 5d1c16436..8e6e477aa 100644 --- a/tests/columns/m2m/base.py +++ b/tests/columns/m2m/base.py @@ -1,4 +1,4 @@ -import typing as t +from typing import Optional from piccolo.columns.column_types import ( ForeignKey, @@ -41,7 +41,7 @@ class M2MBase: (public vs non-public). """ - def _setUp(self, schema: t.Optional[str] = None): + def _setUp(self, schema: Optional[str] = None): self.schema = schema for table_class in (Band, Genre, GenreToBand): diff --git a/tests/columns/test_boolean.py b/tests/columns/test_boolean.py index 1e1efc5c2..2cba15767 100644 --- a/tests/columns/test_boolean.py +++ b/tests/columns/test_boolean.py @@ -1,4 +1,4 @@ -import typing as t +from typing import Any from piccolo.columns.column_types import Boolean from piccolo.table import Table @@ -14,9 +14,7 @@ class TestBoolean(TableTest): def test_return_type(self) -> None: for value in (True, False, None, ...): - kwargs: t.Dict[str, t.Any] = ( - {} if value is ... else {"boolean": value} - ) + kwargs: dict[str, Any] = {} if value is ... else {"boolean": value} expected = MyTable.boolean.default if value is ... else value row = MyTable(**kwargs) diff --git a/tests/columns/test_db_column_name.py b/tests/columns/test_db_column_name.py index 96a182149..1e4195d4e 100644 --- a/tests/columns/test_db_column_name.py +++ b/tests/columns/test_db_column_name.py @@ -1,4 +1,4 @@ -import typing as t +from typing import Optional from piccolo.columns.column_types import ForeignKey, Integer, Serial, Varchar from piccolo.table import Table, create_db_tables_sync, drop_db_tables_sync @@ -35,7 +35,7 @@ def setUp(self): def tearDown(self): drop_db_tables_sync(Band, Manager) - def insert_band(self, manager: t.Optional[Manager] = None) -> Band: + def insert_band(self, manager: Optional[Manager] = None) -> Band: band = Band(name="Pythonistas", popularity=1000, manager=manager) band.save().run_sync() return band diff --git a/tests/engine/test_extra_nodes.py b/tests/engine/test_extra_nodes.py index 0d59868c2..42a2703f9 100644 --- a/tests/engine/test_extra_nodes.py +++ b/tests/engine/test_extra_nodes.py @@ -1,4 +1,4 @@ -import typing as t +from typing import cast from unittest import TestCase from unittest.mock import MagicMock @@ -19,7 +19,7 @@ def test_extra_nodes(self): test_engine = engine_finder() assert test_engine is not None - test_engine = t.cast(PostgresEngine, test_engine) + test_engine = cast(PostgresEngine, test_engine) EXTRA_NODE = MagicMock(spec=PostgresEngine(config=test_engine.config)) EXTRA_NODE.run_querystring = AsyncMock(return_value=[]) diff --git a/tests/engine/test_pool.py b/tests/engine/test_pool.py index 510a77072..28f2db1c3 100644 --- a/tests/engine/test_pool.py +++ b/tests/engine/test_pool.py @@ -1,7 +1,7 @@ import asyncio import os import tempfile -import typing as t +from typing import cast from unittest import TestCase from unittest.mock import call, patch @@ -14,7 +14,7 @@ @engines_only("postgres", "cockroach") class TestPool(DBTestCase): async def _create_pool(self) -> None: - engine = t.cast(PostgresEngine, Manager._meta.db) + engine = cast(PostgresEngine, Manager._meta.db) await engine.start_connection_pool() assert engine.pool is not None @@ -72,7 +72,7 @@ def test_many_queries(self): @engines_only("postgres", "cockroach") class TestPoolProxyMethods(DBTestCase): async def _create_pool(self) -> None: - engine = t.cast(PostgresEngine, Manager._meta.db) + engine = cast(PostgresEngine, Manager._meta.db) # Deliberate typo ('nnn'): await engine.start_connnection_pool() diff --git a/tests/engine/test_transaction.py b/tests/engine/test_transaction.py index 88e4cff15..d381f5d14 100644 --- a/tests/engine/test_transaction.py +++ b/tests/engine/test_transaction.py @@ -1,5 +1,5 @@ import asyncio -import typing as t +from typing import cast from unittest import TestCase import pytest @@ -169,7 +169,7 @@ def test_exists(self): """ Make sure we can detect when code is within a transaction. """ - engine = t.cast(SQLiteEngine, Manager._meta.db) + engine = cast(SQLiteEngine, Manager._meta.db) async def run_inside_transaction(): async with engine.transaction(): @@ -198,7 +198,7 @@ def test_transaction(self): https://github.com/piccolo-orm/piccolo/issues/687 """ - engine = t.cast(SQLiteEngine, Manager._meta.db) + engine = cast(SQLiteEngine, Manager._meta.db) async def run_transaction(name: str): async with engine.transaction( @@ -234,7 +234,7 @@ def test_atomic(self): """ Similar to above, but with ``Atomic``. """ - engine = t.cast(SQLiteEngine, Manager._meta.db) + engine = cast(SQLiteEngine, Manager._meta.db) async def run_atomic(name: str): atomic = engine.atomic(transaction_type=TransactionType.immediate) diff --git a/tests/query/test_freeze.py b/tests/query/test_freeze.py index ca916ba5e..61a54e761 100644 --- a/tests/query/test_freeze.py +++ b/tests/query/test_freeze.py @@ -1,6 +1,6 @@ import timeit -import typing as t from dataclasses import dataclass +from typing import Any, Union from unittest import mock from piccolo.columns import Integer, Varchar @@ -12,8 +12,8 @@ @dataclass class QueryResponse: - query: t.Union[Query, FrozenQuery] - response: t.Any + query: Union[Query, FrozenQuery] + response: Any class TestFreeze(DBTestCase): @@ -23,7 +23,7 @@ def test_frozen_select_queries(self) -> None: """ self.insert_rows() - query_responses: t.List[QueryResponse] = [ + query_responses: list[QueryResponse] = [ QueryResponse( query=( Band.select(Band.name) diff --git a/tests/table/instance/test_get_related.py b/tests/table/instance/test_get_related.py index a54938917..b662f54a0 100644 --- a/tests/table/instance/test_get_related.py +++ b/tests/table/instance/test_get_related.py @@ -1,4 +1,4 @@ -import typing as t +from typing import cast from piccolo.testing.test_case import AsyncTableTest from tests.example_apps.music.tables import Band, Concert, Manager, Venue @@ -52,7 +52,7 @@ async def test_string(self): """ Make sure it also works using a string representation of a foreign key. """ - manager = t.cast(Manager, await self.band.get_related("manager")) + manager = cast(Manager, await self.band.get_related("manager")) self.assertTrue(manager.id == self.manager.id) async def test_invalid_string(self): diff --git a/tests/table/test_alter.py b/tests/table/test_alter.py index 836a552a8..32057b9f0 100644 --- a/tests/table/test_alter.py +++ b/tests/table/test_alter.py @@ -1,6 +1,6 @@ from __future__ import annotations -import typing as t +from typing import Any, Union from unittest import TestCase import pytest @@ -26,7 +26,7 @@ class TestRenameColumn(DBTestCase): def _test_rename( self, - existing_column: t.Union[Column, str], + existing_column: Union[Column, str], new_column_name: str = "rating", ): self.insert_row() @@ -90,7 +90,7 @@ class TestDropColumn(DBTestCase): SQLite has very limited support for ALTER statements. """ - def _test_drop(self, column: t.Union[str, Column]): + def _test_drop(self, column: Union[str, Column]): self.insert_row() Band.alter().drop_column(column).run_sync() @@ -109,7 +109,7 @@ def test_drop_column(self): class TestAddColumn(DBTestCase): def _test_add_column( - self, column: Column, column_name: str, expected_value: t.Any + self, column: Column, column_name: str, expected_value: Any ): self.insert_row() Band.alter().add_column(column_name, column).run_sync() diff --git a/tests/table/test_indexes.py b/tests/table/test_indexes.py index 13d1758de..a3d44bcf1 100644 --- a/tests/table/test_indexes.py +++ b/tests/table/test_indexes.py @@ -1,4 +1,3 @@ -import typing as t from unittest import TestCase from piccolo.columns.base import Column @@ -52,7 +51,7 @@ def test_problematic_name(self) -> None: Make sure we can add an index to a column with a problematic name (which clashes with a SQL keyword). """ - columns: t.List[Column] = [Concert.order] + columns: list[Column] = [Concert.order] Concert.create_index(columns=columns).run_sync() index_name = Concert._get_index_name([i._meta.name for i in columns]) diff --git a/tests/table/test_refresh.py b/tests/table/test_refresh.py index dd2109c7b..ce002bb9a 100644 --- a/tests/table/test_refresh.py +++ b/tests/table/test_refresh.py @@ -1,4 +1,4 @@ -import typing as t +from typing import cast from piccolo.testing.test_case import TableTest from tests.base import DBTestCase @@ -293,6 +293,6 @@ def test_load_json(self): self.recording_studio.refresh(load_json=True).run_sync() self.assertDictEqual( - t.cast(dict, self.recording_studio.facilities), + cast(dict, self.recording_studio.facilities), {"electric piano": True}, ) diff --git a/tests/table/test_update.py b/tests/table/test_update.py index 554774f60..4cb29df26 100644 --- a/tests/table/test_update.py +++ b/tests/table/test_update.py @@ -1,6 +1,6 @@ import dataclasses import datetime -import typing as t +from typing import Any from unittest import TestCase import pytest @@ -181,9 +181,9 @@ class MyTable(Table): class OperatorTestCase: description: str column: Column - initial: t.Any + initial: Any querystring: QueryString - expected: t.Any + expected: Any TEST_CASES = [ diff --git a/tests/testing/test_model_builder.py b/tests/testing/test_model_builder.py index 242bac188..dba0dc791 100644 --- a/tests/testing/test_model_builder.py +++ b/tests/testing/test_model_builder.py @@ -1,6 +1,5 @@ import asyncio import json -import typing as t import unittest from piccolo.columns import ( @@ -83,7 +82,7 @@ def tearDownClass(cls) -> None: drop_db_tables_sync(*TABLES) def test_async(self): - async def build_model(table_class: t.Type[Table]): + async def build_model(table_class: type[Table]): return await ModelBuilder.build(table_class) for table_class in TABLES: diff --git a/tests/type_checking.py b/tests/type_checking.py index 717d62206..47cf6944f 100644 --- a/tests/type_checking.py +++ b/tests/type_checking.py @@ -5,7 +5,7 @@ type inside the function as Any. """ -import typing as t +from typing import TYPE_CHECKING, Any, Optional from typing_extensions import assert_type @@ -15,25 +15,25 @@ from .example_apps.music.tables import Band, Concert, Manager -if t.TYPE_CHECKING: +if TYPE_CHECKING: async def objects() -> None: query = Band.objects() - assert_type(await query, t.List[Band]) - assert_type(await query.run(), t.List[Band]) - assert_type(query.run_sync(), t.List[Band]) + assert_type(await query, list[Band]) + assert_type(await query.run(), list[Band]) + assert_type(query.run_sync(), list[Band]) async def objects_first() -> None: query = Band.objects().first() - assert_type(await query, t.Optional[Band]) - assert_type(await query.run(), t.Optional[Band]) - assert_type(query.run_sync(), t.Optional[Band]) + assert_type(await query, Optional[Band]) + assert_type(await query.run(), Optional[Band]) + assert_type(query.run_sync(), Optional[Band]) async def get() -> None: query = Band.objects().get(Band.name == "Pythonistas") - assert_type(await query, t.Optional[Band]) - assert_type(await query.run(), t.Optional[Band]) - assert_type(query.run_sync(), t.Optional[Band]) + assert_type(await query, Optional[Band]) + assert_type(await query.run(), Optional[Band]) + assert_type(query.run_sync(), Optional[Band]) async def foreign_key_reference() -> None: assert_type(Band.manager, ForeignKey[Manager]) @@ -48,13 +48,13 @@ async def get_related() -> None: band = await Band.objects().get(Band.name == "Pythonistas") assert band is not None manager = await band.get_related(Band.manager) - assert_type(manager, t.Optional[Manager]) + assert_type(manager, Optional[Manager]) async def get_related_multiple_levels() -> None: concert = await Concert.objects().first() assert concert is not None manager = await concert.get_related(Concert.band_1._.manager) - assert_type(manager, t.Optional[Manager]) + assert_type(manager, Optional[Manager]) async def get_or_create() -> None: query = Band.objects().get_or_create(Band.name == "Pythonistas") @@ -64,22 +64,22 @@ async def get_or_create() -> None: async def select() -> None: query = Band.select() - assert_type(await query, t.List[t.Dict[str, t.Any]]) - assert_type(await query.run(), t.List[t.Dict[str, t.Any]]) - assert_type(query.run_sync(), t.List[t.Dict[str, t.Any]]) + assert_type(await query, list[dict[str, Any]]) + assert_type(await query.run(), list[dict[str, Any]]) + assert_type(query.run_sync(), list[dict[str, Any]]) async def select_first() -> None: query = Band.select().first() - assert_type(await query, t.Optional[t.Dict[str, t.Any]]) - assert_type(await query.run(), t.Optional[t.Dict[str, t.Any]]) - assert_type(query.run_sync(), t.Optional[t.Dict[str, t.Any]]) + assert_type(await query, Optional[dict[str, Any]]) + assert_type(await query.run(), Optional[dict[str, Any]]) + assert_type(query.run_sync(), Optional[dict[str, Any]]) async def select_list() -> None: query = Band.select(Band.name).output(as_list=True) - assert_type(await query, t.List) - assert_type(await query.run(), t.List) - assert_type(query.run_sync(), t.List) - # The next step would be to detect that it's t.List[str], but might not + assert_type(await query, list) + assert_type(await query.run(), list) + assert_type(query.run_sync(), list) + # The next step would be to detect that it's list[str], but might not # be possible. async def select_as_json() -> None: @@ -105,9 +105,9 @@ async def from_dict() -> None: async def update() -> None: query = Band.update() - assert_type(await query, t.List[t.Any]) - assert_type(await query.run(), t.List[t.Any]) - assert_type(query.run_sync(), t.List[t.Any]) + assert_type(await query, list[Any]) + assert_type(await query.run(), list[Any]) + assert_type(query.run_sync(), list[Any]) async def insert() -> None: # This is correct: diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index 82096603d..7427fe44e 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -1,5 +1,5 @@ import decimal -import typing as t +from typing import Optional, cast from unittest import TestCase import pydantic @@ -138,7 +138,7 @@ class Band(Table): self.assertEqual( pydantic_model.model_fields["members"].annotation, - t.List[t.List[pydantic.constr(max_length=255)]], + list[list[pydantic.constr(max_length=255)]], ) # Should not raise a validation error: @@ -630,8 +630,8 @@ class Band(Table): ####################################################################### - ManagerModel = t.cast( - t.Type[pydantic.BaseModel], + ManagerModel = cast( + type[pydantic.BaseModel], BandModel.model_fields["manager"].annotation, ) self.assertTrue(issubclass(ManagerModel, pydantic.BaseModel)) @@ -641,8 +641,8 @@ class Band(Table): ####################################################################### - CountryModel = t.cast( - t.Type[pydantic.BaseModel], + CountryModel = cast( + type[pydantic.BaseModel], ManagerModel.model_fields["country"].annotation, ) self.assertTrue(issubclass(CountryModel, pydantic.BaseModel)) @@ -681,8 +681,8 @@ class Concert(Table): BandModel = create_pydantic_model(table=Band, nested=(Band.manager,)) - ManagerModel = t.cast( - t.Type[pydantic.BaseModel], + ManagerModel = cast( + type[pydantic.BaseModel], BandModel.model_fields["manager"].annotation, ) self.assertTrue(issubclass(ManagerModel, pydantic.BaseModel)) @@ -694,7 +694,7 @@ class Concert(Table): AssistantManagerType = BandModel.model_fields[ "assistant_manager" ].annotation - self.assertIs(AssistantManagerType, t.Optional[int]) + self.assertIs(AssistantManagerType, Optional[int]) ####################################################################### # Test two levels deep @@ -703,8 +703,8 @@ class Concert(Table): table=Band, nested=(Band.manager._.country,) ) - ManagerModel = t.cast( - t.Type[pydantic.BaseModel], + ManagerModel = cast( + type[pydantic.BaseModel], BandModel.model_fields["manager"].annotation, ) self.assertTrue(issubclass(ManagerModel, pydantic.BaseModel)) @@ -713,14 +713,14 @@ class Concert(Table): ) self.assertEqual(ManagerModel.__qualname__, "Band.manager") - AssistantManagerType = t.cast( - t.Type[pydantic.BaseModel], + AssistantManagerType = cast( + type[pydantic.BaseModel], BandModel.model_fields["assistant_manager"].annotation, ) - self.assertIs(AssistantManagerType, t.Optional[int]) + self.assertIs(AssistantManagerType, Optional[int]) - CountryModel = t.cast( - t.Type[pydantic.BaseModel], + CountryModel = cast( + type[pydantic.BaseModel], ManagerModel.model_fields["country"].annotation, ) self.assertTrue(issubclass(CountryModel, pydantic.BaseModel)) @@ -737,10 +737,10 @@ class Concert(Table): ) VenueModel = ConcertModel.model_fields["venue"].annotation - self.assertIs(VenueModel, t.Optional[int]) + self.assertIs(VenueModel, Optional[int]) - BandModel = t.cast( - t.Type[pydantic.BaseModel], + BandModel = cast( + type[pydantic.BaseModel], ConcertModel.model_fields["band_1"].annotation, ) self.assertTrue(issubclass(BandModel, pydantic.BaseModel)) @@ -750,8 +750,8 @@ class Concert(Table): ) self.assertEqual(BandModel.__qualname__, "Concert.band_1") - ManagerModel = t.cast( - t.Type[pydantic.BaseModel], + ManagerModel = cast( + type[pydantic.BaseModel], BandModel.model_fields["manager"].annotation, ) self.assertTrue(issubclass(ManagerModel, pydantic.BaseModel)) @@ -764,10 +764,10 @@ class Concert(Table): AssistantManagerType = BandModel.model_fields[ "assistant_manager" ].annotation - self.assertIs(AssistantManagerType, t.Optional[int]) + self.assertIs(AssistantManagerType, Optional[int]) CountryModel = ManagerModel.model_fields["country"].annotation - self.assertIs(CountryModel, t.Optional[int]) + self.assertIs(CountryModel, Optional[int]) ####################################################################### # Test with `model_name` arg @@ -778,8 +778,8 @@ class Concert(Table): model_name="MyConcertModel", ) - BandModel = t.cast( - t.Type[pydantic.BaseModel], + BandModel = cast( + type[pydantic.BaseModel], MyConcertModel.model_fields["band_1"].annotation, ) self.assertEqual(BandModel.__qualname__, "MyConcertModel.band_1") @@ -810,8 +810,8 @@ class Band(Table): table=Band, nested=True, include_default_columns=True ) - ManagerModel = t.cast( - t.Type[pydantic.BaseModel], + ManagerModel = cast( + type[pydantic.BaseModel], BandModel.model_fields["manager"].annotation, ) self.assertTrue(issubclass(ManagerModel, pydantic.BaseModel)) @@ -820,8 +820,8 @@ class Band(Table): ["id", "name", "country"], ) - CountryModel = t.cast( - t.Type[pydantic.BaseModel], + CountryModel = cast( + type[pydantic.BaseModel], ManagerModel.model_fields["country"].annotation, ) self.assertTrue(issubclass(CountryModel, pydantic.BaseModel)) @@ -855,27 +855,27 @@ class Concert(Table): table=Concert, nested=True, max_recursion_depth=2 ) - VenueModel = t.cast( - t.Type[pydantic.BaseModel], + VenueModel = cast( + type[pydantic.BaseModel], ConcertModel.model_fields["venue"].annotation, ) self.assertTrue(issubclass(VenueModel, pydantic.BaseModel)) - BandModel = t.cast( - t.Type[pydantic.BaseModel], + BandModel = cast( + type[pydantic.BaseModel], ConcertModel.model_fields["band"].annotation, ) self.assertTrue(issubclass(BandModel, pydantic.BaseModel)) - ManagerModel = t.cast( - t.Type[pydantic.BaseModel], + ManagerModel = cast( + type[pydantic.BaseModel], BandModel.model_fields["manager"].annotation, ) self.assertTrue(issubclass(ManagerModel, pydantic.BaseModel)) # We should have hit the recursion depth: CountryModel = ManagerModel.model_fields["country"].annotation - self.assertIs(CountryModel, t.Optional[int]) + self.assertIs(CountryModel, Optional[int]) class TestDBColumnName(TestCase): diff --git a/tests/utils/test_table_reflection.py b/tests/utils/test_table_reflection.py index b11858f30..88cba6910 100644 --- a/tests/utils/test_table_reflection.py +++ b/tests/utils/test_table_reflection.py @@ -1,4 +1,3 @@ -import typing as t from unittest import TestCase from piccolo.columns import Varchar @@ -22,7 +21,7 @@ def tearDown(self): table_class.alter().drop_table(if_exists=True).run_sync() def _compare_table_columns( - self, table_1: t.Type[Table], table_2: t.Type[Table] + self, table_1: type[Table], table_2: type[Table] ): """ Make sure that for each column in table_1, there is a corresponding From 71a59a4c9130d4c79166573205040b72aaf70da0 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 9 Jul 2025 23:33:55 +0100 Subject: [PATCH 684/727] add `Array` column to the playground (#1216) --- piccolo/apps/playground/commands/run.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/piccolo/apps/playground/commands/run.py b/piccolo/apps/playground/commands/run.py index 343c02e38..519d725b3 100644 --- a/piccolo/apps/playground/commands/run.py +++ b/piccolo/apps/playground/commands/run.py @@ -12,6 +12,7 @@ from piccolo.columns import ( JSON, UUID, + Array, Boolean, Date, ForeignKey, @@ -149,6 +150,7 @@ class Album(Table): band = ForeignKey(Band) release_date = Date() recorded_at = ForeignKey(RecordingStudio) + awards = Array(Varchar()) @classmethod def get_readable(cls) -> Readable: @@ -265,6 +267,7 @@ def populate(): Album.recorded_at: recording_studio_1, Album.band: pythonistas, Album.release_date: datetime.date(year=2021, month=1, day=1), + Album.awards: ["Grammy Award 2021"], } ), Album( @@ -273,6 +276,7 @@ def populate(): Album.recorded_at: recording_studio_2, Album.band: rustaceans, Album.release_date: datetime.date(year=2022, month=2, day=2), + Album.awards: ["Mercury Prize 2022"], } ), ).run_sync() From 2c8c5271ec0f9924c3553b9c5b07ca14279a3537 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Fri, 11 Jul 2025 01:26:18 +0200 Subject: [PATCH 685/727] Useful array functions (#1214) * alternative approach to using Array column methods * add array_remove test and fix some tests docstrings * more docstring fix * add @dantownsend suggestions * move array.py to query functions * tweak annotations * tweak docstrings * add `Postgres / CockroachDB only` notes --------- Co-authored-by: Daniel Townsend --- docs/src/piccolo/schema/column_types.rst | 25 ++++ piccolo/columns/column_types.py | 87 +++++++++++-- piccolo/query/functions/__init__.py | 12 ++ piccolo/query/functions/array.py | 131 +++++++++++++++++++ tests/columns/test_array.py | 158 ++++++++++++++++++++++- 5 files changed, 400 insertions(+), 13 deletions(-) create mode 100644 piccolo/query/functions/array.py diff --git a/docs/src/piccolo/schema/column_types.rst b/docs/src/piccolo/schema/column_types.rst index 5c70dd482..bdf2258bf 100644 --- a/docs/src/piccolo/schema/column_types.rst +++ b/docs/src/piccolo/schema/column_types.rst @@ -443,3 +443,28 @@ cat === .. automethod:: Array.cat + +====== +remove +====== + +.. automethod:: Array.remove + +====== +append +====== + +.. automethod:: Array.append + + +======= +prepend +======= + +.. automethod:: Array.prepend + +======= +replace +======= + +.. automethod:: Array.replace diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 572aac102..79f00460b 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -2811,9 +2811,9 @@ def all(self, value: Any) -> Where: else: raise ValueError("Unrecognised engine type") - def cat(self, value: Union[Any, list[Any]]) -> QueryString: + def cat(self, value: list[Any]) -> QueryString: """ - Used in an ``update`` query to append items to an array. + Used in an ``update`` query to concatenate two arrays. .. code-block:: python @@ -2829,20 +2829,83 @@ def cat(self, value: Union[Any, list[Any]]) -> QueryString: ... Ticket.seat_numbers: Ticket.seat_numbers + [1000] ... }).where(Ticket.id == 1) + .. note:: Postgres / CockroachDB only + """ - engine_type = self._meta.engine_type - if engine_type != "postgres" and engine_type != "cockroach": - raise ValueError( - "Only Postgres and Cockroach support array appending." - ) + from piccolo.query.functions.array import ArrayCat + + return ArrayCat(self, value=value) + + def remove(self, value: Any) -> QueryString: + """ + Used in an ``update`` query to remove an item from an array. + + .. code-block:: python + + >>> await Ticket.update({ + ... Ticket.seat_numbers: Ticket.seat_numbers.remove(1000) + ... }).where(Ticket.id == 1) + + .. note:: Postgres / CockroachDB only + + """ + from piccolo.query.functions.array import ArrayRemove + + return ArrayRemove(self, value=value) + + def prepend(self, value: Any) -> QueryString: + """ + Used in an ``update`` query to prepend an item to an array. + + .. code-block:: python + + >>> await Ticket.update({ + ... Ticket.seat_numbers: Ticket.seat_numbers.prepend(1000) + ... }).where(Ticket.id == 1) - if not isinstance(value, list): - value = [value] + .. note:: Postgres / CockroachDB only + + """ + from piccolo.query.functions.array import ArrayPrepend + + return ArrayPrepend(self, value=value) + + def append(self, value: Any) -> QueryString: + """ + Used in an ``update`` query to append an item to an array. + + .. code-block:: python + + >>> await Ticket.update({ + ... Ticket.seat_numbers: Ticket.seat_numbers.append(1000) + ... }).where(Ticket.id == 1) + + .. note:: Postgres / CockroachDB only + + """ + from piccolo.query.functions.array import ArrayAppend + + return ArrayAppend(self, value=value) + + def replace(self, old_value: Any, new_value: Any) -> QueryString: + """ + Used in an ``update`` query to replace each array item + equal to the given value with a new value. + + .. code-block:: python + + >>> await Ticket.update({ + ... Ticket.seat_numbers: Ticket.seat_numbers.replace(1000, 500) + ... }).where(Ticket.id == 1) + + .. note:: Postgres / CockroachDB only + + """ + from piccolo.query.functions.array import ArrayReplace - db_column_name = self._meta.db_column_name - return QueryString(f'array_cat("{db_column_name}", {{}})', value) + return ArrayReplace(self, old_value=old_value, new_value=new_value) - def __add__(self, value: Union[Any, list[Any]]) -> QueryString: + def __add__(self, value: list[Any]) -> QueryString: return self.cat(value) ########################################################################### diff --git a/piccolo/query/functions/__init__.py b/piccolo/query/functions/__init__.py index 3163f6d1c..9b233cca8 100644 --- a/piccolo/query/functions/__init__.py +++ b/piccolo/query/functions/__init__.py @@ -1,4 +1,11 @@ from .aggregate import Avg, Count, Max, Min, Sum +from .array import ( + ArrayAppend, + ArrayCat, + ArrayPrepend, + ArrayRemove, + ArrayReplace, +) from .datetime import Day, Extract, Hour, Month, Second, Strftime, Year from .math import Abs, Ceil, Floor, Round from .string import Concat, Length, Lower, Ltrim, Reverse, Rtrim, Upper @@ -30,4 +37,9 @@ "Sum", "Upper", "Year", + "ArrayAppend", + "ArrayCat", + "ArrayPrepend", + "ArrayRemove", + "ArrayReplace", ) diff --git a/piccolo/query/functions/array.py b/piccolo/query/functions/array.py new file mode 100644 index 000000000..02b0fda0f --- /dev/null +++ b/piccolo/query/functions/array.py @@ -0,0 +1,131 @@ +from typing import Any, Union + +from piccolo.columns.base import Column +from piccolo.querystring import QueryString + + +class ArrayCat(QueryString): + def __init__(self, column: Union[Column, QueryString], value: list[Any]): + """ + Concatenate two arrays. + + :param column: + Identifies the column. + :param value: + The value to concatenate. + + """ + if isinstance(column, Column): + engine_type = column._meta.engine_type + if engine_type not in ("postgres", "cockroach"): + raise ValueError( + "Only Postgres and Cockroach support array concatenating." + ) + + if not isinstance(value, list): + value = [value] + + super().__init__("array_cat({}, {})", column, value) + + +class ArrayAppend(QueryString): + def __init__(self, column: Union[Column, QueryString], value: Any): + """ + Append an element to the end of an array. + + :param column: + Identifies the column. + :param value: + The value to append. + + """ + if isinstance(column, Column): + engine_type = column._meta.engine_type + if engine_type not in ("postgres", "cockroach"): + raise ValueError( + "Only Postgres and Cockroach support array appending." + ) + + super().__init__("array_append({}, {})", column, value) + + +class ArrayPrepend(QueryString): + def __init__(self, column: Union[Column, QueryString], value: Any): + """ + Append an element to the beginning of an array. + + :param value: + The value to prepend. + :param column: + Identifies the column. + + """ + if isinstance(column, Column): + engine_type = column._meta.engine_type + if engine_type not in ("postgres", "cockroach"): + raise ValueError( + "Only Postgres and Cockroach support array prepending." + ) + + super().__init__("array_prepend({}, {})", value, column) + + +class ArrayReplace(QueryString): + def __init__( + self, + column: Union[Column, QueryString], + old_value: Any, + new_value: Any, + ): + """ + Replace each array element equal to the given value with a new value. + + :param column: + Identifies the column. + :param old_value: + The old value to be replaced. + :param new_value: + The new value we are replacing with. + + """ + if isinstance(column, Column): + engine_type = column._meta.engine_type + if engine_type not in ("postgres", "cockroach"): + raise ValueError( + "Only Postgres and Cockroach support array substitution." + ) + + super().__init__( + "array_replace({}, {}, {})", column, old_value, new_value + ) + + +class ArrayRemove(QueryString): + def __init__(self, column: Union[Column, QueryString], value: Any): + """ + Remove all elements equal to the given value + from the array (array must be one-dimensional). + + :param column: + Identifies the column. + :param value: + The value to remove. + + """ + if isinstance(column, Column): + engine_type = column._meta.engine_type + if engine_type not in ("postgres", "cockroach"): + raise ValueError( + "Only Postgres and Cockroach support array removing." + ) + + super().__init__("array_remove({}, {})", column, value) + + +__all__ = ( + "ArrayCat", + "ArrayAppend", + "ArrayPrepend", + "ArrayReplace", + "ArrayRemove", +) diff --git a/tests/columns/test_array.py b/tests/columns/test_array.py index 085331671..45139a56a 100644 --- a/tests/columns/test_array.py +++ b/tests/columns/test_array.py @@ -186,7 +186,7 @@ def test_not_any(self): @pytest.mark.cockroach_array_slow def test_cat(self): """ - Make sure values can be appended to an array. + Make sure values can be appended to an array and that we can concatenate two arrays. In CockroachDB <= v22.2.0 we had this error: @@ -238,11 +238,167 @@ def test_cat_sqlite(self): with self.assertRaises(ValueError) as manager: MyTable.value.cat([2]) + self.assertEqual( + str(manager.exception), + "Only Postgres and Cockroach support array concatenating.", + ) + + @engines_skip("sqlite") + @pytest.mark.cockroach_array_slow + def test_prepend(self): + """ + Make sure values can be added to the beginning of the array. + + In CockroachDB <= v22.2.0 we had this error: + + * https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + + In newer CockroachDB versions, it runs but is very slow: + + * https://github.com/piccolo-orm/piccolo/issues/1005 + + """ # noqa: E501 + MyTable(value=[1, 1, 1]).save().run_sync() + + MyTable.update( + {MyTable.value: MyTable.value.prepend(3)}, force=True + ).run_sync() + + self.assertEqual( + MyTable.select(MyTable.value).run_sync(), + [{"value": [3, 1, 1, 1]}], + ) + + @sqlite_only + def test_prepend_sqlite(self): + """ + If using SQLite then an exception should be raised currently. + """ + with self.assertRaises(ValueError) as manager: + MyTable.value.prepend(2) + + self.assertEqual( + str(manager.exception), + "Only Postgres and Cockroach support array prepending.", + ) + + @engines_skip("sqlite") + @pytest.mark.cockroach_array_slow + def test_append(self): + """ + Make sure values can be appended to an array. + + In CockroachDB <= v22.2.0 we had this error: + + * https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + + In newer CockroachDB versions, it runs but is very slow: + + * https://github.com/piccolo-orm/piccolo/issues/1005 + + """ # noqa: E501 + MyTable(value=[1, 1, 1]).save().run_sync() + + MyTable.update( + {MyTable.value: MyTable.value.append(3)}, force=True + ).run_sync() + + self.assertEqual( + MyTable.select(MyTable.value).run_sync(), + [{"value": [1, 1, 1, 3]}], + ) + + @sqlite_only + def test_append_sqlite(self): + """ + If using SQLite then an exception should be raised currently. + """ + with self.assertRaises(ValueError) as manager: + MyTable.value.append(2) + self.assertEqual( str(manager.exception), "Only Postgres and Cockroach support array appending.", ) + @engines_skip("sqlite") + @pytest.mark.cockroach_array_slow + def test_replace(self): + """ + Make sure values can be swapped in the array. + + In CockroachDB <= v22.2.0 we had this error: + + * https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + + In newer CockroachDB versions, it runs but is very slow: + + * https://github.com/piccolo-orm/piccolo/issues/1005 + + """ # noqa: E501 + MyTable(value=[1, 1, 1]).save().run_sync() + + MyTable.update( + {MyTable.value: MyTable.value.replace(1, 2)}, force=True + ).run_sync() + + self.assertEqual( + MyTable.select(MyTable.value).run_sync(), + [{"value": [2, 2, 2]}], + ) + + @sqlite_only + def test_replace_sqlite(self): + """ + If using SQLite then an exception should be raised currently. + """ + with self.assertRaises(ValueError) as manager: + MyTable.value.replace(1, 2) + + self.assertEqual( + str(manager.exception), + "Only Postgres and Cockroach support array substitution.", + ) + + @engines_skip("sqlite") + @pytest.mark.cockroach_array_slow + def test_remove(self): + """ + Make sure values can be removed from an array. + + In CockroachDB <= v22.2.0 we had this error: + + * https://github.com/cockroachdb/cockroach/issues/71908 "could not decorrelate subquery" error under asyncpg + + In newer CockroachDB versions, it runs but is very slow: + + * https://github.com/piccolo-orm/piccolo/issues/1005 + + """ # noqa: E501 + MyTable(value=[1, 2, 3]).save().run_sync() + + MyTable.update( + {MyTable.value: MyTable.value.remove(2)}, force=True + ).run_sync() + + self.assertEqual( + MyTable.select(MyTable.value).run_sync(), + [{"value": [1, 3]}], + ) + + @sqlite_only + def test_remove_sqlite(self): + """ + If using SQLite then an exception should be raised currently. + """ + with self.assertRaises(ValueError) as manager: + MyTable.value.remove(2) + + self.assertEqual( + str(manager.exception), + "Only Postgres and Cockroach support array removing.", + ) + ############################################################################### # Date and time arrays From c600b7fe59588af1be5e969fc79170c345c63f7e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Mon, 14 Jul 2025 21:23:05 +0100 Subject: [PATCH 686/727] 1218 Improve array concatenation (#1219) * make `ArrayCat` args more flexible * add `__radd__` method * use `ArrayType` as type annotation for all array functions * add auto convert to list for backward compatibility * update docs * fix test * add tests --- piccolo/columns/column_types.py | 37 +++++++++++-- piccolo/query/functions/array.py | 91 +++++++++++++++++++------------- tests/columns/test_array.py | 30 +++++++---- 3 files changed, 107 insertions(+), 51 deletions(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 79f00460b..69a80e090 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -2821,7 +2821,8 @@ def cat(self, value: list[Any]) -> QueryString: ... Ticket.seat_numbers: Ticket.seat_numbers.cat([1000]) ... }).where(Ticket.id == 1) - You can also use the ``+`` symbol if you prefer: + You can also use the ``+`` symbol if you prefer. To concatenate to + the end: .. code-block:: python @@ -2829,12 +2830,33 @@ def cat(self, value: list[Any]) -> QueryString: ... Ticket.seat_numbers: Ticket.seat_numbers + [1000] ... }).where(Ticket.id == 1) + To concatenate to the start: + + .. code-block:: python + + >>> await Ticket.update({ + ... Ticket.seat_numbers: [1000] + Ticket.seat_numbers + ... }).where(Ticket.id == 1) + + You can concatenate multiple arrays in one go: + + .. code-block:: python + + >>> await Ticket.update({ + ... Ticket.seat_numbers: [1000] + Ticket.seat_numbers + [2000] + ... }).where(Ticket.id == 1) + .. note:: Postgres / CockroachDB only """ from piccolo.query.functions.array import ArrayCat - return ArrayCat(self, value=value) + # Keep this for backwards compatibility - we had this as a convenience + # for users, but it would be nice to remove it in the future. + if not isinstance(value, list): + value = [value] + + return ArrayCat(array_1=self, array_2=value) def remove(self, value: Any) -> QueryString: """ @@ -2851,7 +2873,7 @@ def remove(self, value: Any) -> QueryString: """ from piccolo.query.functions.array import ArrayRemove - return ArrayRemove(self, value=value) + return ArrayRemove(array=self, value=value) def prepend(self, value: Any) -> QueryString: """ @@ -2868,7 +2890,7 @@ def prepend(self, value: Any) -> QueryString: """ from piccolo.query.functions.array import ArrayPrepend - return ArrayPrepend(self, value=value) + return ArrayPrepend(array=self, value=value) def append(self, value: Any) -> QueryString: """ @@ -2885,7 +2907,7 @@ def append(self, value: Any) -> QueryString: """ from piccolo.query.functions.array import ArrayAppend - return ArrayAppend(self, value=value) + return ArrayAppend(array=self, value=value) def replace(self, old_value: Any, new_value: Any) -> QueryString: """ @@ -2908,6 +2930,11 @@ def replace(self, old_value: Any, new_value: Any) -> QueryString: def __add__(self, value: list[Any]) -> QueryString: return self.cat(value) + def __radd__(self, value: list[Any]) -> QueryString: + from piccolo.query.functions.array import ArrayCat + + return ArrayCat(array_1=value, array_2=self) + ########################################################################### # Descriptors diff --git a/piccolo/query/functions/array.py b/piccolo/query/functions/array.py index 02b0fda0f..e26f61ca4 100644 --- a/piccolo/query/functions/array.py +++ b/piccolo/query/functions/array.py @@ -1,35 +1,54 @@ from typing import Any, Union +from typing_extensions import TypeAlias + from piccolo.columns.base import Column from piccolo.querystring import QueryString +ArrayType: TypeAlias = Union[Column, QueryString, list[Any]] + -class ArrayCat(QueryString): - def __init__(self, column: Union[Column, QueryString], value: list[Any]): +class ArrayQueryString(QueryString): + def __add__(self, array: ArrayType): """ - Concatenate two arrays. + QueryString will use the ``+`` operator by default for addition, but + for arrays we want to concatenate them instead. + """ + return ArrayCat(array_1=self, array_2=array) - :param column: - Identifies the column. - :param value: - The value to concatenate. + def __radd__(self, array: ArrayType): + return ArrayCat(array_1=array, array_2=self) + +class ArrayCat(ArrayQueryString): + def __init__( + self, + array_1: ArrayType, + array_2: ArrayType, + ): """ - if isinstance(column, Column): - engine_type = column._meta.engine_type - if engine_type not in ("postgres", "cockroach"): - raise ValueError( - "Only Postgres and Cockroach support array concatenating." - ) + Concatenate two arrays. - if not isinstance(value, list): - value = [value] + :param array_1: + These values will be at the start of the new array. + :param array_2: + These values will be at the end of the new array. + + """ + for value in (array_1, array_2): + if isinstance(value, Column): + engine_type = value._meta.engine_type + if engine_type not in ("postgres", "cockroach"): + raise ValueError( + "Only Postgres and Cockroach support array " + "concatenation." + ) - super().__init__("array_cat({}, {})", column, value) + super().__init__("array_cat({}, {})", array_1, array_2) -class ArrayAppend(QueryString): - def __init__(self, column: Union[Column, QueryString], value: Any): +class ArrayAppend(ArrayQueryString): + def __init__(self, array: ArrayType, value: Any): """ Append an element to the end of an array. @@ -39,18 +58,18 @@ def __init__(self, column: Union[Column, QueryString], value: Any): The value to append. """ - if isinstance(column, Column): - engine_type = column._meta.engine_type + if isinstance(array, Column): + engine_type = array._meta.engine_type if engine_type not in ("postgres", "cockroach"): raise ValueError( "Only Postgres and Cockroach support array appending." ) - super().__init__("array_append({}, {})", column, value) + super().__init__("array_append({}, {})", array, value) -class ArrayPrepend(QueryString): - def __init__(self, column: Union[Column, QueryString], value: Any): +class ArrayPrepend(ArrayQueryString): + def __init__(self, array: ArrayType, value: Any): """ Append an element to the beginning of an array. @@ -60,20 +79,20 @@ def __init__(self, column: Union[Column, QueryString], value: Any): Identifies the column. """ - if isinstance(column, Column): - engine_type = column._meta.engine_type + if isinstance(array, Column): + engine_type = array._meta.engine_type if engine_type not in ("postgres", "cockroach"): raise ValueError( "Only Postgres and Cockroach support array prepending." ) - super().__init__("array_prepend({}, {})", value, column) + super().__init__("array_prepend({}, {})", value, array) -class ArrayReplace(QueryString): +class ArrayReplace(ArrayQueryString): def __init__( self, - column: Union[Column, QueryString], + array: ArrayType, old_value: Any, new_value: Any, ): @@ -88,20 +107,20 @@ def __init__( The new value we are replacing with. """ - if isinstance(column, Column): - engine_type = column._meta.engine_type + if isinstance(array, Column): + engine_type = array._meta.engine_type if engine_type not in ("postgres", "cockroach"): raise ValueError( "Only Postgres and Cockroach support array substitution." ) super().__init__( - "array_replace({}, {}, {})", column, old_value, new_value + "array_replace({}, {}, {})", array, old_value, new_value ) -class ArrayRemove(QueryString): - def __init__(self, column: Union[Column, QueryString], value: Any): +class ArrayRemove(ArrayQueryString): + def __init__(self, array: ArrayType, value: Any): """ Remove all elements equal to the given value from the array (array must be one-dimensional). @@ -112,14 +131,14 @@ def __init__(self, column: Union[Column, QueryString], value: Any): The value to remove. """ - if isinstance(column, Column): - engine_type = column._meta.engine_type + if isinstance(array, Column): + engine_type = array._meta.engine_type if engine_type not in ("postgres", "cockroach"): raise ValueError( "Only Postgres and Cockroach support array removing." ) - super().__init__("array_remove({}, {})", column, value) + super().__init__("array_remove({}, {})", array, value) __all__ = ( diff --git a/tests/columns/test_array.py b/tests/columns/test_array.py index 45139a56a..d347d0fe5 100644 --- a/tests/columns/test_array.py +++ b/tests/columns/test_array.py @@ -197,37 +197,47 @@ def test_cat(self): * https://github.com/piccolo-orm/piccolo/issues/1005 """ # noqa: E501 - MyTable(value=[1, 1, 1]).save().run_sync() + MyTable(value=[5]).save().run_sync() MyTable.update( - {MyTable.value: MyTable.value.cat([2])}, force=True + {MyTable.value: MyTable.value.cat([6])}, force=True ).run_sync() self.assertEqual( MyTable.select(MyTable.value).run_sync(), - [{"value": [1, 1, 1, 2]}], + [{"value": [5, 6]}], ) - # Try plus symbol + # Try plus symbol - add array to the end MyTable.update( - {MyTable.value: MyTable.value + [3]}, force=True + {MyTable.value: MyTable.value + [7]}, force=True ).run_sync() self.assertEqual( MyTable.select(MyTable.value).run_sync(), - [{"value": [1, 1, 1, 2, 3]}], + [{"value": [5, 6, 7]}], ) - # Make sure non-list values work + # Add array to the start + + MyTable.update( + {MyTable.value: [4] + MyTable.value}, force=True + ).run_sync() + + self.assertEqual( + MyTable.select(MyTable.value).run_sync(), + [{"value": [4, 5, 6, 7]}], + ) + # Add array to the start and end MyTable.update( - {MyTable.value: MyTable.value + 4}, force=True + {MyTable.value: [3] + MyTable.value + [8]}, force=True ).run_sync() self.assertEqual( MyTable.select(MyTable.value).run_sync(), - [{"value": [1, 1, 1, 2, 3, 4]}], + [{"value": [3, 4, 5, 6, 7, 8]}], ) @sqlite_only @@ -240,7 +250,7 @@ def test_cat_sqlite(self): self.assertEqual( str(manager.exception), - "Only Postgres and Cockroach support array concatenating.", + "Only Postgres and Cockroach support array concatenation.", ) @engines_skip("sqlite") From 3943cc4ed6a19c5a65fa92c4bfddd8e792e9255c Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 15 Jul 2025 14:18:07 +0100 Subject: [PATCH 687/727] 1220 Improve documentation for functions (#1221) * add hint that functions are an advanced topic * add array functions to docs * add basic docs about custom functions * Update index.rst * improve type annotations * include docstrings for array functions in docs * link to functions from array methods * improve docs for custom functions --- docs/src/piccolo/api_reference/index.rst | 9 ++++++ docs/src/piccolo/functions/array.rst | 29 +++++++++++++++++++ docs/src/piccolo/functions/custom.rst | 24 ++++++++++++++++ docs/src/piccolo/functions/index.rst | 12 ++++++-- piccolo/columns/column_types.py | 36 +++++++++++++++++++----- piccolo/query/functions/array.py | 15 +++++----- 6 files changed, 108 insertions(+), 17 deletions(-) create mode 100644 docs/src/piccolo/functions/array.rst create mode 100644 docs/src/piccolo/functions/custom.rst diff --git a/docs/src/piccolo/api_reference/index.rst b/docs/src/piccolo/api_reference/index.rst index 6159548a2..e5a5268f4 100644 --- a/docs/src/piccolo/api_reference/index.rst +++ b/docs/src/piccolo/api_reference/index.rst @@ -124,3 +124,12 @@ create_db_tables / drop_db_tables .. autofunction:: create_db_tables_sync .. autofunction:: drop_db_tables .. autofunction:: drop_db_tables_sync + +------------------------------------------------------------------------------- + +QueryString +----------- + +.. currentmodule:: piccolo.querystring + +.. autoclass:: QueryString diff --git a/docs/src/piccolo/functions/array.rst b/docs/src/piccolo/functions/array.rst new file mode 100644 index 000000000..aa457210a --- /dev/null +++ b/docs/src/piccolo/functions/array.rst @@ -0,0 +1,29 @@ +Array functions +=============== + +.. currentmodule:: piccolo.query.functions.array + +ArrayCat +-------- + +.. autoclass:: ArrayCat + +ArrayAppend +----------- + +.. autoclass:: ArrayAppend + +ArrayPrepend +------------ + +.. autoclass:: ArrayPrepend + +ArrayRemove +----------- + +.. autoclass:: ArrayRemove + +ArrayReplace +------------ + +.. autoclass:: ArrayReplace diff --git a/docs/src/piccolo/functions/custom.rst b/docs/src/piccolo/functions/custom.rst new file mode 100644 index 000000000..c832c257c --- /dev/null +++ b/docs/src/piccolo/functions/custom.rst @@ -0,0 +1,24 @@ +Custom functions +================ + +If there's a database function which Piccolo doesn't provide out of the box, +you can still easily access it by using :class:`QueryString ` +directly. + +QueryString +----------- + +:class:`QueryString ` is the building block of +queries in Piccolo. + +If we have a custom function defined in the database called ``slugify``, you +can access it like this: + +.. code-block:: python + + from piccolo.querystring import QueryString + + await Band.select( + Band.name, + QueryString('slugify({})', Band.name, alias='name_slug') + ) diff --git a/docs/src/piccolo/functions/index.rst b/docs/src/piccolo/functions/index.rst index da3dfe43d..4f702a1c8 100644 --- a/docs/src/piccolo/functions/index.rst +++ b/docs/src/piccolo/functions/index.rst @@ -1,14 +1,20 @@ Functions ========= +.. hint:: + This is an advanced topic - if you're new to Piccolo you can skip this for + now. + Functions can be used to modify how queries are run, and what is returned. .. toctree:: :maxdepth: 1 ./basic_usage - ./string - ./math + ./aggregate + ./array ./datetime + ./math + ./string ./type_conversion - ./aggregate + ./custom diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 69a80e090..7c8e6e5b2 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -83,6 +83,7 @@ class Band(Table): if TYPE_CHECKING: # pragma: no cover from piccolo.columns.base import ColumnMeta + from piccolo.query.functions.array import ArrayItemType, ArrayType from piccolo.query.operators.json import ( GetChildElement, GetElementFromPath, @@ -2811,8 +2812,11 @@ def all(self, value: Any) -> Where: else: raise ValueError("Unrecognised engine type") - def cat(self, value: list[Any]) -> QueryString: + def cat(self, value: ArrayType) -> QueryString: """ + A convenient way of accessing the + :class:`ArrayCat ` function. + Used in an ``update`` query to concatenate two arrays. .. code-block:: python @@ -2858,8 +2862,12 @@ def cat(self, value: list[Any]) -> QueryString: return ArrayCat(array_1=self, array_2=value) - def remove(self, value: Any) -> QueryString: + def remove(self, value: ArrayItemType) -> QueryString: """ + A convenient way of accessing the + :class:`ArrayRemove ` + function. + Used in an ``update`` query to remove an item from an array. .. code-block:: python @@ -2875,8 +2883,12 @@ def remove(self, value: Any) -> QueryString: return ArrayRemove(array=self, value=value) - def prepend(self, value: Any) -> QueryString: + def prepend(self, value: ArrayItemType) -> QueryString: """ + A convenient way of accessing the + :class:`ArrayPrepend ` + function. + Used in an ``update`` query to prepend an item to an array. .. code-block:: python @@ -2892,8 +2904,12 @@ def prepend(self, value: Any) -> QueryString: return ArrayPrepend(array=self, value=value) - def append(self, value: Any) -> QueryString: + def append(self, value: ArrayItemType) -> QueryString: """ + A convenient way of accessing the + :class:`ArrayAppend ` + function. + Used in an ``update`` query to append an item to an array. .. code-block:: python @@ -2909,8 +2925,14 @@ def append(self, value: Any) -> QueryString: return ArrayAppend(array=self, value=value) - def replace(self, old_value: Any, new_value: Any) -> QueryString: + def replace( + self, old_value: ArrayItemType, new_value: ArrayItemType + ) -> QueryString: """ + A convenient way of accessing the + :class:`ArrayReplace ` + function. + Used in an ``update`` query to replace each array item equal to the given value with a new value. @@ -2927,10 +2949,10 @@ def replace(self, old_value: Any, new_value: Any) -> QueryString: return ArrayReplace(self, old_value=old_value, new_value=new_value) - def __add__(self, value: list[Any]) -> QueryString: + def __add__(self, value: ArrayType) -> QueryString: return self.cat(value) - def __radd__(self, value: list[Any]) -> QueryString: + def __radd__(self, value: ArrayType) -> QueryString: from piccolo.query.functions.array import ArrayCat return ArrayCat(array_1=value, array_2=self) diff --git a/piccolo/query/functions/array.py b/piccolo/query/functions/array.py index e26f61ca4..13e020929 100644 --- a/piccolo/query/functions/array.py +++ b/piccolo/query/functions/array.py @@ -1,11 +1,12 @@ -from typing import Any, Union +from typing import Union from typing_extensions import TypeAlias from piccolo.columns.base import Column from piccolo.querystring import QueryString -ArrayType: TypeAlias = Union[Column, QueryString, list[Any]] +ArrayType: TypeAlias = Union[Column, QueryString, list[object]] +ArrayItemType: TypeAlias = Union[Column, QueryString, object] class ArrayQueryString(QueryString): @@ -48,7 +49,7 @@ def __init__( class ArrayAppend(ArrayQueryString): - def __init__(self, array: ArrayType, value: Any): + def __init__(self, array: ArrayType, value: ArrayItemType): """ Append an element to the end of an array. @@ -69,7 +70,7 @@ def __init__(self, array: ArrayType, value: Any): class ArrayPrepend(ArrayQueryString): - def __init__(self, array: ArrayType, value: Any): + def __init__(self, array: ArrayType, value: ArrayItemType): """ Append an element to the beginning of an array. @@ -93,8 +94,8 @@ class ArrayReplace(ArrayQueryString): def __init__( self, array: ArrayType, - old_value: Any, - new_value: Any, + old_value: ArrayItemType, + new_value: ArrayItemType, ): """ Replace each array element equal to the given value with a new value. @@ -120,7 +121,7 @@ def __init__( class ArrayRemove(ArrayQueryString): - def __init__(self, array: ArrayType, value: Any): + def __init__(self, array: ArrayType, value: ArrayItemType): """ Remove all elements equal to the given value from the array (array must be one-dimensional). From 8909679b2e2fe193ae68c7e594e38bdf173692a3 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 15 Jul 2025 17:35:58 +0100 Subject: [PATCH 688/727] cast int defaults to float (#1223) --- piccolo/columns/column_types.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 7c8e6e5b2..7137f2e88 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -1554,6 +1554,10 @@ def __init__( default: Union[float, Enum, Callable[[], float], None] = 0.0, **kwargs: Unpack[ColumnKwargs], ) -> None: + if isinstance(default, int): + # For example, allow `0` as a valid default. + default = float(default) + self._validate_default(default, (float, None)) self.default = default super().__init__(default=default, **kwargs) From 15d18af0a626d955c4c6e3c814674efa340f4bb3 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Tue, 15 Jul 2025 23:41:57 +0200 Subject: [PATCH 689/727] Allow a subquery to be passed into `is_in` and `not_in` (#1217) * adds a subquery for is_in and not_in for tables with fk relationships * pass `QueryString` in, instead of `Select` * allow querystring to be passed in to - cuts down on repetition * add more tests * tweak docs * update docs --------- Co-authored-by: Daniel Townsend --- docs/src/piccolo/query_clauses/where.rst | 39 ++++++++++++++++ piccolo/columns/base.py | 43 ++++++++++++++---- piccolo/columns/combination.py | 7 ++- tests/columns/test_combination.py | 30 +++++++++++- tests/table/test_select.py | 58 ++++++++++++++++++++++++ 5 files changed, 164 insertions(+), 13 deletions(-) diff --git a/docs/src/piccolo/query_clauses/where.rst b/docs/src/piccolo/query_clauses/where.rst index edf4aa070..bd88fc633 100644 --- a/docs/src/piccolo/query_clauses/where.rst +++ b/docs/src/piccolo/query_clauses/where.rst @@ -112,6 +112,45 @@ And all rows with a value not contained in the list: Band.name.not_in(['Terrible Band', 'Awful Band']) ) +You can also pass a subquery into the ``is_in`` clause: + +.. code-block:: python + + await Band.select().where( + Band.id.is_in( + Concert.select(Concert.band_1).where( + Concert.starts >= datetime.datetime(year=2025, month=1, day=1) + ) + ) + ) + +.. hint:: + In SQL there are often several ways of solving the same problem. You + can also solve the above using :meth:`join_on `. + + .. code-block:: python + + >>> await Band.select().where( + ... Band.id.join_on(Concert.band_1).starts >= datetime.datetime( + ... year=2025, month=1, day=1 + ... ) + ... ) + + Use whichever you prefer, and whichever suits the situation best. + +Subqueries can also be passed into the ``not_in`` clause: + +.. code-block:: python + + await Band.select().where( + Band.id.not_in( + Concert.select(Concert.band_1).where( + Concert.starts >= datetime.datetime(year=2025, month=1, day=1) + ) + ) + ) + + ------------------------------------------------------------------------------- ``is_null`` / ``is_not_null`` diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index b1994a272..ded8bbcb8 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -46,6 +46,7 @@ if TYPE_CHECKING: # pragma: no cover from piccolo.columns.column_types import ForeignKey + from piccolo.query.methods.select import Select from piccolo.table import Table @@ -599,18 +600,40 @@ def _validate_choices( return True - def is_in(self, values: list[Any]) -> Where: - if len(values) == 0: - raise ValueError( - "The `values` list argument must contain at least one value." - ) + def is_in(self, values: Union[Select, QueryString, list[Any]]) -> Where: + from piccolo.query.methods.select import Select + + if isinstance(values, list): + if len(values) == 0: + raise ValueError( + "The `values` list argument must contain at least one " + "value." + ) + elif isinstance(values, Select): + if len(values.columns_delegate.selected_columns) != 1: + raise ValueError( + "A sub select must only return a single column." + ) + values = values.querystrings[0] + return Where(column=self, values=values, operator=In) - def not_in(self, values: list[Any]) -> Where: - if len(values) == 0: - raise ValueError( - "The `values` list argument must contain at least one value." - ) + def not_in(self, values: Union[Select, QueryString, list[Any]]) -> Where: + from piccolo.query.methods.select import Select + + if isinstance(values, list): + if len(values) == 0: + raise ValueError( + "The `values` list argument must contain at least one " + "value." + ) + elif isinstance(values, Select): + if len(values.columns_delegate.selected_columns) != 1: + raise ValueError( + "A sub select must only return a single column." + ) + values = values.querystrings[0] + return Where(column=self, values=values, operator=NotIn) def like(self, value: str) -> Where: diff --git a/piccolo/columns/combination.py b/piccolo/columns/combination.py index a8179801f..79e14bdb5 100644 --- a/piccolo/columns/combination.py +++ b/piccolo/columns/combination.py @@ -146,7 +146,7 @@ def __init__( self, column: Column, value: Any = UNDEFINED, - values: Union[CustomIterable, Undefined] = UNDEFINED, + values: Union[CustomIterable, Undefined, QueryString] = UNDEFINED, operator: type[ComparisonOperator] = ComparisonOperator, ) -> None: """ @@ -156,7 +156,7 @@ def __init__( self.column = column self.value = value if value == UNDEFINED else self.clean_value(value) - if values == UNDEFINED: + if (values == UNDEFINED) or isinstance(values, QueryString): self.values = values else: self.values = [self.clean_value(i) for i in values] # type: ignore @@ -192,6 +192,9 @@ def clean_value(self, value: Any) -> Any: def values_querystring(self) -> QueryString: values = self.values + if isinstance(values, QueryString): + return values + if isinstance(values, Undefined): raise ValueError("values is undefined") diff --git a/tests/columns/test_combination.py b/tests/columns/test_combination.py index c17c55d99..4f36bf99f 100644 --- a/tests/columns/test_combination.py +++ b/tests/columns/test_combination.py @@ -1,6 +1,6 @@ import unittest -from tests.example_apps.music.tables import Band +from tests.example_apps.music.tables import Band, Concert class TestWhere(unittest.TestCase): @@ -29,6 +29,20 @@ def test_is_in(self): with self.assertRaises(ValueError): Band.name.is_in([]) + def test_is_in_subquery(self): + _where = Band.id.is_in( + Concert.select(Concert.band_1).where(Concert.band_1 == 1) + ) + sql = _where.__str__() + self.assertEqual( + sql, + '"band"."id" IN (SELECT ALL "concert"."band_1" AS "band_1" FROM "concert" WHERE "concert"."band_1" = 1)', # noqa: E501 + ) + + # a sub select must only return a single column + with self.assertRaises(ValueError): + Band.id.is_in(Concert.select().where(Concert.band_1 == 1)) + def test_not_in(self): _where = Band.name.not_in(["CSharps"]) sql = _where.__str__() @@ -37,6 +51,20 @@ def test_not_in(self): with self.assertRaises(ValueError): Band.name.not_in([]) + def test_not_in_subquery(self): + _where = Band.id.not_in( + Concert.select(Concert.band_1).where(Concert.band_1 == 1) + ) + sql = _where.__str__() + self.assertEqual( + sql, + '"band"."id" NOT IN (SELECT ALL "concert"."band_1" AS "band_1" FROM "concert" WHERE "concert"."band_1" = 1)', # noqa: E501 + ) + + # a sub select must only return a single column + with self.assertRaises(ValueError): + Band.id.not_in(Concert.select().where(Concert.band_1 == 1)) + class TestAnd(unittest.TestCase): def test_get_column_values(self): diff --git a/tests/table/test_select.py b/tests/table/test_select.py index 74b876d5f..d41962a01 100644 --- a/tests/table/test_select.py +++ b/tests/table/test_select.py @@ -258,6 +258,64 @@ def test_where_greater_than(self): self.assertEqual(response, [{"name": "Rustaceans"}]) + def test_is_in(self): + self.insert_rows() + + response = ( + Band.select(Band.name) + .where(Band.manager._.name.is_in(["Guido"])) + .run_sync() + ) + + self.assertListEqual(response, [{"name": "Pythonistas"}]) + + def test_is_in_subquery(self): + self.insert_rows() + + # This is a contrived example, just for testing. + response = ( + Band.select(Band.name) + .where( + Band.manager.is_in( + Manager.select(Manager.id).where(Manager.name == "Guido") + ) + ) + .run_sync() + ) + + self.assertListEqual(response, [{"name": "Pythonistas"}]) + + def test_not_in(self): + self.insert_rows() + + response = ( + Band.select(Band.name) + .where(Band.manager._.name.not_in(["Guido"])) + .run_sync() + ) + + self.assertListEqual( + response, [{"name": "Rustaceans"}, {"name": "CSharps"}] + ) + + def test_not_in_subquery(self): + self.insert_rows() + + # This is a contrived example, just for testing. + response = ( + Band.select(Band.name) + .where( + Band.manager.not_in( + Manager.select(Manager.id).where(Manager.name == "Guido") + ) + ) + .run_sync() + ) + + self.assertListEqual( + response, [{"name": "Rustaceans"}, {"name": "CSharps"}] + ) + def test_where_is_null(self): self.insert_rows() From dca04f224bf2254d2a8288f5f223fad544532928 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 17 Jul 2025 18:21:16 +0100 Subject: [PATCH 690/727] fix bug with missing array base column imports when altering a column (#1227) --- piccolo/apps/migrations/auto/serialisation.py | 2 ++ .../auto/integration/test_migrations.py | 28 +++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/piccolo/apps/migrations/auto/serialisation.py b/piccolo/apps/migrations/auto/serialisation.py index bfafc43d6..5db540cd3 100644 --- a/piccolo/apps/migrations/auto/serialisation.py +++ b/piccolo/apps/migrations/auto/serialisation.py @@ -738,6 +738,8 @@ def deserialise_params(params: dict[str, Any]) -> dict[str, Any]: if isinstance(value, str) and not isinstance(value, Enum): if value != "self": params[key] = deserialise_legacy_params(name=key, value=value) + elif isinstance(value, SerialisedColumnInstance): + params[key] = value.instance elif isinstance(value, SerialisedClassInstance): params[key] = value.instance elif isinstance(value, SerialisedUUID): diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index 77cf95edb..887fa6108 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -693,9 +693,13 @@ def test_array_column_varchar(self): def test_array_column_bigint(self): """ - There was a bug with using an array of ``BigInt`` - see issue 500 on - GitHub. It's because ``BigInt`` requires access to the parent table to + There was a bug with using an array of ``BigInt``: + + http://github.com/piccolo-orm/piccolo/issues/500/ + + It's because ``BigInt`` requires access to the parent table to determine what the column type is. + """ self._test_migrations( table_snapshots=[ @@ -706,6 +710,26 @@ def test_array_column_bigint(self): ] ) + def test_array_base_column_change(self): + """ + There was a bug when trying to change the base column of an array: + + https://github.com/piccolo-orm/piccolo/issues/1076 + + It wasn't importing the base column, e.g. for ``Array(Text())`` it + wasn't importing ``Text``. + + """ + self._test_migrations( + table_snapshots=[ + [self.table(column)] + for column in [ + Array(base_column=Varchar()), + Array(base_column=Text()), + ] + ] + ) + ########################################################################### # We deliberately don't test setting JSON or JSONB columns as indexes, as From 77d63f582e9b61af2cdd42183db0f2c15ba120ec Mon Sep 17 00:00:00 2001 From: sinisaos Date: Fri, 18 Jul 2025 15:29:47 +0200 Subject: [PATCH 691/727] Add subquery m2m columns (#1226) * add subquery and on_conflict clause to m2m columns * removed on_conflict clause --- piccolo/columns/m2m.py | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/piccolo/columns/m2m.py b/piccolo/columns/m2m.py index 25d93f731..c3bb9a77e 100644 --- a/piccolo/columns/m2m.py +++ b/piccolo/columns/m2m.py @@ -16,7 +16,7 @@ from piccolo.utils.list import flatten from piccolo.utils.sync import run_sync -if TYPE_CHECKING: +if TYPE_CHECKING: # pragma: no cover from piccolo.table import Table @@ -373,24 +373,16 @@ async def run(self): secondary_table = self.m2m._meta.secondary_table - # TODO - replace this with a subquery in the future. - ids = ( - await joining_table.select( - getattr( - self.m2m._meta.secondary_foreign_key, - secondary_table._meta.primary_key._meta.name, - ) - ) - .where(self.m2m._meta.primary_foreign_key == self.row) - .output(as_list=True) - ) - - results = ( - await secondary_table.objects().where( - secondary_table._meta.primary_key.is_in(ids) + # use a subquery to make only one db query + results = await secondary_table.objects().where( + secondary_table._meta.primary_key.is_in( + joining_table.select( + getattr( + self.m2m._meta.secondary_foreign_key, + secondary_table._meta.primary_key._meta.name, + ) + ).where(self.m2m._meta.primary_foreign_key == self.row) ) - if len(ids) > 0 - else [] ) return results From 51242469afa158e7800a8604567187295f33edf2 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 18 Jul 2025 14:30:37 +0100 Subject: [PATCH 692/727] 1224 Make `Cast` more flexible (#1225) * wip * revert some changes * add type annotation back for table * improve docstring and types * remove `table` param docs --- piccolo/custom_types.py | 21 ++++++++++++ piccolo/query/functions/type_conversion.py | 39 +++++++++++++++------- 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/piccolo/custom_types.py b/piccolo/custom_types.py index ea8ee07f6..49f078076 100644 --- a/piccolo/custom_types.py +++ b/piccolo/custom_types.py @@ -1,8 +1,13 @@ from __future__ import annotations +import datetime +import decimal +import uuid from collections.abc import Iterable from typing import TYPE_CHECKING, Any, TypeVar, Union +from typing_extensions import TypeAlias + if TYPE_CHECKING: # pragma: no cover from piccolo.columns.combination import And, Or, Where, WhereRaw # noqa from piccolo.table import Table @@ -16,6 +21,22 @@ QueryResponseType = TypeVar("QueryResponseType", bound=Any) +# These are types we can reasonably expect to send to the database. +BasicTypes: TypeAlias = Union[ + bytes, + datetime.date, + datetime.datetime, + datetime.time, + datetime.timedelta, + decimal.Decimal, + dict, + float, + int, + list, + str, + uuid.UUID, +] + ############################################################################### # For backwards compatibility: diff --git a/piccolo/query/functions/type_conversion.py b/piccolo/query/functions/type_conversion.py index cc4bd8b39..1bbb44f72 100644 --- a/piccolo/query/functions/type_conversion.py +++ b/piccolo/query/functions/type_conversion.py @@ -1,13 +1,16 @@ +from __future__ import annotations + from typing import Optional, Union from piccolo.columns.base import Column +from piccolo.custom_types import BasicTypes from piccolo.querystring import QueryString class Cast(QueryString): def __init__( self, - identifier: Union[Column, QueryString], + identifier: Union[Column, QueryString, BasicTypes], as_type: Column, alias: Optional[str] = None, ): @@ -17,25 +20,36 @@ def __init__( >>> from piccolo.query.functions import Cast >>> await Concert.select( - ... Cast(Concert.starts, Time(), "start_time") + ... Cast(Concert.starts, Time(), alias="start_time") ... ) [{"start_time": datetime.time(19, 0)}] + You may also need ``Cast`` to explicitly tell the database which type + you're sending in the query (though this is an edge case). Here is a + contrived example:: + + >>> from piccolo.query.functions.math import Count + + # This fails with asyncpg: + >>> await Band.select(Count([1,2,3])) + + If we explicitly specify the type of the array, then it works:: + + >>> await Band.select( + ... Count( + ... Cast( + ... [1,2,3], + ... Array(Integer()) + ... ), + ... ) + ... ) + :param identifier: - Identifies what is being converted (e.g. a column). + Identifies what is being converted (e.g. a column, or a raw value). :param as_type: The type to be converted to. """ - # Make sure the identifier is a supported type. - - if not isinstance(identifier, (Column, QueryString)): - raise ValueError( - "The identifier is an unsupported type - only Column and " - "QueryString instances are allowed." - ) - - ####################################################################### # Convert `as_type` to a string which can be used in the query. if not isinstance(as_type, Column): @@ -44,6 +58,7 @@ def __init__( # We need to give the column a reference to a table, and hence # the database engine, as the column type is sometimes dependent # on which database is being used. + from piccolo.table import Table, create_table_class table: Optional[type[Table]] = None From 93cc9ce5616e8b4edf55fa5bf2734aef789d7493 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Tue, 29 Jul 2025 09:17:25 +0200 Subject: [PATCH 693/727] adds CockroachDB as an option to the Piccolo playground (#1229) * adds CockroachDB as an option to the Piccolo playground * make port optional * add @dantownsend suggestions * removed the orjson dependency for Lilya as it is no longer needed * minor tweaks to docs --------- Co-authored-by: Daniel Townsend --- .../piccolo/getting_started/playground.rst | 4 +- docs/src/piccolo/playground/advanced.rst | 50 +++++++++++++++++++ piccolo/apps/playground/commands/run.py | 38 +++++++++----- 3 files changed, 77 insertions(+), 15 deletions(-) diff --git a/docs/src/piccolo/getting_started/playground.rst b/docs/src/piccolo/getting_started/playground.rst index f24838e42..947da2b04 100644 --- a/docs/src/piccolo/getting_started/playground.rst +++ b/docs/src/piccolo/getting_started/playground.rst @@ -37,8 +37,8 @@ A ``piccolo.sqlite`` file will get created in the current directory. Advanced usage --------------- -To see how to use the playground with Postgres, and other advanced usage, see -:ref:`PlaygroundAdvanced`. +To see how to use the playground with Postgres or Cockroach, and other +advanced usage, see :ref:`PlaygroundAdvanced`. ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/playground/advanced.rst b/docs/src/piccolo/playground/advanced.rst index df090aaed..e8f459b9f 100644 --- a/docs/src/piccolo/playground/advanced.rst +++ b/docs/src/piccolo/playground/advanced.rst @@ -44,6 +44,56 @@ When you have the database setup, you can connect to it as follows: piccolo playground run --engine=postgres +CockroachDB +----------- + +If you want to use CockroachDB instead of SQLite, you need to create a database +first. + + +Install CockroachDB +~~~~~~~~~~~~~~~~~~~ + +See the `installation guide for your OS `_. + +Create database +~~~~~~~~~~~~~~~ +The playground is for testing and learning purposes only, so you can start a CockroachDB +`single node with the insecure flag `_ +(for non-production testing only) like this: + +.. code-block:: bash + + cockroach start-single-node --insecure + +After that, in a new terminal window, you can create a database like this: + +.. code-block:: bash + + cockroach sql --insecure --execute="DROP DATABASE IF EXISTS piccolo_playground CASCADE;CREATE DATABASE piccolo_playground;" + +By default the playground expects a local database to exist with the following +credentials: + + +.. code-block:: bash + + user: "root" + password: "" + host: "localhost" # or 127.0.0.1 + database: "piccolo_playground" + port: 26257 + + +Connecting +~~~~~~~~~~ + +When you have the database setup, you can connect to it as follows: + +.. code-block:: bash + + piccolo playground run --engine=cockroach + iPython ------- diff --git a/piccolo/apps/playground/commands/run.py b/piccolo/apps/playground/commands/run.py index 519d725b3..7e95f7e18 100644 --- a/piccolo/apps/playground/commands/run.py +++ b/piccolo/apps/playground/commands/run.py @@ -8,6 +8,7 @@ import uuid from decimal import Decimal from enum import Enum +from typing import Optional from piccolo.columns import ( JSON, @@ -25,7 +26,7 @@ Varchar, ) from piccolo.columns.readable import Readable -from piccolo.engine import PostgresEngine, SQLiteEngine +from piccolo.engine import CockroachEngine, PostgresEngine, SQLiteEngine from piccolo.engine.base import Engine from piccolo.table import Table from piccolo.utils.warnings import colored_string @@ -284,28 +285,29 @@ def populate(): def run( engine: str = "sqlite", - user: str = "piccolo", - password: str = "piccolo", + user: Optional[str] = None, + password: Optional[str] = None, database: str = "piccolo_playground", host: str = "localhost", - port: int = 5432, + port: Optional[int] = None, ipython_profile: bool = False, ): """ Creates a test database to play with. :param engine: - Which database engine to use - options are sqlite or postgres + Which database engine to use - options are sqlite, postgres or + cockroach :param user: - Postgres user + Database user (ignored for SQLite) :param password: - Postgres password + Database password (ignored for SQLite) :param database: - Postgres database + Database name (ignored for SQLite) :param host: - Postgres host + Database host (ignored for SQLite) :param port: - Postgres port + Database port (ignored for SQLite) :param ipython_profile: Set to true to use your own IPython profile. Located at ~/.ipython/. For more info see the IPython docs @@ -324,9 +326,19 @@ def run( { "host": host, "database": database, - "user": user, - "password": password, - "port": port, + "user": user or "piccolo", + "password": password or "piccolo", + "port": port or 5432, + } + ) + elif engine.upper() == "COCKROACH": + db = CockroachEngine( + { + "host": host, + "database": database, + "user": user or "root", + "password": password or "", + "port": port or 26257, } ) else: From 16276c68395ebb20d291fdea3bb6cb94e9d2f5ec Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 29 Jul 2025 09:09:21 +0100 Subject: [PATCH 694/727] bumped version --- CHANGES.rst | 80 +++++++++++++++++++++++++++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index ebdb631b1..7423ef065 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,86 @@ Changes ======= +1.28.0 +------ + +Playground improvements +~~~~~~~~~~~~~~~~~~~~~~~ + +* Added an ``Array`` column to the playground (``Album.awards``), for easier + experimentation with array columns. +* CoachroachDB is now supported in the playground (thanks to @sinisaos for this). + + .. code-block:: bash + + piccolo playground run --engine=cockroach + +Functions +~~~~~~~~~ + +Added lots of useful array functions (thanks to @sinisaos for this). + +Here's an example, where we can easily fix a typo in an array using ``replace``: + +.. code-block:: python + + >>> await Album.update({ + ... Album.awards: Album.awards.replace('Grammy Award 2021', 'Grammy Award 2022') + ... }, force=True) + +The documentation for functions has also been improved (e.g. how to create a +custom function). + +The ``Cast`` function is now more flexible. + +``Array`` concantenation +~~~~~~~~~~~~~~~~~~~~~~~~ + +Values can be prepended: + +.. code-block:: python + + >>> await Album.update({ + ... Album.awards: ['Grammy Award 2020'] + Album.awards + ... }, force=True) + +And multiple arrays can be concatenated in one go: + +.. code-block:: python + + >>> await Album.update({ + ... Album.awards: ['Grammy Award 2020'] + Album.awards + ['Grammy Award 2025'] + ... }, force=True) + +``is_in`` and ``not_in`` sub queries +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can now use sub queries within ``is_in`` and ``not_in`` Thanks to +@sinisaos for this. + +.. code-block:: python + + >>> await Band.select().where( + ... Band.id.is_in( + ... Concert.select(Concert.band_1).where( + ... Concert.starts >= datetime.datetime(year=2025, month=1, day=1) + ... ) + ... ) + ... ) + +Other improvements +~~~~~~~~~~~~~~~~~~ + +* Auto convert a default value of ``0`` to ``0.0`` in ``Float`` columns. +* Modernised the type hints throughout the codebase (e.g. using ``list`` + instead of ``typing.List``). Thanks to @sinisaos for this. +* Fixed a bug with auto migrations, where the ``Array`` base column class + wasn't being imported. +* Improved M2M query performance by using sub selects (thanks to @sinisaos for + this). + +------------------------------------------------------------------------------- + 1.27.1 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 4e863d10e..c9d2a61cc 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.27.1" +__VERSION__ = "1.28.0" From 9d5748d199d5bd28ba274a8116051f26c294591b Mon Sep 17 00:00:00 2001 From: sinisaos Date: Sun, 3 Aug 2025 20:57:35 +0200 Subject: [PATCH 695/727] tidy up Pydantic tests (#1236) --- scripts/test-cockroach.sh | 2 +- tests/utils/test_pydantic.py | 214 +++++++++++++++++++---------------- 2 files changed, 117 insertions(+), 99 deletions(-) diff --git a/scripts/test-cockroach.sh b/scripts/test-cockroach.sh index 9c8d23ca8..12b102448 100755 --- a/scripts/test-cockroach.sh +++ b/scripts/test-cockroach.sh @@ -5,7 +5,7 @@ # To run a single test tests/test_foo.py::TestFoo::test_foo export PICCOLO_CONF="tests.cockroach_conf" -python3 -m pytest \ +python -m pytest \ --cov=piccolo \ --cov-report=xml \ --cov-report=html \ diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index 7427fe44e..ebfd78843 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -29,10 +29,10 @@ class TestVarcharColumn(TestCase): def test_varchar_length(self): - class Director(Table): + class Manager(Table): name = Varchar(length=10) - pydantic_model = create_pydantic_model(table=Director) + pydantic_model = create_pydantic_model(table=Manager) with self.assertRaises(ValidationError): pydantic_model(name="This is a really long name") @@ -42,10 +42,10 @@ class Director(Table): class TestEmailColumn(TestCase): def test_email(self): - class Director(Table): + class Manager(Table): email = Email() - pydantic_model = create_pydantic_model(table=Director) + pydantic_model = create_pydantic_model(table=Manager) self.assertEqual( pydantic_model.model_json_schema()["properties"]["email"]["anyOf"][ @@ -67,28 +67,28 @@ class TestNumericColumn(TestCase): """ def test_numeric_digits(self): - class Movie(Table): - box_office = Numeric(digits=(5, 1)) + class Band(Table): + royalties = Numeric(digits=(5, 1)) - pydantic_model = create_pydantic_model(table=Movie) + pydantic_model = create_pydantic_model(table=Band) with self.assertRaises(ValidationError): # This should fail as there are too much numbers after the decimal # point - pydantic_model(box_office=decimal.Decimal("1.11")) + pydantic_model(royalties=decimal.Decimal("1.11")) with self.assertRaises(ValidationError): # This should fail as there are too much numbers in total - pydantic_model(box_office=decimal.Decimal("11111.1")) + pydantic_model(royalties=decimal.Decimal("11111.1")) - pydantic_model(box_office=decimal.Decimal("1.0")) + pydantic_model(royalties=decimal.Decimal("1.0")) def test_numeric_without_digits(self): - class Movie(Table): - box_office = Numeric() + class Band(Table): + royalties = Numeric() try: - create_pydantic_model(table=Movie) + create_pydantic_model(table=Band) except TypeError: self.fail( "Creating numeric field without" @@ -297,13 +297,13 @@ class TestColumnHelpText(TestCase): def test_column_help_text_present(self): help_text = "In millions of US dollars." - class Movie(Table): - box_office = Numeric(digits=(5, 1), help_text=help_text) + class Band(Table): + royalties = Numeric(digits=(5, 1), help_text=help_text) - pydantic_model = create_pydantic_model(table=Movie) + pydantic_model = create_pydantic_model(table=Band) self.assertEqual( - pydantic_model.model_json_schema()["properties"]["box_office"][ + pydantic_model.model_json_schema()["properties"]["royalties"][ "extra" ]["help_text"], help_text, @@ -317,12 +317,12 @@ class TestTableHelpText(TestCase): """ def test_table_help_text_present(self): - help_text = "Movies which were released in cinemas." + help_text = "Bands playing concerts." - class Movie(Table, help_text=help_text): + class Band(Table, help_text=help_text): name = Varchar() - pydantic_model = create_pydantic_model(table=Movie) + pydantic_model = create_pydantic_model(table=Band) self.assertEqual( pydantic_model.model_json_schema()["extra"]["help_text"], @@ -332,10 +332,10 @@ class Movie(Table, help_text=help_text): class TestUniqueColumn(TestCase): def test_unique_column_true(self): - class Director(Table): + class Manager(Table): name = Varchar(unique=True) - pydantic_model = create_pydantic_model(table=Director) + pydantic_model = create_pydantic_model(table=Manager) self.assertEqual( pydantic_model.model_json_schema()["properties"]["name"]["extra"][ @@ -345,10 +345,10 @@ class Director(Table): ) def test_unique_column_false(self): - class Director(Table): + class Manager(Table): name = Varchar() - pydantic_model = create_pydantic_model(table=Director) + pydantic_model = create_pydantic_model(table=Manager) self.assertEqual( pydantic_model.model_json_schema()["properties"]["name"]["extra"][ @@ -360,48 +360,66 @@ class Director(Table): class TestJSONColumn(TestCase): def test_default(self): - class Movie(Table): - meta = JSON() - meta_b = JSONB() + class Studio(Table): + facilities = JSON() + facilities_b = JSONB() - pydantic_model = create_pydantic_model(table=Movie) + pydantic_model = create_pydantic_model(table=Studio) - json_string = '{"code": 12345}' + json_string = '{"guitar_amps": 6}' - model_instance = pydantic_model(meta=json_string, meta_b=json_string) - self.assertEqual(model_instance.meta, json_string) # type: ignore - self.assertEqual(model_instance.meta_b, json_string) # type: ignore + model_instance = pydantic_model( + facilities=json_string, facilities_b=json_string + ) + self.assertEqual( + model_instance.facilities, + json_string, + ) + self.assertEqual( + model_instance.facilities_b, + json_string, + ) def test_deserialize_json(self): - class Movie(Table): - meta = JSON() - meta_b = JSONB() + class Studio(Table): + facilities = JSON() + facilities_b = JSONB() pydantic_model = create_pydantic_model( - table=Movie, deserialize_json=True + table=Studio, deserialize_json=True ) - json_string = '{"code": 12345}' - output = {"code": 12345} + json_string = '{"guitar_amps": 6}' + output = {"guitar_amps": 6} - model_instance = pydantic_model(meta=json_string, meta_b=json_string) - self.assertEqual(model_instance.meta, output) # type: ignore - self.assertEqual(model_instance.meta_b, output) # type: ignore + model_instance = pydantic_model( + facilities=json_string, facilities_b=json_string + ) + self.assertEqual( + model_instance.facilities, + output, + ) + self.assertEqual( + model_instance.facilities_b, + output, + ) def test_validation(self): - class Movie(Table): - meta = JSON() - meta_b = JSONB() + class Studio(Table): + facilities = JSON() + facilities_b = JSONB() for deserialize_json in (True, False): pydantic_model = create_pydantic_model( - table=Movie, deserialize_json=deserialize_json + table=Studio, deserialize_json=deserialize_json ) json_string = "error" with self.assertRaises(pydantic.ValidationError): - pydantic_model(meta=json_string, meta_b=json_string) + pydantic_model( + facilities=json_string, facilities_b=json_string + ) def test_json_widget(self): """ @@ -409,112 +427,112 @@ def test_json_widget(self): special widget in Piccolo Admin. """ - class Movie(Table): - features = JSON() + class Studio(Table): + facilities = JSON() - pydantic_model = create_pydantic_model(table=Movie) + pydantic_model = create_pydantic_model(table=Studio) self.assertEqual( - pydantic_model.model_json_schema()["properties"]["features"][ + pydantic_model.model_json_schema()["properties"]["facilities"][ "extra" ]["widget"], "json", ) def test_null_value(self): - class Movie(Table): - meta = JSON(null=True) - meta_b = JSONB(null=True) + class Studio(Table): + facilities = JSON(null=True) + facilities_b = JSONB(null=True) - pydantic_model = create_pydantic_model(table=Movie) - movie = pydantic_model(meta=None, meta_b=None) + pydantic_model = create_pydantic_model(table=Studio) + movie = pydantic_model(facilities=None, facilities_b=None) - self.assertIsNone(movie.meta) # type: ignore - self.assertIsNone(movie.meta_b) # type: ignore + self.assertIsNone(movie.facilities) + self.assertIsNone(movie.facilities_b) class TestExcludeColumns(TestCase): def test_all(self): - class Computer(Table): - CPU = Varchar() - GPU = Varchar() + class Band(Table): + name = Varchar() + bio = Text() - pydantic_model = create_pydantic_model(Computer, exclude_columns=()) + pydantic_model = create_pydantic_model(Band, exclude_columns=()) properties = pydantic_model.model_json_schema()["properties"] - self.assertIsInstance(properties["GPU"], dict) - self.assertIsInstance(properties["CPU"], dict) + self.assertIsInstance(properties["name"], dict) + self.assertIsInstance(properties["bio"], dict) def test_exclude(self): - class Computer(Table): - CPU = Varchar() - GPU = Varchar() + class Band(Table): + name = Varchar() + album = Varchar() pydantic_model = create_pydantic_model( - Computer, - exclude_columns=(Computer.CPU,), + Band, + exclude_columns=(Band.name,), ) properties = pydantic_model.model_json_schema()["properties"] - self.assertIsInstance(properties.get("GPU"), dict) - self.assertIsNone(properties.get("CPU")) + self.assertIsInstance(properties.get("album"), dict) + self.assertIsNone(properties.get("dict")) def test_exclude_all_manually(self): - class Computer(Table): - GPU = Varchar() - CPU = Varchar() + class Band(Table): + name = Varchar() + album = Varchar() pydantic_model = create_pydantic_model( - Computer, - exclude_columns=(Computer.GPU, Computer.CPU), + Band, + exclude_columns=(Band.name, Band.album), ) self.assertEqual(pydantic_model.model_json_schema()["properties"], {}) def test_exclude_all_meta(self): - class Computer(Table): - GPU = Varchar() - CPU = Varchar() + class Band(Table): + name = Varchar() + album = Varchar() pydantic_model = create_pydantic_model( - Computer, - exclude_columns=tuple(Computer._meta.columns), + Band, + exclude_columns=tuple(Band._meta.columns), ) self.assertEqual(pydantic_model.model_json_schema()["properties"], {}) def test_invalid_column_str(self): - class Computer(Table): - CPU = Varchar() - GPU = Varchar() + class Band(Table): + name = Varchar() + album = Varchar() with self.assertRaises(ValueError): create_pydantic_model( - Computer, - exclude_columns=("CPU",), # type: ignore + Band, + exclude_columns=("album",), ) def test_invalid_column_different_table(self): - class Computer(Table): - CPU = Varchar() - GPU = Varchar() + class Band(Table): + name = Varchar() + album = Varchar() - class Computer2(Table): - SSD = Varchar() + class Band2(Table): + photo = Varchar() with self.assertRaises(ValueError): - create_pydantic_model(Computer, exclude_columns=(Computer2.SSD,)) + create_pydantic_model(Band, exclude_columns=(Band2.photo,)) def test_invalid_column_different_table_same_type(self): - class Computer(Table): - CPU = Varchar() - GPU = Varchar() + class Band(Table): + name = Varchar() + album = Varchar() - class Computer2(Table): - CPU = Varchar() + class Band2(Table): + name = Varchar() with self.assertRaises(ValueError): - create_pydantic_model(Computer, exclude_columns=(Computer2.CPU,)) + create_pydantic_model(Band, exclude_columns=(Band2.name,)) def test_exclude_nested(self): class Manager(Table): @@ -892,7 +910,7 @@ class Band(Table): model = BandModel(regrettable_column_name="test") - self.assertEqual(model.name, "test") # type: ignore + self.assertEqual(model.name, "test") class TestJSONSchemaExtra(TestCase): From b5e022a8423a278ed0e3e4e7498187a5c3077e3a Mon Sep 17 00:00:00 2001 From: sinisaos Date: Sun, 3 Aug 2025 21:27:34 +0200 Subject: [PATCH 696/727] Adds a column with the argument ``secret=True`` to ``Table._meta.secret_columns`` (#1235) * adds a column with the argument secret=True to the table meta secret columns * adds @dantownsend sugestions * tweak tests --------- Co-authored-by: Daniel Townsend --- piccolo/table.py | 11 +++++------ tests/table/test_metaclass.py | 10 +++++++--- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/piccolo/table.py b/piccolo/table.py index e4ddd7daf..cda091064 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -16,7 +16,6 @@ Email, ForeignKey, ReferencedTable, - Secret, Serial, ) from piccolo.columns.defaults.base import Default @@ -84,7 +83,7 @@ class TableMeta: foreign_key_columns: list[ForeignKey] = field(default_factory=list) primary_key: Column = field(default_factory=Column) json_columns: list[Union[JSON, JSONB]] = field(default_factory=list) - secret_columns: list[Secret] = field(default_factory=list) + secret_columns: list[Column] = field(default_factory=list) auto_update_columns: list[Column] = field(default_factory=list) tags: list[str] = field(default_factory=list) help_text: Optional[str] = None @@ -274,7 +273,7 @@ def __init_subclass__( non_default_columns: list[Column] = [] array_columns: list[Array] = [] foreign_key_columns: list[ForeignKey] = [] - secret_columns: list[Secret] = [] + secret_columns: list[Column] = [] json_columns: list[Union[JSON, JSONB]] = [] email_columns: list[Email] = [] auto_update_columns: list[Column] = [] @@ -315,15 +314,15 @@ def __init_subclass__( if isinstance(column, Email): email_columns.append(column) - if isinstance(column, Secret): - secret_columns.append(column) - if isinstance(column, ForeignKey): foreign_key_columns.append(column) if isinstance(column, (JSON, JSONB)): json_columns.append(column) + if column._meta.secret: + secret_columns.append(column) + if column._meta.auto_update is not ...: auto_update_columns.append(column) diff --git a/tests/table/test_metaclass.py b/tests/table/test_metaclass.py index 4af11d1aa..7ff186d73 100644 --- a/tests/table/test_metaclass.py +++ b/tests/table/test_metaclass.py @@ -1,13 +1,13 @@ from unittest import TestCase from unittest.mock import MagicMock, patch -from piccolo.columns import Secret from piccolo.columns.column_types import ( JSON, JSONB, Array, Email, ForeignKey, + Secret, Varchar, ) from piccolo.table import TABLENAME_WARNING, Table @@ -99,14 +99,18 @@ class TableB(Table): def test_secret_columns(self): """ - Make sure TableMeta.secret_columns are setup correctly. + Make sure TableMeta.secret_columns are setup correctly with the + ``secret=True`` argument and ``Secret`` column type. """ class Classified(Table): top_secret = Secret() + confidential = Varchar(secret=True) + public = Varchar() self.assertEqual( - Classified._meta.secret_columns, [Classified.top_secret] + Classified._meta.secret_columns, + [Classified.top_secret, Classified.confidential], ) def test_json_columns(self): From 90e2eeb3c11807c4c7bfa1e07c4381fceff0cf72 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Sun, 3 Aug 2025 21:39:05 +0200 Subject: [PATCH 697/727] Documenting ``MigrationManager.get_table_from_snapshot`` (#1232) * documenting MigrationManager.get_table_from_snapshot * adds @dantownsend sugestions * tweak docs - add description to new migration --------- Co-authored-by: Daniel Townsend --- docs/src/piccolo/migrations/create.rst | 48 +++++++++++++++++++++----- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/docs/src/piccolo/migrations/create.rst b/docs/src/piccolo/migrations/create.rst index c769f3c1c..486c46165 100644 --- a/docs/src/piccolo/migrations/create.rst +++ b/docs/src/piccolo/migrations/create.rst @@ -79,8 +79,8 @@ If you want to run raw SQL within your migration, you can do so as follows: from piccolo.table import Table - ID = "2022-02-26T17:38:44:758593" - VERSION = "0.69.2" + ID = "2025-07-28T09:51:54:296860" + VERSION = "1.27.1" DESCRIPTION = "Updating each band's popularity" @@ -136,8 +136,8 @@ We have to be quite careful with this. Here's an example: from music.tables import Band - ID = "2022-02-26T17:38:44:758593" - VERSION = "0.69.2" + ID = "2025-07-28T09:51:54:296860" + VERSION = "1.27.1" DESCRIPTION = "Updating each band's popularity" @@ -149,7 +149,7 @@ We have to be quite careful with this. Here's an example: ) async def run(): - await Band.update({Band.popularity: 1000}) + await Band.update({Band.popularity: 1000}, force=True) manager.add_raw(run) return manager @@ -175,8 +175,8 @@ it's better to copy the relevant tables into your migration file: from piccolo.table import Table - ID = "2022-02-26T17:38:44:758593" - VERSION = "0.69.2" + ID = "2025-07-28T09:51:54:296860" + VERSION = "1.27.1" DESCRIPTION = "Updating each band's popularity" @@ -193,9 +193,41 @@ it's better to copy the relevant tables into your migration file: ) async def run(): - await Band.update({Band.popularity: 1000}) + await Band.update({Band.popularity: 1000}, force=True) + + manager.add_raw(run) + return manager + +Another alternative is to use the ``MigrationManager.get_table_from_snapshot`` +method to get a table from the migration history. This is very convenient, +especially if the table is large, with many foreign keys. + +.. code-block:: python + + from piccolo.apps.migrations.auto.migration_manager import MigrationManager + + + ID = "2025-07-28T09:51:54:296860" + VERSION = "1.27.1" + DESCRIPTION = "Updating each band's popularity" + + + async def forwards(): + manager = MigrationManager( + migration_id=ID, + app_name="", + description=DESCRIPTION + ) + + async def run(): + # We get a table from the migration history. + Band = await manager.get_table_from_snapshot( + app_name="music", table_class_name="Band" + ) + await Band.update({"popularity": 1000}, force=True) manager.add_raw(run) + return manager ------------------------------------------------------------------------------- From 2e1e6f97a423a88c81c674b9f23d6d8ceccfe316 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Wed, 6 Aug 2025 01:03:54 +0200 Subject: [PATCH 698/727] adds m2m column to the playground (#1234) * adds m2m column to the playground * adds @dantownsend sugestions * add a reason --------- Co-authored-by: Daniel Townsend --- piccolo/apps/playground/commands/run.py | 43 ++++++++++++++++ tests/columns/m2m/base.py | 65 +++++++------------------ 2 files changed, 61 insertions(+), 47 deletions(-) diff --git a/piccolo/apps/playground/commands/run.py b/piccolo/apps/playground/commands/run.py index 7e95f7e18..670dcd664 100644 --- a/piccolo/apps/playground/commands/run.py +++ b/piccolo/apps/playground/commands/run.py @@ -12,6 +12,7 @@ from piccolo.columns import ( JSON, + M2M, UUID, Array, Boolean, @@ -19,6 +20,7 @@ ForeignKey, Integer, Interval, + LazyTableReference, Numeric, Serial, Text, @@ -49,6 +51,7 @@ class Band(Table): name = Varchar(length=50) manager = ForeignKey(references=Manager, null=True) popularity = Integer() + genres = M2M(LazyTableReference("GenreToBand", module_path=__name__)) @classmethod def get_readable(cls) -> Readable: @@ -161,6 +164,26 @@ def get_readable(cls) -> Readable: ) +class Genre(Table): + id: Serial + name = Varchar() + bands = M2M(LazyTableReference("GenreToBand", module_path=__name__)) + + @classmethod + def get_readable(cls) -> Readable: + return Readable( + template="%s", + columns=[cls.name], + ) + + +class GenreToBand(Table): + id: Serial + band = ForeignKey(Band) + genre = ForeignKey(Genre) + reason = Text(null=True, default=None) + + TABLES = ( Manager, Band, @@ -171,6 +194,8 @@ def get_readable(cls) -> Readable: DiscountCode, RecordingStudio, Album, + Genre, + GenreToBand, ) @@ -282,6 +307,24 @@ def populate(): ), ).run_sync() + genres = Genre.insert( + Genre(name="Rock"), + Genre(name="Classical"), + Genre(name="Folk"), + ).run_sync() + + GenreToBand.insert( + GenreToBand( + band=pythonistas.id, + genre=genres[0]["id"], + reason="Because they rock.", + ), + GenreToBand(band=pythonistas.id, genre=genres[2]["id"]), + GenreToBand(band=rustaceans.id, genre=genres[2]["id"]), + GenreToBand(band=c_sharps.id, genre=genres[0]["id"]), + GenreToBand(band=c_sharps.id, genre=genres[1]["id"]), + ).run_sync() + def run( engine: str = "sqlite", diff --git a/tests/columns/m2m/base.py b/tests/columns/m2m/base.py index 8e6e477aa..066ebab11 100644 --- a/tests/columns/m2m/base.py +++ b/tests/columns/m2m/base.py @@ -11,7 +11,7 @@ from piccolo.engine.finder import engine_finder from piccolo.schema import SchemaManager from piccolo.table import Table, create_db_tables_sync, drop_db_tables_sync -from tests.base import engine_is, engines_skip +from tests.base import engines_skip engine = engine_finder() @@ -51,54 +51,25 @@ def _setUp(self, schema: Optional[str] = None): create_db_tables_sync(*self.all_tables, if_not_exists=True) - if engine_is("cockroach"): - bands = ( - Band.insert( - Band(name="Pythonistas"), - Band(name="Rustaceans"), - Band(name="C-Sharps"), - ) - .returning(Band.id) - .run_sync() - ) - - genres = ( - Genre.insert( - Genre(name="Rock"), - Genre(name="Folk"), - Genre(name="Classical"), - ) - .returning(Genre.id) - .run_sync() - ) - - GenreToBand.insert( - GenreToBand(band=bands[0]["id"], genre=genres[0]["id"]), - GenreToBand(band=bands[0]["id"], genre=genres[1]["id"]), - GenreToBand(band=bands[1]["id"], genre=genres[1]["id"]), - GenreToBand(band=bands[2]["id"], genre=genres[0]["id"]), - GenreToBand(band=bands[2]["id"], genre=genres[2]["id"]), - ).run_sync() - else: - Band.insert( - Band(name="Pythonistas"), - Band(name="Rustaceans"), - Band(name="C-Sharps"), - ).run_sync() + bands = Band.insert( + Band(name="Pythonistas"), + Band(name="Rustaceans"), + Band(name="C-Sharps"), + ).run_sync() - Genre.insert( - Genre(name="Rock"), - Genre(name="Folk"), - Genre(name="Classical"), - ).run_sync() + genres = Genre.insert( + Genre(name="Rock"), + Genre(name="Folk"), + Genre(name="Classical"), + ).run_sync() - GenreToBand.insert( - GenreToBand(band=1, genre=1), - GenreToBand(band=1, genre=2), - GenreToBand(band=2, genre=2), - GenreToBand(band=3, genre=1), - GenreToBand(band=3, genre=3), - ).run_sync() + GenreToBand.insert( + GenreToBand(band=bands[0]["id"], genre=genres[0]["id"]), + GenreToBand(band=bands[0]["id"], genre=genres[1]["id"]), + GenreToBand(band=bands[1]["id"], genre=genres[1]["id"]), + GenreToBand(band=bands[2]["id"], genre=genres[0]["id"]), + GenreToBand(band=bands[2]["id"], genre=genres[2]["id"]), + ).run_sync() def tearDown(self): drop_db_tables_sync(*self.all_tables) From a4870e7848bc750a4fe0dbc65abfe846db783edf Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 8 Aug 2025 23:09:57 +0100 Subject: [PATCH 699/727] Document the `migration` table (#516) * add `fake` argument to `MigrationManager` * added docs about fake running migrations * add `ForeignKey` to type annotation * tweak docs --- docs/src/piccolo/migrations/running.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/src/piccolo/migrations/running.rst b/docs/src/piccolo/migrations/running.rst index 8979a70b1..5b00db3f6 100644 --- a/docs/src/piccolo/migrations/running.rst +++ b/docs/src/piccolo/migrations/running.rst @@ -13,7 +13,6 @@ When the migration is run, the forwards function is executed. To do this: piccolo migrations forwards my_app - Multiple apps ~~~~~~~~~~~~~ @@ -23,6 +22,13 @@ If you have multiple apps you can run them all using: piccolo migrations forwards all +Migration table +~~~~~~~~~~~~~~~ + +When running the migrations, Piccolo will automatically create a database table +called ``migration`` if it doesn't already exist. Each time a migration is +succesfully ran, a new row is added to this table. + .. _FakeMigration: Fake From 5139ec0644675bc3b195d46de2d50571cfa9e4c7 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Sun, 10 Aug 2025 00:19:32 +0200 Subject: [PATCH 700/727] Update asgi templates (#1243) * update FastAPI template * update BlackSheep template * update Litestar template * update Esmerald template * update Quart template * update Falcon template because PiccoloCRUD is not working properly * update Sanic template * status code has been changed from 204 to 200 because Quart and Sanic do not work well on the delete endpoint with no content * removed redundant tags from Esmerald's single task endpoint * updating docs for supported frameworks * sort imports --------- Co-authored-by: Daniel Townsend --- docs/src/index.rst | 6 +- .../templates/app/_blacksheep_app.py.jinja | 86 ++++++++++++------- .../templates/app/_esmerald_app.py.jinja | 69 ++++++++++----- .../templates/app/_falcon_app.py.jinja | 71 +++++++++++++-- .../templates/app/_fastapi_app.py.jinja | 75 ++++++++++------ .../templates/app/_litestar_app.py.jinja | 44 +++++++--- .../templates/app/_quart_app.py.jinja | 53 ++++++++---- .../templates/app/_sanic_app.py.jinja | 45 +++++++--- 8 files changed, 328 insertions(+), 121 deletions(-) diff --git a/docs/src/index.rst b/docs/src/index.rst index 20b48aa5f..849694a21 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -60,10 +60,10 @@ Give me an ASGI web app! piccolo asgi new -FastAPI, Starlette, BlackSheep, Litestar, Esmerald and Lilya are currently supported, -with more coming soon. +FastAPI, Starlette, BlackSheep, Litestar, Esmerald, Lilya, Quart, Falcon and Sanic +are currently supported, with more coming soon. -------------------------------------------------------------------------------- +---------------------------------------------------------------------------------- Videos ------ diff --git a/piccolo/apps/asgi/commands/templates/app/_blacksheep_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_blacksheep_app.py.jinja index d2e9e4d8b..7873fa47c 100644 --- a/piccolo/apps/asgi/commands/templates/app/_blacksheep_app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/_blacksheep_app.py.jinja @@ -1,10 +1,9 @@ -import typing as t +from typing import Any +from blacksheep.exceptions import HTTPException from blacksheep.server import Application from blacksheep.server.bindings import FromJSON -from blacksheep.server.openapi.v3 import OpenAPIHandler -from blacksheep.server.responses import json -from openapidocs.v3 import Info +from blacksheep.server.openapi.v3 import OpenAPIHandler, Info from piccolo.engine import engine_finder from piccolo_admin.endpoints import create_admin from piccolo_api.crud.serializers import create_pydantic_model @@ -34,38 +33,67 @@ app.serve_files("static", root_path="/static") app.router.add_get("/", home) -TaskModelIn: t.Any = create_pydantic_model(table=Task, model_name="TaskModelIn") -TaskModelOut: t.Any = create_pydantic_model( - table=Task, include_default_columns=True, model_name="TaskModelOut" +TaskModelIn: Any = create_pydantic_model( + table=Task, + model_name="TaskModelIn", ) -TaskModelPartial: t.Any = create_pydantic_model( - table=Task, model_name="TaskModelPartial", all_optional=True +TaskModelOut: Any = create_pydantic_model( + table=Task, + include_default_columns=True, + model_name="TaskModelOut", ) +TaskModelPartial: Any = ( + create_pydantic_model( + table=Task, + model_name="TaskModelPartial", + all_optional=True, + ), +) + + +# Check if the record is None. Use for query callback +def check_record_not_found(result: dict[str, Any]) -> dict[str, Any]: + if result is None: + raise HTTPException(status=404) + return result @app.router.get("/tasks/") -async def tasks() -> t.List[TaskModelOut]: - return await Task.select().order_by(Task._meta.primary_key, ascending=False) +async def tasks() -> list[TaskModelOut]: + tasks = await Task.select().order_by(Task._meta.primary_key, ascending=False) + return [TaskModelOut(**task) for task in tasks] + + +@app.router.get("/tasks/{task_id}/") +async def single_task(task_id: int) -> TaskModelOut: + task = ( + await Task.select() + .where(Task._meta.primary_key == task_id) + .first() + .callback(check_record_not_found) + ) + return TaskModelOut(**task) @app.router.post("/tasks/") async def create_task(task_model: FromJSON[TaskModelIn]) -> TaskModelOut: - task = Task(**task_model.value.dict()) + task = Task(**task_model.value.model_dump()) await task.save() return TaskModelOut(**task.to_dict()) @app.router.put("/tasks/{task_id}/") async def put_task(task_id: int, task_model: FromJSON[TaskModelIn]) -> TaskModelOut: - task = await Task.objects().get(Task._meta.primary_key == task_id) - if not task: - return json({}, status=404) + task = ( + await Task.objects() + .get(Task._meta.primary_key == task_id) + .callback(check_record_not_found) + ) - for key, value in task_model.value.dict().items(): + for key, value in task_model.value.model_dump().items(): setattr(task, key, value) await task.save() - return TaskModelOut(**task.to_dict()) @@ -73,29 +101,29 @@ async def put_task(task_id: int, task_model: FromJSON[TaskModelIn]) -> TaskModel async def patch_task( task_id: int, task_model: FromJSON[TaskModelPartial] ) -> TaskModelOut: - task = await Task.objects().get(Task._meta.primary_key == task_id) - if not task: - return json({}, status=404) + task = ( + await Task.objects() + .get(Task._meta.primary_key == task_id) + .callback(check_record_not_found) + ) - for key, value in task_model.value.dict().items(): + for key, value in task_model.value.model_dump().items(): if value is not None: setattr(task, key, value) await task.save() - return TaskModelOut(**task.to_dict()) @app.router.delete("/tasks/{task_id}/") -async def delete_task(task_id: int): - task = await Task.objects().get(Task._meta.primary_key == task_id) - if not task: - return json({}, status=404) - +async def delete_task(task_id: int) -> None: + task = ( + await Task.objects() + .get(Task._meta.primary_key == task_id) + .callback(check_record_not_found) + ) await task.remove() - return json({}) - async def open_database_connection_pool(application): try: diff --git a/piccolo/apps/asgi/commands/templates/app/_esmerald_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_esmerald_app.py.jinja index 01e6dfe45..3556eba14 100644 --- a/piccolo/apps/asgi/commands/templates/app/_esmerald_app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/_esmerald_app.py.jinja @@ -1,12 +1,12 @@ -import typing as t +from typing import Any from pathlib import Path from esmerald import ( APIView, Esmerald, Gateway, + HTTPException, Include, - JSONResponse, delete, get, post, @@ -38,45 +38,72 @@ async def close_database_connection_pool(): print("Unable to connect to the database") -TaskModelIn: t.Any = create_pydantic_model(table=Task, model_name="TaskModelIn") -TaskModelOut: t.Any = create_pydantic_model( - table=Task, include_default_columns=True, model_name="TaskModelOut" +TaskModelIn: Any = create_pydantic_model( + table=Task, + model_name="TaskModelIn", ) +TaskModelOut: Any = create_pydantic_model( + table=Task, + include_default_columns=True, + model_name="TaskModelOut", +) + + +# Check if the record is None. Use for query callback +def check_record_not_found(result: dict[str, Any]) -> dict[str, Any]: + if result is None: + raise HTTPException( + detail="Record not found", + status_code=404, + ) + return result class TaskAPIView(APIView): path: str = "/" - tags: str = ["Task"] + tags: list[str] = ["Task"] @get("/") - async def tasks(self) -> t.List[TaskModelOut]: - return await Task.select().order_by(Task._meta.primary_key, ascending=False) + async def tasks(self) -> list[TaskModelOut]: + tasks = await Task.select().order_by(Task._meta.primary_key, ascending=False) + return [TaskModelOut(**task) for task in tasks] + + @get("/{task_id}") + async def single_task(self, task_id: int) -> TaskModelOut: + task = ( + await Task.select() + .where(Task._meta.primary_key == task_id) + .first() + .callback(check_record_not_found) + ) + return TaskModelOut(**task) @post("/") async def create_task(self, payload: TaskModelIn) -> TaskModelOut: - task = Task(**payload.dict()) + task = Task(**payload.model_dump()) await task.save() - return task.to_dict() + return TaskModelOut(**task.to_dict()) @put("/{task_id}") async def update_task(self, payload: TaskModelIn, task_id: int) -> TaskModelOut: - task = await Task.objects().get(Task._meta.primary_key == task_id) - if not task: - return JSONResponse({}, status_code=404) - - for key, value in payload.dict().items(): + task = ( + await Task.objects() + .get(Task._meta.primary_key == task_id) + .callback(check_record_not_found) + ) + for key, value in payload.model_dump().items(): setattr(task, key, value) await task.save() - - return task.to_dict() + return TaskModelOut(**task.to_dict()) @delete("/{task_id}") async def delete_task(self, task_id: int) -> None: - task = await Task.objects().get(Task._meta.primary_key == task_id) - if not task: - return JSONResponse({}, status_code=404) - + task = ( + await Task.objects() + .get(Task._meta.primary_key == task_id) + .callback(check_record_not_found) + ) await task.remove() diff --git a/piccolo/apps/asgi/commands/templates/app/_falcon_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_falcon_app.py.jinja index 0827debb4..1c5918999 100644 --- a/piccolo/apps/asgi/commands/templates/app/_falcon_app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/_falcon_app.py.jinja @@ -1,11 +1,10 @@ import os -import typing as t +from typing import Any import falcon.asgi from hypercorn.middleware import DispatcherMiddleware from piccolo.engine import engine_finder from piccolo_admin.endpoints import create_admin -from piccolo_api.crud.endpoints import PiccoloCRUD from home.endpoints import HomeEndpoint from home.piccolo_app import APP_CONFIG @@ -28,25 +27,82 @@ async def close_database_connection_pool(): print("Unable to connect to the database") +# Check if the record is None. Use for query callback +def check_record_not_found(result: dict[str, Any]) -> dict[str, Any]: + if result is None: + raise falcon.HTTPNotFound() + return result + + class LifespanMiddleware: async def process_startup( - self, scope: t.Dict[str, t.Any], event: t.Dict[str, t.Any] + self, scope: dict[str, Any], event: dict[str, Any] ) -> None: await open_database_connection_pool() async def process_shutdown( - self, scope: t.Dict[str, t.Any], event: t.Dict[str, t.Any] + self, scope: dict[str, Any], event: dict[str, Any] ) -> None: await close_database_connection_pool() -app: t.Any = falcon.asgi.App(middleware=LifespanMiddleware()) +class TaskCollectionResource: + async def on_get(self, req, resp): + tasks = await Task.select().order_by(Task._meta.primary_key, ascending=False) + resp.media = tasks + + async def on_post(self, req, resp): + data = await req.media + task = Task(**data) + await task.save() + resp.status = falcon.HTTP_201 + resp.media = task.to_dict() + + +class TaskItemResource: + async def on_get(self, req, resp, task_id): + task = ( + await Task.select() + .where(Task._meta.primary_key == task_id) + .first() + .callback(check_record_not_found) + ) + resp.status = falcon.HTTP_200 + resp.media = task + + async def on_put(self, req, resp, task_id): + task = ( + await Task.objects() + .get(Task._meta.primary_key == task_id) + .callback(check_record_not_found) + ) + + data = await req.media + for key, value in data.items(): + setattr(task, key, value) + + await task.save() + resp.status = falcon.HTTP_200 + resp.media = task.to_dict() + + async def on_delete(self, req, resp, task_id): + task = ( + await Task.objects() + .get(Task._meta.primary_key == task_id) + .callback(check_record_not_found) + ) + resp.status = falcon.HTTP_204 + await task.remove() + + +app: Any = falcon.asgi.App(middleware=LifespanMiddleware()) app.add_static_route("/static", directory=os.path.abspath("static")) app.add_route("/", HomeEndpoint()) +app.add_route("/tasks/", TaskCollectionResource()) +app.add_route("/tasks/{task_id:int}", TaskItemResource()) -PICCOLO_CRUD: t.Any = PiccoloCRUD(table=Task) -# enable the Admin and PiccoloCrud app using DispatcherMiddleware +# enable the admin application using DispatcherMiddleware app = DispatcherMiddleware( # type: ignore { "/admin": create_admin( @@ -54,7 +110,6 @@ app = DispatcherMiddleware( # type: ignore # Required when running under HTTPS: # allowed_hosts=['my_site.com'] ), - "/tasks": PICCOLO_CRUD, "": app, } ) diff --git a/piccolo/apps/asgi/commands/templates/app/_fastapi_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_fastapi_app.py.jinja index 0fd9efac0..7a096528a 100644 --- a/piccolo/apps/asgi/commands/templates/app/_fastapi_app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/_fastapi_app.py.jinja @@ -1,8 +1,8 @@ -import typing as t from contextlib import asynccontextmanager +from typing import Any -from fastapi import FastAPI -from fastapi.responses import JSONResponse +from fastapi import FastAPI, status +from fastapi.exceptions import HTTPException from piccolo.engine import engine_finder from piccolo_admin.endpoints import create_admin from piccolo_api.crud.serializers import create_pydantic_model @@ -54,47 +54,74 @@ app = FastAPI( ) -TaskModelIn: t.Any = create_pydantic_model( +TaskModelIn: Any = create_pydantic_model( table=Task, model_name="TaskModelIn", ) -TaskModelOut: t.Any = create_pydantic_model( - table=Task, include_default_columns=True, model_name="TaskModelOut" + +TaskModelOut: Any = create_pydantic_model( + table=Task, + include_default_columns=True, + model_name="TaskModelOut", ) -@app.get("/tasks/", response_model=t.List[TaskModelOut]) +# Check if the record is None. Use for query callback +def check_record_not_found(result: dict[str, Any]) -> dict[str, Any]: + if result is None: + raise HTTPException( + detail="Record not found", + status_code=status.HTTP_404_NOT_FOUND, + ) + return result + + +@app.get("/tasks/", response_model=list[TaskModelOut], tags=["Task"]) async def tasks(): - return await Task.select().order_by(Task.id) + return await Task.select().order_by(Task._meta.primary_key, ascending=False) + + +@app.get("/tasks/{task_id}/", response_model=TaskModelOut, tags=["Task"]) +async def single_task(task_id: int): + task = ( + await Task.select() + .where(Task._meta.primary_key == task_id) + .first() + .callback(check_record_not_found) + ) + return task -@app.post("/tasks/", response_model=TaskModelOut) +@app.post("/tasks/", response_model=TaskModelOut, tags=["Task"]) async def create_task(task_model: TaskModelIn): - task = Task(**task_model.dict()) + task = Task(**task_model.model_dump()) await task.save() return task.to_dict() -@app.put("/tasks/{task_id}/", response_model=TaskModelOut) +@app.put("/tasks/{task_id}/", response_model=TaskModelOut, tags=["Task"]) async def update_task(task_id: int, task_model: TaskModelIn): - task = await Task.objects().get(Task._meta.primary_key == task_id) - if not task: - return JSONResponse({}, status_code=404) - - for key, value in task_model.dict().items(): + task = ( + await Task.objects() + .get(Task._meta.primary_key == task_id) + .callback(check_record_not_found) + ) + for key, value in task_model.model_dump().items(): setattr(task, key, value) await task.save() - return task.to_dict() -@app.delete("/tasks/{task_id}/") +@app.delete( + "/tasks/{task_id}/", + status_code=status.HTTP_204_NO_CONTENT, + tags=["Task"], +) async def delete_task(task_id: int): - task = await Task.objects().get(Task._meta.primary_key == task_id) - if not task: - return JSONResponse({}, status_code=404) - + task = ( + await Task.objects() + .get(Task._meta.primary_key == task_id) + .callback(check_record_not_found) + ) await task.remove() - - return JSONResponse({}) diff --git a/piccolo/apps/asgi/commands/templates/app/_litestar_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_litestar_app.py.jinja index a8df30169..963ea3c42 100644 --- a/piccolo/apps/asgi/commands/templates/app/_litestar_app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/_litestar_app.py.jinja @@ -1,4 +1,4 @@ -import typing as t +from typing import Any from litestar import Litestar, asgi, delete, get, patch, post from litestar.contrib.jinja import JinjaTemplateEngine @@ -21,11 +21,11 @@ you can use `create_pydantic_model` as in other asgi templates from piccolo.utils.pydantic import create_pydantic_model -TaskModelIn: t.Any = create_pydantic_model( +TaskModelIn: Any = create_pydantic_model( table=Task, model_name="TaskModelIn", ) -TaskModelOut: t.Any = create_pydantic_model( +TaskModelOut: Any = create_pydantic_model( table=Task, include_default_columns=True, model_name="TaskModelOut", @@ -50,12 +50,30 @@ async def admin(scope: "Scope", receive: "Receive", send: "Send") -> None: await create_admin(tables=APP_CONFIG.table_classes)(scope, receive, send) +# Check if the record is None. Use for query callback +def check_record_not_found(result: dict[str, Any]) -> dict[str, Any]: + if result is None: + raise NotFoundException(detail="Record not found") + return result + + @get("/tasks", tags=["Task"]) -async def tasks() -> t.List[TaskModelOut]: +async def tasks() -> list[TaskModelOut]: tasks = await Task.select().order_by(Task._meta.primary_key, ascending=False) return [TaskModelOut(**task) for task in tasks] +@get("/tasks/{task_id:int}", tags=["Task"]) +async def single_task(task_id: int) -> TaskModelOut: + task = ( + await Task.select() + .where(Task._meta.primary_key == task_id) + .first() + .callback(check_record_not_found) + ) + return TaskModelOut(**task) + + @post("/tasks", tags=["Task"]) async def create_task(data: TaskModelIn) -> TaskModelOut: task = Task(**data.model_dump()) @@ -65,9 +83,12 @@ async def create_task(data: TaskModelIn) -> TaskModelOut: @patch("/tasks/{task_id:int}", tags=["Task"]) async def update_task(task_id: int, data: TaskModelIn) -> TaskModelOut: - task = await Task.objects().get(Task._meta.primary_key == task_id) - if not task: - raise NotFoundException("Task does not exist") + task = ( + await Task.objects() + .get(Task._meta.primary_key == task_id) + .callback(check_record_not_found) + ) + for key, value in data.model_dump().items(): setattr(task, key, value) @@ -77,9 +98,11 @@ async def update_task(task_id: int, data: TaskModelIn) -> TaskModelOut: @delete("/tasks/{task_id:int}", tags=["Task"]) async def delete_task(task_id: int) -> None: - task = await Task.objects().get(Task._meta.primary_key == task_id) - if not task: - raise NotFoundException("Task does not exist") + task = ( + await Task.objects() + .get(Task._meta.primary_key == task_id) + .callback(check_record_not_found) + ) await task.remove() @@ -104,6 +127,7 @@ app = Litestar( admin, home, tasks, + single_task, create_task, update_task, delete_task, diff --git a/piccolo/apps/asgi/commands/templates/app/_quart_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_quart_app.py.jinja index 8de0228e8..cf648469a 100644 --- a/piccolo/apps/asgi/commands/templates/app/_quart_app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/_quart_app.py.jinja @@ -1,4 +1,4 @@ -import typing as t +from typing import Any from http import HTTPStatus from hypercorn.middleware import DispatcherMiddleware @@ -6,6 +6,7 @@ from piccolo.engine import engine_finder from piccolo_admin.endpoints import create_admin from piccolo_api.crud.serializers import create_pydantic_model from quart import Quart +from quart.helpers import abort from quart_schema import ( Info, QuartSchema, @@ -24,17 +25,24 @@ app = Quart(__name__, static_folder="static") QuartSchema(app, info=Info(title="Quart API", version="0.1.0")) -TaskModelIn: t.Any = create_pydantic_model( +TaskModelIn: Any = create_pydantic_model( table=Task, model_name="TaskModelIn", ) -TaskModelOut: t.Any = create_pydantic_model( +TaskModelOut: Any = create_pydantic_model( table=Task, include_default_columns=True, model_name="TaskModelOut", ) +# Check if the record is None. Use for query callback +def check_record_not_found(result: dict[str, Any]) -> dict[str, Any]: + if result is None: + abort(code=HTTPStatus.NOT_FOUND) + return result + + @app.get("/") @hide def home(): @@ -42,10 +50,24 @@ def home(): @app.get("/tasks/") -@validate_response(t.List[TaskModelOut]) +@validate_response(list[TaskModelOut]) @tag(["Task"]) async def tasks(): - return await Task.select().order_by(Task._meta.primary_key, ascending=False) + tasks = await Task.select().order_by(Task._meta.primary_key, ascending=False) + return [TaskModelOut(**task) for task in tasks], HTTPStatus.OK + + +@app.get("/tasks//") +@validate_response(TaskModelOut) +@tag(["Task"]) +async def single_task(task_id: int): + task = ( + await Task.select() + .where(Task._meta.primary_key == task_id) + .first() + .callback(check_record_not_found) + ) + return TaskModelOut(**task), HTTPStatus.OK @app.post("/tasks/") @@ -63,28 +85,29 @@ async def create_task(data: TaskModelIn): @validate_response(TaskModelOut) @tag(["Task"]) async def update_task(task_id: int, data: TaskModelIn): - task = await Task.objects().get(Task._meta.primary_key == task_id) - if not task: - return {}, HTTPStatus.NOT_FOUND + task = ( + await Task.objects() + .get(Task._meta.primary_key == task_id) + .callback(check_record_not_found) + ) for key, value in data.model_dump().items(): setattr(task, key, value) await task.save() - return task.to_dict(), HTTPStatus.OK + return TaskModelOut(**task.to_dict()), HTTPStatus.OK @app.delete("/tasks//") -@validate_response(TaskModelOut) @tag(["Task"]) async def delete_task(task_id: int): - task = await Task.objects().get(Task._meta.primary_key == task_id) - if not task: - return {}, HTTPStatus.NOT_FOUND - + task = ( + await Task.objects() + .get(Task._meta.primary_key == task_id) + .callback(check_record_not_found) + ) await task.remove() - return {}, HTTPStatus.OK diff --git a/piccolo/apps/asgi/commands/templates/app/_sanic_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_sanic_app.py.jinja index d76a13d5e..de5836d4c 100644 --- a/piccolo/apps/asgi/commands/templates/app/_sanic_app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/_sanic_app.py.jinja @@ -1,11 +1,10 @@ -import asyncio -import typing as t +from typing import Any from hypercorn.middleware import DispatcherMiddleware from piccolo.engine import engine_finder from piccolo_admin.endpoints import create_admin from piccolo_api.crud.serializers import create_pydantic_model -from sanic import Request, Sanic, json +from sanic import NotFound, Request, Sanic, json from sanic_ext import openapi from home.endpoints import index @@ -16,17 +15,24 @@ app = Sanic(__name__) app.static("/static/", "static") -TaskModelIn: t.Any = create_pydantic_model( +TaskModelIn: Any = create_pydantic_model( table=Task, model_name="TaskModelIn", ) -TaskModelOut: t.Any = create_pydantic_model( +TaskModelOut: Any = create_pydantic_model( table=Task, include_default_columns=True, model_name="TaskModelOut", ) +# Check if the record is None. Use for query callback +def check_record_not_found(result: dict[str, Any]) -> dict[str, Any]: + if result is None: + raise NotFound(message="Record not found") + return result + + @app.get("/") @openapi.exclude() def home(request: Request): @@ -43,6 +49,19 @@ async def tasks(request: Request): ) +@app.get("/tasks//") +@openapi.tag("Task") +@openapi.response(200, {"application/json": TaskModelOut.model_json_schema()}) +async def single_task(request: Request, task_id: int): + task = ( + await Task.select() + .where(Task._meta.primary_key == task_id) + .first() + .callback(check_record_not_found) + ) + return json(task, status=200) + + @app.post("/tasks/") @openapi.definition( body={"application/json": TaskModelIn.model_json_schema()}, @@ -62,9 +81,11 @@ async def create_task(request: Request): ) @openapi.response(200, {"application/json": TaskModelOut.model_json_schema()}) async def update_task(request: Request, task_id: int): - task = await Task.objects().get(Task._meta.primary_key == task_id) - if not task: - return json({}, status=404) + task = ( + await Task.objects() + .get(Task._meta.primary_key == task_id) + .callback(check_record_not_found) + ) for key, value in request.json.items(): setattr(task, key, value) @@ -75,9 +96,11 @@ async def update_task(request: Request, task_id: int): @app.delete("/tasks//") @openapi.tag("Task") async def delete_task(request: Request, task_id: int): - task = await Task.objects().get(Task._meta.primary_key == task_id) - if not task: - return json({}, status=404) + task = ( + await Task.objects() + .get(Task._meta.primary_key == task_id) + .callback(check_record_not_found) + ) await task.remove() return json({}, status=200) From 8544747ea50c7463957565b1744254cb338d8681 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Sat, 11 Oct 2025 12:54:12 +0200 Subject: [PATCH 701/727] change Blacksheep dependencies (#1263) --- piccolo/apps/asgi/commands/new.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index 4db7ee2a3..673de9af9 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -12,7 +12,7 @@ ROUTER_DEPENDENCIES = { "starlette": ["starlette"], "fastapi": ["fastapi"], - "blacksheep": ["blacksheep"], + "blacksheep": ["blacksheep[full]"], "litestar": ["litestar"], "esmerald": ["esmerald"], "lilya": ["lilya"], From dd3002e371670a800b77a9b7f4da4708c397a100 Mon Sep 17 00:00:00 2001 From: Ryan Varley Date: Sat, 11 Oct 2025 12:23:04 +0100 Subject: [PATCH 702/727] Fix int arithmetic bug (#1260) * first attempt * test a second way * undo fix to confirm failing tests * Revert "undo fix to confirm failing tests" This reverts commit 7e53f8a151afeca4d2ec55b503cb42eb3a50ba8f. * Fix i think * more test cases * more similar to current style * remove dupe test * remove > and < tests for now --------- Co-authored-by: Daniel Townsend --- piccolo/columns/column_types.py | 4 ++-- tests/table/test_update.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 7137f2e88..bbb665ebb 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -158,8 +158,8 @@ def get_querystring( raise ValueError( "Adding values across joins isn't currently supported." ) - column_name = column._meta.db_column_name - return QueryString(f"{column_name} {operator} {column_name}") + other_column_name = value._meta.db_column_name + return QueryString(f"{column_name} {operator} {other_column_name}") elif isinstance(value, (int, float)): if reverse: return QueryString(f"{{}} {operator} {column_name}", value) diff --git a/tests/table/test_update.py b/tests/table/test_update.py index 4cb29df26..9599cb465 100644 --- a/tests/table/test_update.py +++ b/tests/table/test_update.py @@ -158,6 +158,7 @@ def test_update_returning_alias(self): class MyTable(Table): integer = Integer(null=True) + other_integer = Integer(null=True, default=5) timestamp = Timestamp(null=True) timestamptz = Timestamptz(null=True) date = Date(null=True) @@ -295,6 +296,20 @@ class OperatorTestCase: querystring=2000 - MyTable.integer, expected=1000, ), + OperatorTestCase( + description="Subtract Integer Columns", + column=MyTable.integer, + initial=1000, + querystring=MyTable.integer - MyTable.other_integer, + expected=995, + ), + OperatorTestCase( + description="Add Integer Columns", + column=MyTable.integer, + initial=1000, + querystring=MyTable.integer + MyTable.other_integer, + expected=1005, + ), OperatorTestCase( description="Multiply Integer", column=MyTable.integer, From 3ab7ce2861aa02374c50d75a2f2c0aee43bdeb3a Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 11 Oct 2025 21:55:23 +0100 Subject: [PATCH 703/727] fix bug with decimal values in `ModelBuilder` (#1267) --- piccolo/testing/model_builder.py | 6 ++++-- piccolo/testing/random_builder.py | 12 ++++++++++++ tests/testing/test_random_builder.py | 5 +++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/piccolo/testing/model_builder.py b/piccolo/testing/model_builder.py index e45dc1dbe..b493f9afa 100644 --- a/piccolo/testing/model_builder.py +++ b/piccolo/testing/model_builder.py @@ -1,6 +1,7 @@ from __future__ import annotations import datetime +import decimal import json from collections.abc import Callable from decimal import Decimal @@ -20,6 +21,7 @@ class ModelBuilder: datetime.date: RandomBuilder.next_date, datetime.datetime: RandomBuilder.next_datetime, float: RandomBuilder.next_float, + decimal.Decimal: RandomBuilder.next_decimal, int: RandomBuilder.next_int, str: RandomBuilder.next_str, datetime.time: RandomBuilder.next_time, @@ -163,8 +165,8 @@ def _randomize_attribute(cls, column: Column) -> Any: random_value: Any if column.value_type == Decimal: precision, scale = column._meta.params["digits"] or (4, 2) - random_value = RandomBuilder.next_float( - maximum=10 ** (precision - scale), scale=scale + random_value = RandomBuilder.next_decimal( + precision=precision, scale=scale ) elif column.value_type == datetime.datetime: tz_aware = getattr(column, "tz_aware", False) diff --git a/piccolo/testing/random_builder.py b/piccolo/testing/random_builder.py index 684cd1d9f..537cb8113 100644 --- a/piccolo/testing/random_builder.py +++ b/piccolo/testing/random_builder.py @@ -1,4 +1,5 @@ import datetime +import decimal import enum import random import string @@ -43,6 +44,17 @@ def next_enum(cls, e: type[enum.Enum]) -> Any: def next_float(cls, minimum=0, maximum=2147483647, scale=5) -> float: return round(random.uniform(minimum, maximum), scale) + @classmethod + def next_decimal( + cls, precision: int = 4, scale: int = 2 + ) -> decimal.Decimal: + # For precision 4 and scale 2, maximum needs to be 99.99. + maximum = (10 ** (precision - scale)) - (10 ** (-1 * scale)) + float_number = cls.next_float(maximum=maximum, scale=scale) + # We convert float_number to a string first, otherwise the decimal + # value is slightly off due to floating point precision. + return decimal.Decimal(str(float_number)) + @classmethod def next_int(cls, minimum=0, maximum=2147483647) -> int: return random.randint(minimum, maximum) diff --git a/tests/testing/test_random_builder.py b/tests/testing/test_random_builder.py index 1f078cb9e..da406dc3b 100644 --- a/tests/testing/test_random_builder.py +++ b/tests/testing/test_random_builder.py @@ -1,3 +1,4 @@ +import decimal import unittest from enum import Enum @@ -35,6 +36,10 @@ def test_next_float(self): random_float = RandomBuilder.next_float(maximum=1000) self.assertLessEqual(random_float, 1000) + def test_next_decimal(self): + random_decimal = RandomBuilder.next_decimal(precision=4, scale=2) + self.assertLessEqual(random_decimal, decimal.Decimal("99.99")) + def test_next_int(self): random_int = RandomBuilder.next_int() self.assertLessEqual(random_int, 2147483647) From f9561d33ab1afe6ab14db496f13b1b7baafba6f4 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 11 Oct 2025 23:15:25 +0100 Subject: [PATCH 704/727] 1268 M2M columns not showing up in playground (#1269) * add m2m columns to `Table._table_str` * simplify `TestTableStr` * add `test_m2m` --- piccolo/table.py | 9 +++++++ tests/table/test_str.py | 52 ++++++++++++++++++++++++----------------- 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/piccolo/table.py b/piccolo/table.py index cda091064..32be2a47b 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -1372,6 +1372,15 @@ def _table_str( columns.append( f"{col._meta.name} = {col.__class__.__name__}({params_string})" ) + + for m2m_relationship in cls._meta.m2m_relationships: + joining_table_name = ( + m2m_relationship._meta.resolved_joining_table.__name__ + ) + columns.append( + f"{m2m_relationship._meta.name} = M2M({joining_table_name})" + ) + columns_string = spacer.join(columns) tablename = repr(cls._meta.tablename) diff --git a/tests/table/test_str.py b/tests/table/test_str.py index 604002f6c..de0323912 100644 --- a/tests/table/test_str.py +++ b/tests/table/test_str.py @@ -1,30 +1,20 @@ from unittest import TestCase -from tests.base import engine_is -from tests.example_apps.music.tables import Manager +from piccolo.apps.playground.commands.run import Genre, Manager class TestTableStr(TestCase): - def test_str(self): - if engine_is("cockroach"): - self.assertEqual( - Manager._table_str(), - ( - "class Manager(Table, tablename='manager'):\n" - " id = Serial(null=False, primary_key=True, unique=False, index=False, index_method=IndexMethod.btree, choices=None, db_column_name='id', secret=False)\n" # noqa: E501 - " name = Varchar(length=50, default='', null=False, primary_key=False, unique=False, index=False, index_method=IndexMethod.btree, choices=None, db_column_name=None, secret=False)\n" # noqa: E501 - ), - ) - else: - self.assertEqual( - Manager._table_str(), - ( - "class Manager(Table, tablename='manager'):\n" - " id = Serial(null=False, primary_key=True, unique=False, index=False, index_method=IndexMethod.btree, choices=None, db_column_name='id', secret=False)\n" # noqa: E501 - " name = Varchar(length=50, default='', null=False, primary_key=False, unique=False, index=False, index_method=IndexMethod.btree, choices=None, db_column_name=None, secret=False)\n" # noqa: E501 - ), - ) + def test_all_attributes(self): + self.assertEqual( + Manager._table_str(), + ( + "class Manager(Table, tablename='manager'):\n" + " id = Serial(null=False, primary_key=True, unique=False, index=False, index_method=IndexMethod.btree, choices=None, db_column_name='id', secret=False)\n" # noqa: E501 + " name = Varchar(length=50, default='', null=False, primary_key=False, unique=False, index=False, index_method=IndexMethod.btree, choices=None, db_column_name=None, secret=False)\n" # noqa: E501 + ), + ) + def test_abbreviated(self): self.assertEqual( Manager._table_str(abbreviated=True), ( @@ -34,5 +24,23 @@ def test_str(self): ), ) - # We should also be able to print it directly. + def test_m2m(self): + """ + Make sure M2M relationships appear in the Table string. + """ + + self.assertEqual( + Genre._table_str(abbreviated=True), + ( + "class Genre(Table):\n" + " id = Serial()\n" + " name = Varchar()\n" + " bands = M2M(GenreToBand)\n" + ), + ) + + def test_print(self): + """ + Make sure we can print it directly without any errors. + """ print(Manager) From 29544f4b709d1dd7e5ed40b9b2fe2d2509be2012 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 11 Oct 2025 23:26:38 +0100 Subject: [PATCH 705/727] bumped version --- CHANGES.rst | 21 +++++++++++++++++++++ piccolo/__init__.py | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 7423ef065..4c473cab5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,27 @@ Changes ======= +1.29.0 +------ + +* Fixed a bug with adding / substrating ``Integer`` columns from one another in + queries (thanks to @ryanvarley for this). +* Updated the ASGI templates, and BlackSheep dependencies (thanks to @sinisaos + for this). +* Fixed a bug where decimal values generated by ``ModelBuilder`` could be too + large. +* Added an example ``M2M`` relationship in the playground to make learning + ``M2M`` easier (thanks to @sinisaos for this). +* Added documentation for ``MigrationManager.get_table_from_snapshot``, which + is a way of getting a ``Table`` from the migration history - useful when + running data migrations (thanks to @sinisaos for this). +* Columns with the ``secret=True`` argument are now added to + ``Table._meta.secret_columns`` (thanks to @sinisaos for this). +* Added documentation for the ``migration`` table. +* Tidied up Pydantic tests (thanks to @sinisaos for this). + +------------------------------------------------------------------------------- + 1.28.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index c9d2a61cc..6689c02a2 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.28.0" +__VERSION__ = "1.29.0" From 11ada0bc83bd60545b269925140ffaf2b43397df Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 11 Oct 2025 23:29:59 +0100 Subject: [PATCH 706/727] fix typo in changelog --- CHANGES.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 4c473cab5..1b0ce570a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,8 +4,8 @@ Changes 1.29.0 ------ -* Fixed a bug with adding / substrating ``Integer`` columns from one another in - queries (thanks to @ryanvarley for this). +* Fixed a bug with adding / subtracting ``Integer`` columns from one another + in queries (thanks to @ryanvarley for this). * Updated the ASGI templates, and BlackSheep dependencies (thanks to @sinisaos for this). * Fixed a bug where decimal values generated by ``ModelBuilder`` could be too From 5046256872d2741cc9a9436e469e0201d09600af Mon Sep 17 00:00:00 2001 From: sinisaos Date: Wed, 22 Oct 2025 00:50:45 +0200 Subject: [PATCH 707/727] remove Python 3.9 and add support for Python 3.14 (#1270) * remove Python 3.9 and add support for Python 3.14 * migration table force drop * Revert "migration table force drop" This reverts commit 02c76973ae3ed7dd2beb971605a949a6b4c96539. * remove exclusions * fix tests which don't tidy up after themselves --------- Co-authored-by: Daniel Townsend --- .github/workflows/tests.yaml | 8 ++--- requirements/dev-requirements.txt | 2 +- requirements/test-requirements.txt | 2 +- setup.py | 4 +-- .../migrations/auto/test_serialisation.py | 36 +++++++------------ tests/apps/migrations/commands/test_check.py | 9 +++-- tests/test_main.py | 9 +++-- 7 files changed, 34 insertions(+), 36 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index e23be030f..ea0a35462 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -16,7 +16,7 @@ jobs: timeout-minutes: 60 strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v3 @@ -85,7 +85,7 @@ jobs: timeout-minutes: 60 strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] postgres-version: [12, 13, 14, 15, 16, 17] # Service containers to run with `container-job` @@ -141,7 +141,7 @@ jobs: timeout-minutes: 60 strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] cockroachdb-version: ["v24.1.0"] steps: - uses: actions/checkout@v3 @@ -175,7 +175,7 @@ jobs: timeout-minutes: 60 strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v3 diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt index 726e62994..6807bf875 100644 --- a/requirements/dev-requirements.txt +++ b/requirements/dev-requirements.txt @@ -5,7 +5,7 @@ flake8==6.1.0 isort==5.10.1 slotscheck==0.17.1 twine==3.8.0 -mypy==1.7.1 +mypy==1.18.1 pip-upgrader==1.4.15 pyright==1.1.367 wheel==0.38.1 diff --git a/requirements/test-requirements.txt b/requirements/test-requirements.txt index 6b8be1348..f419596cf 100644 --- a/requirements/test-requirements.txt +++ b/requirements/test-requirements.txt @@ -1,6 +1,6 @@ coveralls==3.3.1 httpx==0.28.0 pytest-cov==3.0.0 -pytest==6.2.5 +pytest==8.3.5 python-dateutil==2.8.2 typing-extensions>=4.3.0 diff --git a/setup.py b/setup.py index 7ec55f967..2f2f320e0 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ def extras_require() -> dict[str, list[str]]: long_description_content_type="text/markdown", author="Daniel Townsend", author_email="dan@dantownsend.co.uk", - python_requires=">=3.9.0", + python_requires=">=3.10.0", url="https://github.com/piccolo-orm/piccolo", packages=find_packages(exclude=("tests",)), package_data={ @@ -83,11 +83,11 @@ def extras_require() -> dict[str, list[str]]: "License :: OSI Approved :: MIT License", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Framework :: AsyncIO", "Typing :: Typed", diff --git a/tests/apps/migrations/auto/test_serialisation.py b/tests/apps/migrations/auto/test_serialisation.py index 2920b72c9..1f622f3eb 100644 --- a/tests/apps/migrations/auto/test_serialisation.py +++ b/tests/apps/migrations/auto/test_serialisation.py @@ -61,14 +61,12 @@ def test_contains_column_types(self): assert getattr(UniqueGlobalNames, "COLUMN_ARRAY", "Array") def test_warn_if_is_conflicting_name(self): - with pytest.warns(None) as recorded_warnings: + with warnings.catch_warnings() as recorded_warnings: + warnings.simplefilter("error") UniqueGlobalNames.warn_if_is_conflicting_name( "SuperMassiveBlackHole" ) - if len(recorded_warnings) != 0: - pytest.fail("Unexpected warning!") - with pytest.warns( UniqueGlobalNameConflictWarning ) as recorded_warnings: @@ -97,14 +95,12 @@ class ConflictingCls3(CanConflictWithGlobalNames): def warn_if_is_conflicting_with_global_name(self): warnings.warn("test", UniqueGlobalNameConflictWarning) - with pytest.warns(None) as recorded_warnings: + with warnings.catch_warnings() as recorded_warnings: + warnings.simplefilter("error") UniqueGlobalNames.warn_if_are_conflicting_objects( [ConflictingCls1(), ConflictingCls2()] ) - if len(recorded_warnings) != 0: - pytest.fail("Unexpected warning!") - with pytest.warns( UniqueGlobalNameConflictWarning ) as recorded_warnings: @@ -124,12 +120,10 @@ def test_with_module_and_target(self): assert repr(Import(module="a.b", target="c")) == "from a.b import c" def test_warn_if_is_conflicting_with_global_name_with_module_only(self): - with pytest.warns(None) as recorded_warnings: + with warnings.catch_warnings() as recorded_warnings: + warnings.simplefilter("error") Import(module="a.b.c").warn_if_is_conflicting_with_global_name() - if len(recorded_warnings) != 0: - pytest.fail("Unexpected warning!") - with pytest.warns( UniqueGlobalNameConflictWarning ) as recorded_warnings: @@ -138,25 +132,21 @@ def test_warn_if_is_conflicting_with_global_name_with_module_only(self): if len(recorded_warnings) != 1: pytest.fail("Expected 1 warning!") - with pytest.warns(None) as recorded_warnings: + with warnings.catch_warnings() as recorded_warnings: + warnings.simplefilter("error") Import( module="Varchar", expect_conflict_with_global_name="Varchar" ).warn_if_is_conflicting_with_global_name() - if len(recorded_warnings) != 0: - pytest.fail("Unexpected warning!") - def test_warn_if_is_conflicting_with_global_name_with_module_and_target( self, ): - with pytest.warns(None) as recorded_warnings: + with warnings.catch_warnings() as recorded_warnings: + warnings.simplefilter("error") Import( module="a.b", target="c" ).warn_if_is_conflicting_with_global_name() - if len(recorded_warnings) != 0: - pytest.fail("Unexpected warning!") - with pytest.warns( UniqueGlobalNameConflictWarning ) as recorded_warnings: @@ -167,16 +157,14 @@ def test_warn_if_is_conflicting_with_global_name_with_module_and_target( if len(recorded_warnings) != 1: pytest.fail("Expected 1 warning!") - with pytest.warns(None) as recorded_warnings: + with warnings.catch_warnings() as recorded_warnings: + warnings.simplefilter("error") Import( module="a.b", target="Varchar", expect_conflict_with_global_name="Varchar", ).warn_if_is_conflicting_with_global_name() - if len(recorded_warnings) != 0: - pytest.fail("Unexpected warning!") - def example_function(): pass diff --git a/tests/apps/migrations/commands/test_check.py b/tests/apps/migrations/commands/test_check.py index 397c28336..b88cbc378 100644 --- a/tests/apps/migrations/commands/test_check.py +++ b/tests/apps/migrations/commands/test_check.py @@ -1,12 +1,17 @@ -from unittest import TestCase +from unittest import IsolatedAsyncioTestCase from unittest.mock import MagicMock, patch from piccolo.apps.migrations.commands.check import CheckMigrationManager, check +from piccolo.apps.migrations.tables import Migration from piccolo.conf.apps import AppRegistry from piccolo.utils.sync import run_sync -class TestCheckMigrationCommand(TestCase): +class TestCheckMigrationCommand(IsolatedAsyncioTestCase): + + async def asyncTearDown(self): + await Migration.alter().drop_table(if_exists=True) + @patch.object( CheckMigrationManager, "get_app_registry", diff --git a/tests/test_main.py b/tests/test_main.py index 0a0367532..745507f69 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,9 +1,14 @@ -from unittest import TestCase +from unittest import IsolatedAsyncioTestCase +from piccolo.apps.migrations.tables import Migration from piccolo.main import main -class TestMain(TestCase): +class TestMain(IsolatedAsyncioTestCase): + + async def asyncTearDown(self): + await Migration.alter().drop_table(if_exists=True) + def test_main(self): # Just make sure it runs without raising any errors. main() From f03e2dfda04f3e0f38ae48eb136b2166d2158f12 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 22 Oct 2025 00:03:07 +0100 Subject: [PATCH 708/727] bumped version --- CHANGES.rst | 7 +++++++ piccolo/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 1b0ce570a..ea841cac1 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,13 @@ Changes ======= +1.30.0 +------ + +Added support for Python 3.14 (thanks to @sinisaos for this). + +------------------------------------------------------------------------------- + 1.29.0 ------ diff --git a/piccolo/__init__.py b/piccolo/__init__.py index 6689c02a2..00a17cafa 100644 --- a/piccolo/__init__.py +++ b/piccolo/__init__.py @@ -1 +1 @@ -__VERSION__ = "1.29.0" +__VERSION__ = "1.30.0" From 18c359e8758b15d2403d90c420c282e1a3a48c56 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 22 Oct 2025 13:03:16 +0100 Subject: [PATCH 709/727] 1279 Add Postgres 18 support (#1281) * add Postgres 18 and drop Postgres 12 * fix insert test - was fetching wrong constraint name from db --- .github/workflows/tests.yaml | 4 ++-- tests/table/test_insert.py | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index ea0a35462..2f6655c2d 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -4,7 +4,7 @@ on: push: branches: ["master", "v1"] paths-ignore: - - "docs/**" + - "docs/**" pull_request: branches: ["master", "v1"] paths-ignore: @@ -86,7 +86,7 @@ jobs: strategy: matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] - postgres-version: [12, 13, 14, 15, 16, 17] + postgres-version: [13, 14, 15, 16, 17, 18] # Service containers to run with `container-job` services: diff --git a/tests/table/test_insert.py b/tests/table/test_insert.py index 72a1de512..bf298cd09 100644 --- a/tests/table/test_insert.py +++ b/tests/table/test_insert.py @@ -232,14 +232,18 @@ def test_target_string(self): """ Band = self.Band - constraint_name = Band.raw( - """ + constraint_name = [ + i["constraint_name"] + for i in Band.raw( + """ SELECT constraint_name FROM information_schema.constraint_column_usage WHERE column_name = 'name' AND table_name = 'band'; """ - ).run_sync()[0]["constraint_name"] + ).run_sync() + if i["constraint_name"].endswith("_key") + ][0] query = Band.insert(Band(name=self.band.name)).on_conflict( target=constraint_name, From 20bc3a056651d336d495c3cac8a3b76d491c13cf Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 22 Oct 2025 13:50:16 +0100 Subject: [PATCH 710/727] remove backport of graphlib (#1283) --- piccolo/conf/apps.py | 2 +- piccolo/table.py | 2 +- piccolo/utils/graphlib/__init__.py | 5 - piccolo/utils/graphlib/_graphlib.py | 255 ---------------------------- 4 files changed, 2 insertions(+), 262 deletions(-) delete mode 100644 piccolo/utils/graphlib/__init__.py delete mode 100644 piccolo/utils/graphlib/_graphlib.py diff --git a/piccolo/conf/apps.py b/piccolo/conf/apps.py index d000a52cf..983e31e04 100644 --- a/piccolo/conf/apps.py +++ b/piccolo/conf/apps.py @@ -9,6 +9,7 @@ from abc import abstractmethod from collections.abc import Callable, Sequence from dataclasses import dataclass, field +from graphlib import TopologicalSorter from importlib import import_module from types import ModuleType from typing import Optional, Union, cast @@ -18,7 +19,6 @@ from piccolo.apps.migrations.auto.migration_manager import MigrationManager from piccolo.engine.base import Engine from piccolo.table import Table -from piccolo.utils.graphlib import TopologicalSorter from piccolo.utils.warnings import Level, colored_warning diff --git a/piccolo/table.py b/piccolo/table.py index 32be2a47b..692b977fb 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -6,6 +6,7 @@ import warnings from collections.abc import Sequence from dataclasses import dataclass, field +from graphlib import TopologicalSorter from typing import TYPE_CHECKING, Any, Optional, Union, cast, overload from piccolo.columns import Column @@ -50,7 +51,6 @@ from piccolo.query.methods.refresh import Refresh from piccolo.querystring import QueryString from piccolo.utils import _camel_to_snake -from piccolo.utils.graphlib import TopologicalSorter from piccolo.utils.sql_values import convert_to_sql_value from piccolo.utils.sync import run_sync from piccolo.utils.warnings import colored_warning diff --git a/piccolo/utils/graphlib/__init__.py b/piccolo/utils/graphlib/__init__.py deleted file mode 100644 index 75b339dbd..000000000 --- a/piccolo/utils/graphlib/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -try: - from graphlib import CycleError, TopologicalSorter # type: ignore -except ImportError: - # For version < Python 3.9 - from ._graphlib import CycleError, TopologicalSorter # type: ignore diff --git a/piccolo/utils/graphlib/_graphlib.py b/piccolo/utils/graphlib/_graphlib.py deleted file mode 100644 index 73f0c0499..000000000 --- a/piccolo/utils/graphlib/_graphlib.py +++ /dev/null @@ -1,255 +0,0 @@ -""" -This is a backport of graphlib from Python 3.9. -""" - -# flake8: noqa - - -__all__ = ["TopologicalSorter", "CycleError"] - -_NODE_OUT = -1 -_NODE_DONE = -2 - - -class _NodeInfo: - __slots__ = "node", "npredecessors", "successors" - - def __init__(self, node): - # The node this class is augmenting. - self.node = node - - # Number of predecessors, generally >= 0. When this value falls to 0, - # and is returned by get_ready(), this is set to _NODE_OUT and when the - # node is marked done by a call to done(), set to _NODE_DONE. - self.npredecessors = 0 - - # List of successor nodes. The list can contain duplicated elements as - # long as they're all reflected in the successor's npredecessors attribute). - self.successors = [] - - -class CycleError(ValueError): - """Subclass of ValueError raised by TopologicalSorter.prepare if cycles - exist in the working graph. - - If multiple cycles exist, only one undefined choice among them will be reported - and included in the exception. The detected cycle can be accessed via the second - element in the *args* attribute of the exception instance and consists in a list - of nodes, such that each node is, in the graph, an immediate predecessor of the - next node in the list. In the reported list, the first and the last node will be - the same, to make it clear that it is cyclic. - """ - - pass - - -class TopologicalSorter: - """Provides functionality to topologically sort a graph of hashable nodes""" - - def __init__(self, graph=None): - self._node2info = {} - self._ready_nodes = None - self._npassedout = 0 - self._nfinished = 0 - - if graph is not None: - for node, predecessors in graph.items(): - self.add(node, *predecessors) - - def _get_nodeinfo(self, node): - result = self._node2info.get(node) - if result is None: - self._node2info[node] = result = _NodeInfo(node) - return result - - def add(self, node, *predecessors): - """Add a new node and its predecessors to the graph. - - Both the *node* and all elements in *predecessors* must be hashable. - - If called multiple times with the same node argument, the set of dependencies - will be the union of all dependencies passed in. - - It is possible to add a node with no dependencies (*predecessors* is not provided) - as well as provide a dependency twice. If a node that has not been provided before - is included among *predecessors* it will be automatically added to the graph with - no predecessors of its own. - - Raises ValueError if called after "prepare". - """ - if self._ready_nodes is not None: - raise ValueError("Nodes cannot be added after a call to prepare()") - - # Create the node -> predecessor edges - nodeinfo = self._get_nodeinfo(node) - nodeinfo.npredecessors += len(predecessors) - - # Create the predecessor -> node edges - for pred in predecessors: - pred_info = self._get_nodeinfo(pred) - pred_info.successors.append(node) - - def prepare(self): - """Mark the graph as finished and check for cycles in the graph. - - If any cycle is detected, "CycleError" will be raised, but "get_ready" can - still be used to obtain as many nodes as possible until cycles block more - progress. After a call to this function, the graph cannot be modified and - therefore no more nodes can be added using "add". - """ - if self._ready_nodes is not None: - raise ValueError("cannot prepare() more than once") - - self._ready_nodes = [ - i.node for i in self._node2info.values() if i.npredecessors == 0 - ] - # ready_nodes is set before we look for cycles on purpose: - # if the user wants to catch the CycleError, that's fine, - # they can continue using the instance to grab as many - # nodes as possible before cycles block more progress - cycle = self._find_cycle() - if cycle: - raise CycleError("nodes are in a cycle", cycle) - - def get_ready(self): - """Return a tuple of all the nodes that are ready. - - Initially it returns all nodes with no predecessors; once those are marked - as processed by calling "done", further calls will return all new nodes that - have all their predecessors already processed. Once no more progress can be made, - empty tuples are returned. - - Raises ValueError if called without calling "prepare" previously. - """ - if self._ready_nodes is None: - raise ValueError("prepare() must be called first") - - # Get the nodes that are ready and mark them - result = tuple(self._ready_nodes) - n2i = self._node2info - for node in result: - n2i[node].npredecessors = _NODE_OUT - - # Clean the list of nodes that are ready and update - # the counter of nodes that we have returned. - self._ready_nodes.clear() - self._npassedout += len(result) - - return result - - def is_active(self): - """Return ``True`` if more progress can be made and ``False`` otherwise. - - Progress can be made if cycles do not block the resolution and either there - are still nodes ready that haven't yet been returned by "get_ready" or the - number of nodes marked "done" is less than the number that have been returned - by "get_ready". - - Raises ValueError if called without calling "prepare" previously. - """ - if self._ready_nodes is None: - raise ValueError("prepare() must be called first") - return self._nfinished < self._npassedout or bool(self._ready_nodes) - - def __bool__(self): - return self.is_active() - - def done(self, *nodes): - """Marks a set of nodes returned by "get_ready" as processed. - - This method unblocks any successor of each node in *nodes* for being returned - in the future by a call to "get_ready". - - Raises :exec:`ValueError` if any node in *nodes* has already been marked as - processed by a previous call to this method, if a node was not added to the - graph by using "add" or if called without calling "prepare" previously or if - node has not yet been returned by "get_ready". - """ - - if self._ready_nodes is None: - raise ValueError("prepare() must be called first") - - n2i = self._node2info - - for node in nodes: - - # Check if we know about this node (it was added previously using add() - nodeinfo = n2i.get(node) - if nodeinfo is None: - raise ValueError(f"node {node!r} was not added using add()") - - # If the node has not being returned (marked as ready) previously, inform the user. - stat = nodeinfo.npredecessors - if stat != _NODE_OUT: - if stat >= 0: - raise ValueError( - f"node {node!r} was not passed out (still not ready)" - ) - elif stat == _NODE_DONE: - raise ValueError(f"node {node!r} was already marked done") - else: - assert False, f"node {node!r}: unknown status {stat}" - - # Mark the node as processed - nodeinfo.npredecessors = _NODE_DONE - - # Go to all the successors and reduce the number of predecessors, collecting all the ones - # that are ready to be returned in the next get_ready() call. - for successor in nodeinfo.successors: - successor_info = n2i[successor] - successor_info.npredecessors -= 1 - if successor_info.npredecessors == 0: - self._ready_nodes.append(successor) - self._nfinished += 1 - - def _find_cycle(self): - n2i = self._node2info - stack = [] - itstack = [] - seen = set() - node2stacki = {} - - for node in n2i: - if node in seen: - continue - - while True: - if node in seen: - # If we have seen already the node and is in the - # current stack we have found a cycle. - if node in node2stacki: - return stack[node2stacki[node] :] + [node] - # else go on to get next successor - else: - seen.add(node) - itstack.append(iter(n2i[node].successors).__next__) - node2stacki[node] = len(stack) - stack.append(node) - - # Backtrack to the topmost stack entry with - # at least another successor. - while stack: - try: - node = itstack[-1]() - break - except StopIteration: - del node2stacki[stack.pop()] - itstack.pop() - else: - break - return None - - def static_order(self): - """Returns an iterable of nodes in a topological order. - - The particular order that is returned may depend on the specific - order in which the items were inserted in the graph. - - Using this method does not require to call "prepare" or "done". If any - cycle is detected, :exc:`CycleError` will be raised. - """ - self.prepare() - while self.is_active(): - node_group = self.get_ready() - yield from node_group - self.done(*node_group) From 9aa0475f9a0abe97708667596aa6505866a2cd4b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 25 Oct 2025 21:57:23 +0100 Subject: [PATCH 711/727] add missing `ColumnKwargs` to docs, and update Sphinx (#1288) --- docs/src/piccolo/api_reference/index.rst | 4 ++++ piccolo/columns/base.py | 5 +++++ requirements/doc-requirements.txt | 6 +++--- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/src/piccolo/api_reference/index.rst b/docs/src/piccolo/api_reference/index.rst index e5a5268f4..45e3341c1 100644 --- a/docs/src/piccolo/api_reference/index.rst +++ b/docs/src/piccolo/api_reference/index.rst @@ -30,6 +30,10 @@ Column :members: +.. autoclass:: ColumnKwargs + :members: + :undoc-members: + ------------------------------------------------------------------------------- Aggregate functions diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index ded8bbcb8..885768bf2 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -361,6 +361,11 @@ def __deepcopy__(self, memo) -> ColumnMeta: class ColumnKwargs(TypedDict, total=False): + """ + Additional arguments which can be passed to :class:`Column` from + subclasses. + """ + null: bool primary_key: bool unique: bool diff --git a/requirements/doc-requirements.txt b/requirements/doc-requirements.txt index 493c9b46b..525aded5c 100644 --- a/requirements/doc-requirements.txt +++ b/requirements/doc-requirements.txt @@ -1,3 +1,3 @@ -Sphinx==7.3.7 -piccolo-theme==0.23.0 -sphinx-autobuild==2021.3.14 +Sphinx==8.3.0 +piccolo-theme==0.24.0 +sphinx-autobuild==2025.8.25 From 1a18547123ff205868550bf41dd652d2d691957d Mon Sep 17 00:00:00 2001 From: sinisaos Date: Sun, 26 Oct 2025 09:41:55 +0100 Subject: [PATCH 712/727] adds Piccolo ecosystem to readme file (#1237) * adds Piccolo ecosystem to readme file * update readme --- README.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/README.md b/README.md index 9b8d1e173..b4da32199 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,48 @@ piccolo asgi new [Starlette](https://www.starlette.io/), [FastAPI](https://fastapi.tiangolo.com/), [BlackSheep](https://www.neoteroi.dev/blacksheep/), [Litestar](https://litestar.dev/), [Esmerald](https://esmerald.dev/), [Lilya](https://lilya.dev), [Quart](https://quart.palletsprojects.com/en/latest/), [Falcon](https://falconframework.org/) and [Sanic](https://sanic.dev/en/) are currently supported. +## Piccolo ecosystem + +### Piccolo Admin + +Piccolo Admin is a powerful admin interface / content management system for Python, built on top of Piccolo. + +It was created at a design agency to serve the needs of customers who demand a high quality, beautiful admin interface for their websites. It's a modern alternative to tools like Wordpress and Django Admin. + +It's built using the latest technologies, with Vue.js on the front end, and a powerful REST backend. + +Some of it's standout features: + +* Powerful data filtering +* Builtin security +* Multi-factor Authentication +* Media support, both locally and in S3 compatible services +* Dark mode support +* CSV exports +* Easily create custom forms +* Works on mobile and desktop +* Use standalone, or integrate with several supported ASGI frameworks +* Multilingual out of box +* Bulk actions, like updating and deleting data +* Flexible UI - only show the columns you want your users to see + +You can read the docs [here](https://piccolo-admin.readthedocs.io/en/latest/). + +### Piccolo API + +Utilities for easily exposing [Piccolo](https://piccolo-orm.readthedocs.io/en/latest/) tables as REST endpoints in ASGI apps, such as [Starlette](https://www.starlette.io) and [FastAPI](https://fastapi.tiangolo.com/). + +Includes a bunch of useful ASGI middleware: + +- Session Auth +- Token Auth +- Rate Limiting +- CSRF +- Content Security Policy (CSP) +- And more + +You can read the docs [here](https://piccolo-api.readthedocs.io/en/latest/). + ## Are you a Django user? We have a handy page which shows the equivalent of [common Django queries in Piccolo](https://piccolo-orm.readthedocs.io/en/latest/piccolo/query_types/django_comparison.html). From 9bbdb054d7a487edc19f999409ae0bd7006a82a8 Mon Sep 17 00:00:00 2001 From: Agustin Arce <59893355+aarcex3@users.noreply.github.com> Date: Sun, 26 Oct 2025 06:32:56 -0300 Subject: [PATCH 713/727] refactor: add __eq__ on Table (#1098) * refactor: add __eq__ on Table * feat: add __eq__ on Table * testing (Table): test equality * final amends * add missing delete * allow comparison with a primary key value * use `value_type` instead * add docs * fix typo in docs * show `band_1.id == band_2.id` * remove raw value comparison * fix linter errors --------- Co-authored-by: Daniel Townsend --- docs/src/piccolo/query_types/objects.rst | 48 +++++++++++++++ piccolo/columns/column_types.py | 2 +- piccolo/table.py | 66 +++++++++++++++++++++ tests/table/instance/test_equality.py | 74 ++++++++++++++++++++++++ 4 files changed, 189 insertions(+), 1 deletion(-) create mode 100644 tests/table/instance/test_equality.py diff --git a/docs/src/piccolo/query_types/objects.rst b/docs/src/piccolo/query_types/objects.rst index 0d6f93d26..691ed55c2 100644 --- a/docs/src/piccolo/query_types/objects.rst +++ b/docs/src/piccolo/query_types/objects.rst @@ -361,6 +361,54 @@ It works with ``prefetch`` too: ------------------------------------------------------------------------------- +Comparing objects +----------------- + +If you have two objects, and you want to know whether they refer to the same +row in the database, you can simply use the equality operator: + +.. code-block:: python + + band_1 = await Band.objects().where(Band.name == "Pythonistas").first() + band_2 = await Band.objects().where(Band.name == "Pythonistas").first() + + >>> band_1 == band_2 + True + +It works by comparing the primary key value of each object. It's equivalent to +this: + +.. code-block:: python + + >>> band_1.id == band_2.id + True + +If the object has no primary key value yet (e.g. it uses a ``Serial`` column, +and it hasn't been saved in the database), then the result will always be +``False``: + +.. code-block:: python + + band_1 = Band() + band_2 = Band() + + >>> band_1 == band_2 + False + +If you want to compare every value on the objects, and not just the primary +key, you can use ``to_dict``. For example: + +.. code-block:: python + + >>> band_1.to_dict() == band_2.to_dict() + True + + >>> band_1.popularity = 10_000 + >>> band_1.to_dict() == band_2.to_dict() + False + +------------------------------------------------------------------------------- + Query clauses ------------- diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index bbb665ebb..8df00d130 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -785,7 +785,7 @@ def column_type(self): return "INTEGER" raise Exception("Unrecognized engine type") - def default(self): + def default(self) -> QueryString: engine_type = self._meta.engine_type if engine_type == "postgres": diff --git a/piccolo/table.py b/piccolo/table.py index 692b977fb..7009f22e5 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -851,6 +851,72 @@ def __repr__(self) -> str: ) return f"<{self.__class__.__name__}: {pk}>" + def __eq__(self, other: t.Any) -> bool: + """ + Lets us check if two ``Table`` instances represent the same row in the + database, based on their primary key value:: + + band_1 = await Band.objects().where( + Band.name == "Pythonistas" + ).first() + + band_2 = await Band.objects().where( + Band.name == "Pythonistas" + ).first() + + band_3 = await Band.objects().where( + Band.name == "Rustaceans" + ).first() + + >>> band_1 == band_2 + True + + >>> band_1 == band_3 + False + + """ + if not isinstance(other, Table): + # This is the correct way to tell Python that this operation + # isn't supported: + # https://docs.python.org/3/library/constants.html#NotImplemented + return NotImplemented + + # Make sure we're comparing the same table. + # There are several ways we could do this (like comparing tablename), + # but this should be OK. + if not isinstance(other, self.__class__): + return False + + pk = self._meta.primary_key + + pk_value = getattr( + self, + pk._meta.name, + ) + + other_pk_value = getattr( + other, + pk._meta.name, + ) + + # Make sure the primary key values are of the correct type. + # We need this for `Serial` columns, which have a `QueryString` + # value until saved in the database. We don't want to use `==` on + # two QueryString values, because QueryString has a custom `__eq__` + # method which doesn't return a boolean. + if isinstance( + pk_value, + pk.value_type, + ) and isinstance( + other_pk_value, + pk.value_type, + ): + return pk_value == other_pk_value + else: + # As a fallback, even if it hasn't been saved in the database, + # an object should still be equal to itself. + return other is self + ########################################################################### # Classmethods diff --git a/tests/table/instance/test_equality.py b/tests/table/instance/test_equality.py new file mode 100644 index 000000000..40ae59517 --- /dev/null +++ b/tests/table/instance/test_equality.py @@ -0,0 +1,74 @@ +from piccolo.columns.column_types import UUID, Varchar +from piccolo.table import Table +from piccolo.testing.test_case import AsyncTableTest +from tests.example_apps.music.tables import Manager + + +class ManagerUUID(Table): + id = UUID(primary_key=True) + name = Varchar() + + +class TestInstanceEquality(AsyncTableTest): + tables = [ + Manager, + ManagerUUID, + ] + + async def test_instance_equality(self) -> None: + """ + Make sure instance equality works, for tables with a `Serial` primary + key. + """ + manager_1 = Manager(name="Guido") + await manager_1.save() + + manager_2 = Manager(name="Graydon") + await manager_2.save() + + self.assertEqual(manager_1, manager_1) + self.assertNotEqual(manager_1, manager_2) + + # Try fetching the row from the database. + manager_1_from_db = ( + await Manager.objects().where(Manager.id == manager_1.id).first() + ) + self.assertEqual(manager_1, manager_1_from_db) + self.assertNotEqual(manager_2, manager_1_from_db) + + # Try rows which haven't been saved yet. + # They have no primary key value (because they use Serial columns + # as the primary key), so they shouldn't be equal. + self.assertNotEqual(Manager(), Manager()) + self.assertNotEqual(manager_1, Manager()) + + # Make sure an object is equal to itself, even if not saved. + manager_unsaved = Manager() + self.assertEqual(manager_unsaved, manager_unsaved) + + async def test_instance_equality_uuid(self) -> None: + """ + Make sure instance equality works, for tables with a `UUID` primary + key. + """ + manager_1 = ManagerUUID(name="Guido") + await manager_1.save() + + manager_2 = ManagerUUID(name="Graydon") + await manager_2.save() + + self.assertEqual(manager_1, manager_1) + self.assertNotEqual(manager_1, manager_2) + + # Try fetching the row from the database. + manager_1_from_db = ( + await ManagerUUID.objects() + .where(ManagerUUID.id == manager_1.id) + .first() + ) + self.assertEqual(manager_1, manager_1_from_db) + self.assertNotEqual(manager_2, manager_1_from_db) + + # Make sure an object is equal to itself, even if not saved. + manager_unsaved = ManagerUUID() + self.assertEqual(manager_unsaved, manager_unsaved) From 87492fd0dc6f2545d0c34cd0982b93f0c8a5eaa2 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Sun, 26 Oct 2025 12:35:43 +0100 Subject: [PATCH 714/727] fix linter errors from latest commit to master (#1290) --- piccolo/table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piccolo/table.py b/piccolo/table.py index 7009f22e5..bd69998f3 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -851,7 +851,7 @@ def __repr__(self) -> str: ) return f"<{self.__class__.__name__}: {pk}>" - def __eq__(self, other: t.Any) -> bool: + def __eq__(self, other: Any) -> bool: """ Lets us check if two ``Table`` instances represent the same row in the database, based on their primary key value:: From b2331568243185669957e21b3e694bc718427318 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Mon, 27 Oct 2025 01:33:01 +0100 Subject: [PATCH 715/727] fixed missing primary key import for FK migration using LazyTableReference (#1231) --- piccolo/apps/migrations/auto/serialisation.py | 15 +++++++++++++++ tests/apps/migrations/auto/test_serialisation.py | 10 ++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/piccolo/apps/migrations/auto/serialisation.py b/piccolo/apps/migrations/auto/serialisation.py index 5db540cd3..21e4a3b3c 100644 --- a/piccolo/apps/migrations/auto/serialisation.py +++ b/piccolo/apps/migrations/auto/serialisation.py @@ -667,6 +667,21 @@ def serialise_params(params: dict[str, Any]) -> SerialisedParams: expect_conflict_with_global_name=UniqueGlobalNames.TABLE, ) ) + # also add missing primary key to extra_imports when creating a + # migration with a ForeignKey that uses a LazyTableReference + # https://github.com/piccolo-orm/piccolo/issues/865 + primary_key_class = table_type._meta.primary_key.__class__ + extra_imports.append( + Import( + module=primary_key_class.__module__, + target=primary_key_class.__name__, + expect_conflict_with_global_name=getattr( + UniqueGlobalNames, + f"COLUMN_{primary_key_class.__name__.upper()}", + None, + ), + ) + ) continue # Replace any Table class values into class and table names diff --git a/tests/apps/migrations/auto/test_serialisation.py b/tests/apps/migrations/auto/test_serialisation.py index 1f622f3eb..7171af3ae 100644 --- a/tests/apps/migrations/auto/test_serialisation.py +++ b/tests/apps/migrations/auto/test_serialisation.py @@ -221,10 +221,16 @@ def test_lazy_table_reference(self): self.assertTrue( serialised.params["references"].__repr__() == "Manager" ) + # sorted extra_imports for consistency between tests + sorted_extra_imports = sorted(serialised.extra_imports) - self.assertTrue(len(serialised.extra_imports) == 1) + self.assertTrue(len(serialised.extra_imports) == 2) self.assertEqual( - serialised.extra_imports[0].__str__(), + sorted_extra_imports[0].__str__(), + "from piccolo.columns.column_types import Serial", + ) + self.assertEqual( + sorted_extra_imports[1].__str__(), "from piccolo.table import Table", ) From ffbb9b77a13e7f3c9b670e9751afd07510dc2b87 Mon Sep 17 00:00:00 2001 From: Rob Date: Tue, 28 Oct 2025 23:20:44 +0000 Subject: [PATCH 716/727] Add a small note on `id` column type (#1291) Helps avoid the `sqlite3.OperationalError: duplicate column name: id` in #1289 --- docs/src/piccolo/schema/defining.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/piccolo/schema/defining.rst b/docs/src/piccolo/schema/defining.rst index a110e36e8..defcaf4b3 100644 --- a/docs/src/piccolo/schema/defining.rst +++ b/docs/src/piccolo/schema/defining.rst @@ -31,7 +31,7 @@ Primary Key ----------- Piccolo tables are automatically given a primary key column called ``id``, -which is an auto incrementing integer. +which is an auto incrementing integer (a ``Serial(primary_key=True)`` column). There is currently experimental support for specifying a custom primary key column. For example: From ecf236b24b0ebb4bdfa6ae494903c7229ac22ee6 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 29 Oct 2025 14:30:56 +0000 Subject: [PATCH 717/727] 1289 Improve `Table._table_str` (#1293) * include primary key when abbreviated * inspect column meta * inspect column constructor instead * remove default_column_meta * remove whitespace * fix logic * fix `TestTableStr` tests * improve enum serialisation --- piccolo/apps/migrations/auto/serialisation.py | 68 ++++++++++++++++-- piccolo/table.py | 72 +++++++++++++++---- tests/table/test_str.py | 6 +- 3 files changed, 122 insertions(+), 24 deletions(-) diff --git a/piccolo/apps/migrations/auto/serialisation.py b/piccolo/apps/migrations/auto/serialisation.py index 21e4a3b3c..e3d166353 100644 --- a/piccolo/apps/migrations/auto/serialisation.py +++ b/piccolo/apps/migrations/auto/serialisation.py @@ -334,6 +334,20 @@ def __repr__(self): return f"{self.instance.__class__.__name__}.{self.instance.name}" +@dataclass +class SerialisedReference: + name: str + + def __hash__(self): + return hash(self.__repr__()) + + def __eq__(self, other): + return check_equality(self, other) + + def __repr__(self): + return self.name + + @dataclass class SerialisedTableType(Definition): table_type: type[Table] @@ -426,7 +440,7 @@ def warn_if_is_conflicting_with_global_name(self) -> None: @dataclass -class SerialisedEnumType: +class InlineSerialisedEnumType: enum_type: type[Enum] def __hash__(self): @@ -441,6 +455,26 @@ def __repr__(self): return f"{UniqueGlobalNames.STD_LIB_ENUM}('{class_name}', {params})" +@dataclass +class SerialisedEnumTypeDefinition(Definition): + enum_type: type[Enum] + + def __hash__(self): + return hash(self.enum_type.__name__) + + def __eq__(self, other): + return check_equality(self, other) + + def __repr__(self): + definition = InlineSerialisedEnumType( + enum_type=self.enum_type + ).__repr__() + return f"{self.enum_type.__name__} = {definition}" + + def warn_if_is_conflicting_with_global_name(self) -> None: + UniqueGlobalNames.warn_if_is_conflicting_name(self.enum_type.__name__) + + @dataclass class SerialisedCallable: callable_: Callable @@ -488,10 +522,25 @@ def __repr__(self): ############################################################################### -def serialise_params(params: dict[str, Any]) -> SerialisedParams: +def serialise_params( + params: dict[str, Any], inline_enums: bool = True +) -> SerialisedParams: """ - When writing column params to a migration file, we need to serialise some - of the values. + When writing column params to a migration file, or outputting to the + playground, we need to serialise some of the values. + + :param inline_enums: + If ``True``, enum value are inlined, for example:: + + value=Enum('MyEnum', {'some_value': 'some_value'})) + + Otherwise, it is reproduced as:: + + value=MyEnum + + And the enum definition is added to + ``SerialisedParams.extra_definitions``. + """ params = deepcopy(params) extra_imports: list[Import] = [] @@ -621,7 +670,6 @@ def serialise_params(params: dict[str, Any]) -> SerialisedParams: # Enum types if inspect.isclass(value) and issubclass(value, Enum): - params[key] = SerialisedEnumType(enum_type=value) extra_imports.append( Import( module="enum", @@ -641,6 +689,14 @@ def serialise_params(params: dict[str, Any]) -> SerialisedParams: Import(module=module_name, target=type_.__name__) ) + if inline_enums: + params[key] = InlineSerialisedEnumType(enum_type=value) + else: + params[key] = SerialisedReference(name=value.__name__) + extra_definitions.append( + SerialisedEnumTypeDefinition(enum_type=value) + ) + # Functions if inspect.isfunction(value): if value.__name__ == "": @@ -765,7 +821,7 @@ def deserialise_params(params: dict[str, Any]) -> dict[str, Any]: params[key] = value.callable_ elif isinstance(value, SerialisedTableType): params[key] = value.table_type - elif isinstance(value, SerialisedEnumType): + elif isinstance(value, InlineSerialisedEnumType): params[key] = value.enum_type elif isinstance(value, SerialisedEnumInstance): params[key] = value.instance diff --git a/piccolo/table.py b/piccolo/table.py index bd69998f3..bdfda2cdd 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -1402,42 +1402,79 @@ def _get_index_name(cls, column_names: list[str]) -> str: @classmethod def _table_str( - cls, abbreviated=False, excluded_params: Optional[list[str]] = None + cls, + abbreviated: bool = False, + excluded_params: Optional[list[str]] = None, ): """ Returns a basic string representation of the table and its columns. Used by the playground. :param abbreviated: - If True, a very high level representation is printed out. + If True, a very high level representation is printed out (it just + shows any non-default values). :param excluded_params: Lets us find a middle ground between outputting every kwarg, and the abbreviated version with very few kwargs. For example `['index_method']`, if we want to show all kwargs but index_method. """ + from piccolo.apps.migrations.auto.serialisation import ( + SerialisedEnumTypeDefinition, + serialise_params, + ) + if excluded_params is None: excluded_params = [] + spacer = "\n " columns = [] + extra_definitions = [] for col in cls._meta.columns: - params: list[str] = [] + base_column_defaults = { + key: value.default + for key, value in inspect.signature(Column).parameters.items() + } + column_defaults = { + key: value.default + for key, value in inspect.signature( + col.__class__ + ).parameters.items() + } + defaults = {**base_column_defaults, **column_defaults} + + params = {} for key, value in col._meta.params.items(): if key in excluded_params: continue - _value: str = "" - if inspect.isclass(value): - _value = value.__name__ - params.append(f"{key}={_value}") - else: - _value = repr(value) - if not abbreviated: - params.append(f"{key}={_value}") - params_string = ", ".join(params) + if abbreviated: + # If the value is just the default one, don't include it. + if defaults.get(key, ...) == value: + continue + + # If db_column is the same as the column name then don't + # include it - it does nothing. + if key == "db_column_name" and value == col._meta.name: + continue + + params[key] = value + + serialised_params = serialise_params(params, inline_enums=False) + params_string = ", ".join( + f"{key}={repr(value)}" + for key, value in serialised_params.params.items() + ) columns.append( f"{col._meta.name} = {col.__class__.__name__}({params_string})" ) + extra_definitions.extend( + [ + i + for i in serialised_params.extra_definitions + if isinstance(i, SerialisedEnumTypeDefinition) + ] + ) for m2m_relationship in cls._meta.m2m_relationships: joining_table_name = ( @@ -1447,6 +1484,9 @@ def _table_str( f"{m2m_relationship._meta.name} = M2M({joining_table_name})" ) + extra_definitions_string = spacer.join( + [repr(i) for i in extra_definitions] + ) columns_string = spacer.join(columns) tablename = repr(cls._meta.tablename) @@ -1458,9 +1498,11 @@ def _table_str( else f"{parent_class_name}, tablename={tablename}" ) - return ( - f"class {cls.__name__}({class_args}):\n" f" {columns_string}\n" - ) + output = f"class {cls.__name__}({class_args}):\n" + if extra_definitions_string: + output += f" {extra_definitions_string}\n" + output += f" {columns_string}\n" + return output def create_table_class( diff --git a/tests/table/test_str.py b/tests/table/test_str.py index de0323912..9255331de 100644 --- a/tests/table/test_str.py +++ b/tests/table/test_str.py @@ -19,8 +19,8 @@ def test_abbreviated(self): Manager._table_str(abbreviated=True), ( "class Manager(Table):\n" - " id = Serial()\n" - " name = Varchar()\n" + " id = Serial(primary_key=True)\n" + " name = Varchar(length=50)\n" ), ) @@ -33,7 +33,7 @@ def test_m2m(self): Genre._table_str(abbreviated=True), ( "class Genre(Table):\n" - " id = Serial()\n" + " id = Serial(primary_key=True)\n" " name = Varchar()\n" " bands = M2M(GenreToBand)\n" ), From 6ec3318e12bd2584f5a9d74b213faab62e0cd621 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 29 Oct 2025 23:57:28 +0000 Subject: [PATCH 718/727] 1294 `ModelBuilder` doesn't generate correct values for `Array` columns with `choices` (#1295) * fix `ModelBuilder` for array columns with choices * make test work with older Python versions --- piccolo/testing/model_builder.py | 14 ++++++++++---- tests/testing/test_model_builder.py | 16 ++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/piccolo/testing/model_builder.py b/piccolo/testing/model_builder.py index b493f9afa..7e25d7e10 100644 --- a/piccolo/testing/model_builder.py +++ b/piccolo/testing/model_builder.py @@ -173,10 +173,16 @@ def _randomize_attribute(cls, column: Column) -> Any: random_value = RandomBuilder.next_datetime(tz_aware=tz_aware) elif column.value_type == list: length = RandomBuilder.next_int(maximum=10) - base_type = cast(Array, column).base_column.value_type - random_value = [ - cls.__DEFAULT_MAPPER[base_type]() for _ in range(length) - ] + if column._meta.choices: + random_value = [ + RandomBuilder.next_enum(column._meta.choices) + for _ in range(length) + ] + else: + base_type = cast(Array, column).base_column.value_type + random_value = [ + cls.__DEFAULT_MAPPER[base_type]() for _ in range(length) + ] elif column._meta.choices: random_value = RandomBuilder.next_enum(column._meta.choices) else: diff --git a/tests/testing/test_model_builder.py b/tests/testing/test_model_builder.py index dba0dc791..b1d07376a 100644 --- a/tests/testing/test_model_builder.py +++ b/tests/testing/test_model_builder.py @@ -1,4 +1,5 @@ import asyncio +import enum import json import unittest @@ -30,9 +31,14 @@ class TableWithArrayField(Table): + class Choices(enum.Enum): + a = "a" + b = "b" + strings = Array(Varchar(30)) integers = Array(Integer()) floats = Array(Real()) + choices = Array(Varchar(), choices=Choices) class TableWithDecimal(Table): @@ -104,6 +110,16 @@ def test_choices(self): ["s", "l", "m"], ) + def test_array_choices(self): + """ + Make sure that ``ModelBuilder`` generates arrays where each array + element is a valid choice. + """ + instance = ModelBuilder.build_sync(TableWithArrayField) + for value in instance.choices: + # Will raise an exception if the enum value isn't found: + TableWithArrayField.Choices[value] + def test_datetime(self): """ Make sure that ``ModelBuilder`` generates timezone aware datetime From 58d8754526207e69d92245ae88aaf7c7b1935b16 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 15 Nov 2025 23:16:39 +0000 Subject: [PATCH 719/727] return id of joining table row (#1305) --- piccolo/columns/m2m.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/piccolo/columns/m2m.py b/piccolo/columns/m2m.py index c3bb9a77e..b8d34d46b 100644 --- a/piccolo/columns/m2m.py +++ b/piccolo/columns/m2m.py @@ -311,11 +311,8 @@ async def run(self): transaction, or wrapped in a new transaction. """ engine = self.rows[0]._meta.db - if engine.transaction_exists(): - await self._run() - else: - async with engine.transaction(): - await self._run() + async with engine.transaction(): + return await self._run() def run_sync(self): return run_sync(self.run()) From 8cc88e1a64063ea084a6d3003c625cb977b91060 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 15 Nov 2025 23:54:32 +0000 Subject: [PATCH 720/727] add docs for `UUID4` default (#1308) --- docs/src/piccolo/api_reference/index.rst | 7 +++++++ piccolo/columns/defaults/uuid.py | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/docs/src/piccolo/api_reference/index.rst b/docs/src/piccolo/api_reference/index.rst index 45e3341c1..fbe6feaba 100644 --- a/docs/src/piccolo/api_reference/index.rst +++ b/docs/src/piccolo/api_reference/index.rst @@ -106,6 +106,13 @@ Date .. autoclass:: DateOffset :members: + +UUID +~~~~ + +.. autoclass:: UUID4 + :members: + ------------------------------------------------------------------------------- Testing diff --git a/piccolo/columns/defaults/uuid.py b/piccolo/columns/defaults/uuid.py index 5f2289612..15b235c08 100644 --- a/piccolo/columns/defaults/uuid.py +++ b/piccolo/columns/defaults/uuid.py @@ -7,6 +7,14 @@ class UUID4(Default): + """ + This makes the default value for a + :class:`UUID ` column a randomly + generated UUID v4 value. The advantage over using :func:`uuid.uuid4` from + the standard library, is the default is set on the column definition in the + database too. + """ + @property def postgres(self): return "uuid_generate_v4()" From a0d802c154d29a98f88bb9c0ab15d31ce4e68e67 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 3 Dec 2025 22:02:21 +0000 Subject: [PATCH 721/727] add `sniffio` (#1314) From https://github.com/piccolo-orm/piccolo/pull/1312 --- piccolo/apps/asgi/commands/new.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index 673de9af9..1721cb692 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -13,7 +13,7 @@ "starlette": ["starlette"], "fastapi": ["fastapi"], "blacksheep": ["blacksheep[full]"], - "litestar": ["litestar"], + "litestar": ["litestar", "sniffio"], "esmerald": ["esmerald"], "lilya": ["lilya"], "quart": ["quart", "quart_schema"], From 623679bec7c80c5c7a1d327478c37cf1cdbf4dc6 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Sat, 6 Dec 2025 01:15:02 +0100 Subject: [PATCH 722/727] Move connection pooling to its own page in docs (#1315) * move connection pooling to its own page * tweak docs --------- Co-authored-by: Daniel Townsend --- docs/src/piccolo/engines/cockroach_engine.rst | 49 +----------- docs/src/piccolo/engines/connection_pool.rst | 80 +++++++++++++++++++ docs/src/piccolo/engines/index.rst | 1 + docs/src/piccolo/engines/postgres_engine.rst | 49 +----------- 4 files changed, 85 insertions(+), 94 deletions(-) create mode 100644 docs/src/piccolo/engines/connection_pool.rst diff --git a/docs/src/piccolo/engines/cockroach_engine.rst b/docs/src/piccolo/engines/cockroach_engine.rst index 5dbfced80..a3c3cd85f 100644 --- a/docs/src/piccolo/engines/cockroach_engine.rst +++ b/docs/src/piccolo/engines/cockroach_engine.rst @@ -27,55 +27,10 @@ to learn more. ------------------------------------------------------------------------------- -Connection pool +Connection Pool --------------- -To use a connection pool, you need to first initialise it. The best place to do -this is in the startup event handler of whichever web framework you are using. - -Here's an example using Starlette. Notice that we also close the connection -pool in the shutdown event handler. - -.. code-block:: python - - from piccolo.engine import engine_finder - from starlette.applications import Starlette - - - app = Starlette() - - - @app.on_event('startup') - async def open_database_connection_pool(): - engine = engine_finder() - await engine.start_connection_pool() - - - @app.on_event('shutdown') - async def close_database_connection_pool(): - engine = engine_finder() - await engine.close_connection_pool() - -.. hint:: Using a connection pool helps with performance, since connections - are reused instead of being created for each query. - -Once a connection pool has been started, the engine will use it for making -queries. - -.. hint:: If you're running several instances of an app on the same server, - you may prefer an external connection pooler - like pgbouncer. - -Configuration -~~~~~~~~~~~~~ - -The connection pool uses the same configuration as your engine. You can also -pass in additional parameters, which are passed to the underlying database -adapter. Here's an example: - -.. code-block:: python - - # To increase the number of connections available: - await engine.start_connection_pool(max_size=20) +See :ref:`ConnectionPool`. ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/engines/connection_pool.rst b/docs/src/piccolo/engines/connection_pool.rst new file mode 100644 index 000000000..f5856a917 --- /dev/null +++ b/docs/src/piccolo/engines/connection_pool.rst @@ -0,0 +1,80 @@ +.. _ConnectionPool: + +Connection Pool +=============== + +.. hint:: Connection pools can be used with Postgres and CockroachDB. + +Setup +~~~~~ + +To use a connection pool, you need to first initialise it. The best place to do +this is in the startup event handler of whichever web framework you are using. +We also want to close the connection pool in the shutdown event handler. + +The recommended way for Starlette and FastAPI apps is to use the ``lifespan`` +parameter: + +.. code-block:: python + + from contextlib import asynccontextmanager + from piccolo.engine import engine_finder + from starlette.applications import Starlette + + + @asynccontextmanager + async def lifespan(app: Starlette): + engine = engine_finder() + assert engine + await engine.start_connection_pool() + yield + await engine.close_connection_pool() + + + app = Starlette(lifespan=lifespan) + +In older versions of Starlette and FastAPI, you may need event handlers +instead: + +.. code-block:: python + + from piccolo.engine import engine_finder + from starlette.applications import Starlette + + + app = Starlette() + + + @app.on_event('startup') + async def open_database_connection_pool(): + engine = engine_finder() + await engine.start_connection_pool() + + + @app.on_event('shutdown') + async def close_database_connection_pool(): + engine = engine_finder() + await engine.close_connection_pool() + +.. hint:: Using a connection pool helps with performance, since connections + are reused instead of being created for each query. + +Once a connection pool has been started, the engine will use it for making +queries. + +.. hint:: If you're running several instances of an app on the same server, + you may prefer an external connection pooler - like pgbouncer. + +------------------------------------------------------------------------------- + +Configuration +~~~~~~~~~~~~~ + +The connection pool uses the same configuration as your engine. You can also +pass in additional parameters, which are passed to the underlying database +adapter. Here's an example: + +.. code-block:: python + + # To increase the number of connections available: + await engine.start_connection_pool(max_size=20) \ No newline at end of file diff --git a/docs/src/piccolo/engines/index.rst b/docs/src/piccolo/engines/index.rst index fad00546c..db655c76b 100644 --- a/docs/src/piccolo/engines/index.rst +++ b/docs/src/piccolo/engines/index.rst @@ -127,3 +127,4 @@ Engine types ./sqlite_engine ./postgres_engine ./cockroach_engine + ./connection_pool diff --git a/docs/src/piccolo/engines/postgres_engine.rst b/docs/src/piccolo/engines/postgres_engine.rst index 16a9f6bac..6292c2d71 100644 --- a/docs/src/piccolo/engines/postgres_engine.rst +++ b/docs/src/piccolo/engines/postgres_engine.rst @@ -26,55 +26,10 @@ to learn more. ------------------------------------------------------------------------------- -Connection pool +Connection Pool --------------- -To use a connection pool, you need to first initialise it. The best place to do -this is in the startup event handler of whichever web framework you are using. - -Here's an example using Starlette. Notice that we also close the connection -pool in the shutdown event handler. - -.. code-block:: python - - from piccolo.engine import engine_finder - from starlette.applications import Starlette - - - app = Starlette() - - - @app.on_event('startup') - async def open_database_connection_pool(): - engine = engine_finder() - await engine.start_connection_pool() - - - @app.on_event('shutdown') - async def close_database_connection_pool(): - engine = engine_finder() - await engine.close_connection_pool() - -.. hint:: Using a connection pool helps with performance, since connections - are reused instead of being created for each query. - -Once a connection pool has been started, the engine will use it for making -queries. - -.. hint:: If you're running several instances of an app on the same server, - you may prefer an external connection pooler - like pgbouncer. - -Configuration -~~~~~~~~~~~~~ - -The connection pool uses the same configuration as your engine. You can also -pass in additional parameters, which are passed to the underlying database -adapter. Here's an example: - -.. code-block:: python - - # To increase the number of connections available: - await engine.start_connection_pool(max_size=20) +See :ref:`ConnectionPool`. ------------------------------------------------------------------------------- From 53b76289349f9aee38b0e0b0412ba56f16a8628f Mon Sep 17 00:00:00 2001 From: sinisaos Date: Tue, 16 Dec 2025 16:37:36 +0100 Subject: [PATCH 723/727] remove sniffio dependency (#1316) --- piccolo/apps/asgi/commands/new.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index 1721cb692..673de9af9 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -13,7 +13,7 @@ "starlette": ["starlette"], "fastapi": ["fastapi"], "blacksheep": ["blacksheep[full]"], - "litestar": ["litestar", "sniffio"], + "litestar": ["litestar"], "esmerald": ["esmerald"], "lilya": ["lilya"], "quart": ["quart", "quart_schema"], From 6d61c1fcb69cde597538fb3185b7ab1b123e8a0f Mon Sep 17 00:00:00 2001 From: sinisaos Date: Sat, 10 Jan 2026 01:04:58 +0100 Subject: [PATCH 724/727] target is required on do_update (#1322) --- .../src/piccolo/query_clauses/on_conflict.rst | 6 ++++-- piccolo/query/mixins.py | 5 +++++ tests/table/test_insert.py | 21 +++++++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/docs/src/piccolo/query_clauses/on_conflict.rst b/docs/src/piccolo/query_clauses/on_conflict.rst index 9ae8444e3..c3adfa51c 100644 --- a/docs/src/piccolo/query_clauses/on_conflict.rst +++ b/docs/src/piccolo/query_clauses/on_conflict.rst @@ -78,6 +78,7 @@ Instead, if we want to update the ``popularity``: ... Band(name="Pythonistas", popularity=1200) ... ).on_conflict( ... action="DO UPDATE", + ... target=Band.name, ... values=[Band.popularity] ... ) @@ -93,8 +94,9 @@ If we fetch the data from the database, we'll see that it was updated: Using the ``target`` argument, we can specify which constraint we're concerned with. By specifying ``target=Band.name`` we're only concerned with the unique -constraint for the ``band`` column. If you omit the ``target`` argument, then -it works for all constraints on the table. +constraint for the ``band`` column. If you omit the ``target`` argument on +``DO NOTHING`` action, then it works for all constraints on the table. For +``DO UPDATE`` action, ``target`` is mandatory and must be provided. .. code-block:: python :emphasize-lines: 5 diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index 16b69e8b7..178d793bf 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -769,6 +769,11 @@ def on_conflict( else: raise ValueError("Unrecognised `on conflict` action.") + if target is None and action_ == OnConflictAction.do_update: + raise ValueError( + "The `target` option must be provided with DO UPDATE." + ) + if where and action_ == OnConflictAction.do_nothing: raise ValueError( "The `where` option can only be used with DO NOTHING." diff --git a/tests/table/test_insert.py b/tests/table/test_insert.py index bf298cd09..19c1b0acb 100644 --- a/tests/table/test_insert.py +++ b/tests/table/test_insert.py @@ -174,6 +174,27 @@ def test_do_update_tuple_values(self): ], ) + def test_do_update_no_target(self): + """ + Make sure that `DO UPDATE` with no `target` raises an exception. + """ + Band = self.Band + + new_popularity = self.band.popularity + 1000 + + with self.assertRaises(ValueError) as manager: + Band.insert( + Band(name=self.band.name, popularity=new_popularity) + ).on_conflict( + action="DO UPDATE", + values=[(Band.popularity, new_popularity + 2000)], + ).run_sync() + + self.assertEqual( + manager.exception.__str__(), + "The `target` option must be provided with DO UPDATE.", + ) + def test_do_update_no_values(self): """ Make sure that `DO UPDATE` with no `values` raises an exception. From c0274311af2a1916b0356c1401f0785c3e21787a Mon Sep 17 00:00:00 2001 From: sinisaos Date: Sat, 10 Jan 2026 07:42:44 +0100 Subject: [PATCH 725/727] Change Esmerald to Ravyn in asgi templates (#1320) * change Esmerald to Ravyn in asgi templates * update docs for Ravyn --- README.md | 2 +- docs/src/index.rst | 2 +- docs/src/piccolo/asgi/index.rst | 6 +++--- piccolo/apps/asgi/commands/new.py | 2 +- .../app/{_esmerald_app.py.jinja => _ravyn_app.py.jinja} | 8 ++++---- piccolo/apps/asgi/commands/templates/app/app.py.jinja | 4 ++-- ...erald_endpoints.py.jinja => _ravyn_endpoints.py.jinja} | 4 ++-- .../asgi/commands/templates/app/home/endpoints.py.jinja | 4 ++-- .../templates/app/home/templates/home.html.jinja_raw | 2 +- 9 files changed, 17 insertions(+), 17 deletions(-) rename piccolo/apps/asgi/commands/templates/app/{_esmerald_app.py.jinja => _ravyn_app.py.jinja} (96%) rename piccolo/apps/asgi/commands/templates/app/home/{_esmerald_endpoints.py.jinja => _ravyn_endpoints.py.jinja} (82%) diff --git a/README.md b/README.md index b4da32199..3cf8c6119 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ Let Piccolo scaffold you an ASGI web app, using Piccolo as the ORM: piccolo asgi new ``` -[Starlette](https://www.starlette.io/), [FastAPI](https://fastapi.tiangolo.com/), [BlackSheep](https://www.neoteroi.dev/blacksheep/), [Litestar](https://litestar.dev/), [Esmerald](https://esmerald.dev/), [Lilya](https://lilya.dev), [Quart](https://quart.palletsprojects.com/en/latest/), [Falcon](https://falconframework.org/) and [Sanic](https://sanic.dev/en/) are currently supported. +[Starlette](https://www.starlette.io/), [FastAPI](https://fastapi.tiangolo.com/), [BlackSheep](https://www.neoteroi.dev/blacksheep/), [Litestar](https://litestar.dev/), [Ravyn](https://www.ravyn.dev/), [Lilya](https://lilya.dev/), [Quart](https://quart.palletsprojects.com/en/latest/), [Falcon](https://falconframework.org/) and [Sanic](https://sanic.dev/en/) are currently supported. ## Piccolo ecosystem diff --git a/docs/src/index.rst b/docs/src/index.rst index 849694a21..82b6bd690 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -60,7 +60,7 @@ Give me an ASGI web app! piccolo asgi new -FastAPI, Starlette, BlackSheep, Litestar, Esmerald, Lilya, Quart, Falcon and Sanic +FastAPI, Starlette, BlackSheep, Litestar, Ravyn, Lilya, Quart, Falcon and Sanic are currently supported, with more coming soon. ---------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/asgi/index.rst b/docs/src/piccolo/asgi/index.rst index a0756ca6f..a05c75874 100644 --- a/docs/src/piccolo/asgi/index.rst +++ b/docs/src/piccolo/asgi/index.rst @@ -22,7 +22,7 @@ Routing frameworks `Starlette `_, `FastAPI `_, `BlackSheep `_, -`Litestar `_, `Esmerald `_, +`Litestar `_, `Ravyn `_, `Lilya `_, `Quart `_, `Falcon `_ @@ -31,8 +31,8 @@ and `Sanic `_ are supported. Which to use? ============= -All are great choices. FastAPI is built on top of Starlette and Esmerald is built on top of Lilya, so they're -very similar. FastAPI, BlackSheep, Litestar and Esmerald are great if you want to document a REST +All are great choices. FastAPI is built on top of Starlette and Ravyn is built on top of Lilya, so they're +very similar. FastAPI, BlackSheep, Litestar and Ravyn are great if you want to document a REST API, as they have built-in OpenAPI support. ------------------------------------------------------------------------------- diff --git a/piccolo/apps/asgi/commands/new.py b/piccolo/apps/asgi/commands/new.py index 673de9af9..f85b7b980 100644 --- a/piccolo/apps/asgi/commands/new.py +++ b/piccolo/apps/asgi/commands/new.py @@ -14,7 +14,7 @@ "fastapi": ["fastapi"], "blacksheep": ["blacksheep[full]"], "litestar": ["litestar"], - "esmerald": ["esmerald"], + "ravyn": ["ravyn"], "lilya": ["lilya"], "quart": ["quart", "quart_schema"], "falcon": ["falcon"], diff --git a/piccolo/apps/asgi/commands/templates/app/_esmerald_app.py.jinja b/piccolo/apps/asgi/commands/templates/app/_ravyn_app.py.jinja similarity index 96% rename from piccolo/apps/asgi/commands/templates/app/_esmerald_app.py.jinja rename to piccolo/apps/asgi/commands/templates/app/_ravyn_app.py.jinja index 3556eba14..ad9cc3407 100644 --- a/piccolo/apps/asgi/commands/templates/app/_esmerald_app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/_ravyn_app.py.jinja @@ -1,9 +1,9 @@ from typing import Any from pathlib import Path -from esmerald import ( +from ravyn import ( APIView, - Esmerald, + Ravyn, Gateway, HTTPException, Include, @@ -12,7 +12,7 @@ from esmerald import ( post, put, ) -from esmerald.core.config import StaticFilesConfig +from ravyn.core.config import StaticFilesConfig from piccolo.engine import engine_finder from piccolo.utils.pydantic import create_pydantic_model from piccolo_admin.endpoints import create_admin @@ -107,7 +107,7 @@ class TaskAPIView(APIView): await task.remove() -app = Esmerald( +app = Ravyn( routes=[ Gateway("/", handler=home), Gateway("/tasks", handler=TaskAPIView), diff --git a/piccolo/apps/asgi/commands/templates/app/app.py.jinja b/piccolo/apps/asgi/commands/templates/app/app.py.jinja index a4bf71ee2..9e22c8f28 100644 --- a/piccolo/apps/asgi/commands/templates/app/app.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/app.py.jinja @@ -6,8 +6,8 @@ {% include '_blacksheep_app.py.jinja' %} {% elif router == 'litestar' %} {% include '_litestar_app.py.jinja' %} -{% elif router == 'esmerald' %} - {% include '_esmerald_app.py.jinja' %} +{% elif router == 'ravyn' %} + {% include '_ravyn_app.py.jinja' %} {% elif router == 'lilya' %} {% include '_lilya_app.py.jinja' %} {% elif router == 'quart' %} diff --git a/piccolo/apps/asgi/commands/templates/app/home/_esmerald_endpoints.py.jinja b/piccolo/apps/asgi/commands/templates/app/home/_ravyn_endpoints.py.jinja similarity index 82% rename from piccolo/apps/asgi/commands/templates/app/home/_esmerald_endpoints.py.jinja rename to piccolo/apps/asgi/commands/templates/app/home/_ravyn_endpoints.py.jinja index a8c9bfdff..0a400c3df 100644 --- a/piccolo/apps/asgi/commands/templates/app/home/_esmerald_endpoints.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/home/_ravyn_endpoints.py.jinja @@ -1,8 +1,8 @@ import os import jinja2 -from esmerald import Request, Response, get -from esmerald.responses import HTMLResponse +from ravyn import Request, Response, get +from ravyn.responses import HTMLResponse ENVIRONMENT = jinja2.Environment( loader=jinja2.FileSystemLoader( diff --git a/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja b/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja index 4f0023134..21c3903ab 100644 --- a/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja +++ b/piccolo/apps/asgi/commands/templates/app/home/endpoints.py.jinja @@ -4,8 +4,8 @@ {% include '_blacksheep_endpoints.py.jinja' %} {% elif router == 'litestar' %} {% include '_litestar_endpoints.py.jinja' %} -{% elif router == 'esmerald' %} - {% include '_esmerald_endpoints.py.jinja' %} +{% elif router == 'ravyn' %} + {% include '_ravyn_endpoints.py.jinja' %} {% elif router == 'lilya' %} {% include '_lilya_endpoints.py.jinja' %} {% elif router == 'quart' %} diff --git a/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw b/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw index 47cb9a039..f5e447a9b 100644 --- a/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw +++ b/piccolo/apps/asgi/commands/templates/app/home/templates/home.html.jinja_raw @@ -56,7 +56,7 @@
  • Admin
  • Swagger API
    • -

    Esmerald

    +

    Ravyn

    • Admin
    • Swagger API
      • From 748d802209dbca49698e7ae507dc48745e85a0c1 Mon Sep 17 00:00:00 2001 From: sinisaos Date: Sun, 11 Jan 2026 10:07:28 +0100 Subject: [PATCH 726/727] Fix for issue #1301 (#1321) * fix for issue #1301 * change return statement --- piccolo/apps/migrations/auto/migration_manager.py | 9 +++++---- piccolo/query/constraints.py | 8 ++++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/piccolo/apps/migrations/auto/migration_manager.py b/piccolo/apps/migrations/auto/migration_manager.py index 53f4a3a7c..ce085783f 100644 --- a/piccolo/apps/migrations/auto/migration_manager.py +++ b/piccolo/apps/migrations/auto/migration_manager.py @@ -546,11 +546,12 @@ async def _run_alter_columns(self, backwards: bool = False): constraint_name = await get_fk_constraint_name( column=fk_column ) - await self._run_query( - _Table.alter().drop_constraint( - constraint_name=constraint_name + if constraint_name: + await self._run_query( + _Table.alter().drop_constraint( + constraint_name=constraint_name + ) ) - ) # Then add a new foreign key constraint await self._run_query( diff --git a/piccolo/query/constraints.py b/piccolo/query/constraints.py index 7f6d1f565..a5859c100 100644 --- a/piccolo/query/constraints.py +++ b/piccolo/query/constraints.py @@ -1,10 +1,11 @@ from dataclasses import dataclass +from typing import Optional from piccolo.columns import ForeignKey from piccolo.columns.base import OnDelete, OnUpdate -async def get_fk_constraint_name(column: ForeignKey) -> str: +async def get_fk_constraint_name(column: ForeignKey) -> Optional[str]: """ Checks what the foreign key constraint is called in the database. """ @@ -40,7 +41,10 @@ async def get_fk_constraint_name(column: ForeignKey) -> str: column_name, ) - return constraints[0]["fk_constraint_name"] + # if we change the column type from a non-FK column to + # an FK column, the previous column type has no FK constraints + # and we skip this to allow the migration to continue + return constraints[0]["fk_constraint_name"] if constraints else None @dataclass From 1945476febf3ce376d75ccd4d43c77e643bef21e Mon Sep 17 00:00:00 2001 From: sinisaos Date: Sun, 11 Jan 2026 11:26:45 +0100 Subject: [PATCH 727/727] use Postgres built-in uuid 4 function (#1323) --- piccolo/columns/defaults/uuid.py | 2 +- piccolo/engine/postgres.py | 2 +- tests/apps/migrations/auto/integration/test_migrations.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/piccolo/columns/defaults/uuid.py b/piccolo/columns/defaults/uuid.py index 15b235c08..ad0e34679 100644 --- a/piccolo/columns/defaults/uuid.py +++ b/piccolo/columns/defaults/uuid.py @@ -17,7 +17,7 @@ class UUID4(Default): @property def postgres(self): - return "uuid_generate_v4()" + return "gen_random_uuid()" @property def cockroach(self): diff --git a/piccolo/engine/postgres.py b/piccolo/engine/postgres.py index 8d09d350b..f4ac2d17e 100644 --- a/piccolo/engine/postgres.py +++ b/piccolo/engine/postgres.py @@ -354,7 +354,7 @@ class PostgresEngine(Engine[PostgresTransaction]): def __init__( self, config: dict[str, Any], - extensions: Sequence[str] = ("uuid-ossp",), + extensions: Sequence[str] = tuple(), log_queries: bool = False, log_responses: bool = False, extra_nodes: Optional[Mapping[str, PostgresEngine]] = None, diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index 887fa6108..a065d9588 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -437,7 +437,7 @@ def test_uuid_column(self): [ x.data_type == "uuid", x.is_nullable == "NO", - x.column_default == "uuid_generate_v4()", + x.column_default == "gen_random_uuid()", ] ), )