Fixtures app (piccolo-orm#229)

* prototype for fixture dumping

* add pretty option to dump_json

* add fixture loading

* added docs

* improved docs - mentioned how multiple apps / tables can be specified

* renamed app (fixtures -> fixture)

* moved `MegaTable` into it's own file for reuse

* refactored test file, to support multiple example apps

* fix typo

* added `digits` arg to `MegaTable.numeric_col`

* use pydantic to serialise / deserialise json

* tweaked docs

* fix test

* conditional asyncpg import

* add missing docstrings

* make TestDumpLoad work with sqlite

* added tests for `create_pydantic_model`

Author: dantownsend

docs/src/piccolo/projects_and_apps/included_apps.rst

@@ -4,10 +4,15 @@ Included Apps
 Just as you can modularise your own code using :ref:`apps<PiccoloApps>`, Piccolo itself
 ships with several builtin apps, which provide a lot of its functionality.
 
+-------------------------------------------------------------------------------
+
 Auto includes
 -------------
 
-The following are registered with your :ref:`AppRegistry<AppRegistry>` automatically:
+The following are registered with your :ref:`AppRegistry<AppRegistry>` automatically.
+
+.. hint:: To find out more about each of these commands you can use the
+    ``--help`` flag on the command line. For example ``piccolo app new --help``.
 
 -------------------------------------------------------------------------------
 
@@ -33,6 +38,47 @@ Lets you scaffold an ASGI web app. See :ref:`ASGICommand`.
 
 -------------------------------------------------------------------------------
 
+fixture
+~~~~~~~
+
+Fixtures are used when you want to seed your database with essential data (for
+example, country names).
+
+Once you have created a fixture, it can be used by your colleagues when setting
+up an application on their local machines, or when deploying to a new
+environment.
+
+Databases such as Postgres have inbuilt ways of dumping and restoring data
+(via ``pg_dump`` and ``pg_restore``). Some reasons to use the fixtures app
+instead:
+
+ * When you want the data to be loadable in a range of database versions.
+ * Fixtures are stored in JSON, which are a bit friendlier for source control.
+
+To dump the data into a new fixture file:
+
+.. code-block:: bash
+
+    piccolo fixtures dump > fixtures.json
+
+By default, the fixture contains data from all apps and tables. You can specify
+a subset of apps and tables instead, for example:
+
+.. code-block:: bash
+
+    piccolo fixture dump --apps=blog --tables=Post > fixtures.json
+
+    # Or for multiple apps / tables
+    piccolo fixtures dump --apps=blog,shop --tables=Post,Product > fixtures.json
+
+To load the fixture:
+
+.. code-block:: bash
+
+    piccolo fixture load fixtures.json
+
+-------------------------------------------------------------------------------
+
 meta
 ~~~~
 

tests/example_app/__init__.py piccolo/apps/fixture/__init__.pytests/example_app/__init__.py renamed to piccolo/apps/fixture/__init__.py

@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+import typing as t
+
+from piccolo.apps.fixture.commands.shared import (
+    FixtureConfig,
+    create_pydantic_fixture_model,
+)
+from piccolo.apps.migrations.auto.migration_manager import sort_table_classes
+from piccolo.conf.apps import Finder
+
+
+async def get_dump(
+    fixture_configs: t.List[FixtureConfig],
+) -> t.Dict[str, t.Any]:
+    """
+    Gets the data for each table specified and returns a data structure like:
+
+    .. code-block:: python
+
+        {
+            'my_app_name': {
+                'MyTableName': [
+                    {
+                        'id': 1,
+                        'my_column_name': 'foo'
+                    }
+                ]
+            }
+        }
+
+    """
+    finder = Finder()
+
+    output: t.Dict[str, t.Any] = {}
+
+    for fixture_config in fixture_configs:
+        app_config = finder.get_app_config(app_name=fixture_config.app_name)
+        table_classes = [
+            i
+            for i in app_config.table_classes
+            if i.__name__ in fixture_config.table_class_names
+        ]
+        sorted_table_classes = sort_table_classes(table_classes)
+
+        output[fixture_config.app_name] = {}
+
+        for table_class in sorted_table_classes:
+            data = await table_class.select().run()
+            output[fixture_config.app_name][table_class.__name__] = data
+
+    return output
+
+
+async def dump_to_json_string(
+    fixture_configs: t.List[FixtureConfig],
+) -> str:
+    """
+    Dumps all of the data for the given tables into a JSON string.
+    """
+    dump = await get_dump(fixture_configs=fixture_configs)
+    pydantic_model = create_pydantic_fixture_model(
+        fixture_configs=fixture_configs
+    )
+    json_output = pydantic_model(**dump).json()
+    return json_output
+
+
+def parse_args(apps: str, tables: str) -> t.List[FixtureConfig]:
+    """
+    Works out which apps and tables the user is referring to.
+    """
+    finder = Finder()
+    app_names = []
+
+    if apps == "all":
+        app_names = finder.get_sorted_app_names()
+    elif "," in apps:
+        app_names = apps.split(",")
+    else:
+        # Must be a single app name
+        app_names.append(apps)
+
+    table_class_names: t.Optional[t.List[str]] = None
+
+    if tables != "all":
+        if "," in tables:
+            table_class_names = tables.split(",")
+        else:
+            # Must be a single table class name
+            table_class_names = [tables]
+
+    output: t.List[FixtureConfig] = []
+
+    for app_name in app_names:
+        app_config = finder.get_app_config(app_name=app_name)
+        table_classes = app_config.table_classes
+
+        if table_class_names is None:
+            fixture_configs = [i.__name__ for i in table_classes]
+        else:
+            fixture_configs = [
+                i.__name__
+                for i in table_classes
+                if i.__name__ in table_class_names
+            ]
+        output.append(
+            FixtureConfig(
+                app_name=app_name,
+                table_class_names=fixture_configs,
+            )
+        )
+
+    return output
+
+
+async def dump(apps: str = "all", tables: str = "all"):
+    """
+    Serialises the data from the given Piccolo apps / tables, and prints it
+    out.
+
+    :param apps:
+        For all apps, specify `all`. For specific apps, pass in a comma
+        separated list e.g. `blog,profiles,billing`. For a single app, just
+        pass in the name of that app, e.g. `blog`.
+    :param tables:
+        For all tables, specify `all`. For specific tables, pass in a comma
+        separated list e.g. `Post,Tag`. For a single app, just
+        pass in the name of that app, e.g. `Post`.
+
+    """
+    fixture_configs = parse_args(apps=apps, tables=tables)
+    json_string = await dump_to_json_string(fixture_configs=fixture_configs)
+    print(json_string)

piccolo/apps/fixture/commands/__init__.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+from piccolo.apps.fixture.commands.shared import (
+    FixtureConfig,
+    create_pydantic_fixture_model,
+)
+from piccolo.conf.apps import Finder
+from piccolo.engine import engine_finder
+from piccolo.utils.encoding import load_json
+
+
+async def load_json_string(json_string: str):
+    """
+    Parses the JSON string, and inserts the parsed data into the database.
+    """
+    # We have to deserialise the JSON to find out which apps and tables it
+    # contains, so we can create a Pydantic model.
+    # Then we let Pydantic do the proper deserialisation, as it does a much
+    # better job of deserialising dates, datetimes, bytes etc.
+    deserialised_contents = load_json(json_string)
+
+    app_names = deserialised_contents.keys()
+
+    fixture_configs = [
+        FixtureConfig(
+            app_name=app_name,
+            table_class_names=[
+                i for i in deserialised_contents[app_name].keys()
+            ],
+        )
+        for app_name in app_names
+    ]
+    pydantic_model_class = create_pydantic_fixture_model(
+        fixture_configs=fixture_configs
+    )
+
+    fixture_pydantic_model = pydantic_model_class.parse_raw(json_string)
+
+    finder = Finder()
+    engine = engine_finder()
+
+    if not engine:
+        raise Exception("Unable to find the engine.")
+
+    async with engine.transaction():
+        for app_name in app_names:
+            app_model = getattr(fixture_pydantic_model, app_name)
+
+            for (
+                table_class_name,
+                model_instance_list,
+            ) in app_model.__dict__.items():
+                table_class = finder.get_table_with_name(
+                    app_name, table_class_name
+                )
+
+                await table_class.insert(
+                    *[
+                        table_class(**row.__dict__)
+                        for row in model_instance_list
+                    ]
+                ).run()
+
+
+async def load(path: str = "fixture.json"):
+    """
+    Reads the fixture file, and loads the contents into the database.
+    """
+    with open(path, "r") as f:
+        contents = f.read()
+
+    await load_json_string(contents)

piccolo/apps/fixture/commands/dump.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+import typing as t
+from dataclasses import dataclass
+
+import pydantic
+
+from piccolo.conf.apps import Finder
+from piccolo.utils.pydantic import create_pydantic_model
+
+if t.TYPE_CHECKING:
+    from piccolo.table import Table
+
+
+@dataclass
+class FixtureConfig:
+    app_name: str
+    table_class_names: t.List[str]
+
+
+def create_pydantic_fixture_model(fixture_configs: t.List[FixtureConfig]):
+    """
+    Returns a nested Pydantic model for serialising and deserialising fixtures.
+    """
+    columns: t.Dict[str, t.Any] = {}
+
+    finder = Finder()
+
+    for fixture_config in fixture_configs:
+
+        app_columns: t.Dict[str, t.Any] = {}
+
+        for table_class_name in fixture_config.table_class_names:
+            table_class: t.Type[Table] = finder.get_table_with_name(
+                app_name=fixture_config.app_name,
+                table_class_name=table_class_name,
+            )
+            app_columns[table_class_name] = (
+                t.List[  # type: ignore
+                    create_pydantic_model(
+                        table_class, include_default_columns=True
+                    )
+                ],
+                ...,
+            )
+
+        app_model: t.Any = pydantic.create_model(
+            f"{fixture_config.app_name.title()}Model", **app_columns
+        )
+
+        columns[fixture_config.app_name] = (app_model, ...)
+
+    model: t.Type[pydantic.BaseModel] = pydantic.create_model(
+        "FixtureModel", **columns
+    )
+
+    return model

piccolo/apps/fixture/commands/load.py

@@ -0,0 +1,11 @@
+from piccolo.conf.apps import AppConfig
+
+from .commands.dump import dump
+
+APP_CONFIG = AppConfig(
+    app_name="fixtures",
+    migrations_folder_path="",
+    table_classes=[],
+    migration_dependencies=[],
+    commands=[dump],
+)

piccolo/apps/fixture/commands/shared.py

@@ -54,6 +54,10 @@ async def run_querystring(self, querystring: QueryString, in_pool: bool):
     async def run_ddl(self, ddl: str, in_pool: bool = True):
         pass
 
+    @abstractmethod
+    def transaction(self):
+        pass
+
     async def check_version(self):
         """
         Warn if the database version isn't supported.

piccolo/apps/fixture/piccolo_app.py

@@ -141,12 +141,21 @@ def convert_boolean_out(value: bytes) -> bool:
     return _value == "1"
 
 
+def convert_timestamp_out(value: bytes) -> datetime.datetime:
+    """
+    If the value is from a timestamp column, convert it to a datetime value.
+    """
+    return datetime.datetime.fromisoformat(value.decode("utf8"))
+
+
 def convert_timestamptz_out(value: bytes) -> datetime.datetime:
     """
-    If the value is from a timstamptz column, convert it to a datetime value,
+    If the value is from a timestamptz column, convert it to a datetime value,
     with a timezone of UTC.
     """
-    return datetime.datetime.fromisoformat(value.decode("utf8"))
+    _value = datetime.datetime.fromisoformat(value.decode("utf8"))
+    _value = _value.replace(tzinfo=datetime.timezone.utc)
+    return _value
 
 
 def convert_array_out(value: bytes) -> t.List:
@@ -164,6 +173,7 @@ def convert_array_out(value: bytes) -> t.List:
 sqlite3.register_converter("Time", convert_time_out)
 sqlite3.register_converter("Seconds", convert_seconds_out)
 sqlite3.register_converter("Boolean", convert_boolean_out)
+sqlite3.register_converter("Timestamp", convert_timestamp_out)
 sqlite3.register_converter("Timestamptz", convert_timestamptz_out)
 sqlite3.register_converter("Array", convert_array_out)