Nested objects prototype (piccolo-orm#217)

* prototype for nested objects

* fix typo

* finish prototype

* fix linting errors

* added `all_related` method

* more tests

* adds more tests and docs

* add notes about the `prefetch` clause

* make sure `prefetch` work with `get` and `get_or_create`

* load all intermediate objects

* another test

* update docs - declaring intermediate objects no longer required

Author: dantownsend

docs/src/piccolo/query_types/objects.rst

@@ -11,6 +11,8 @@ can manipulate them, and save the changes back to the database.
 In Piccolo, an instance of a ``Table`` class represents a row. Let's do some
 examples.
 
+-------------------------------------------------------------------------------
+
 Fetching objects
 ----------------
 
@@ -45,6 +47,8 @@ To get the first row:
 You'll notice that the API is similar to :ref:`Select` - except it returns all
 columns.
 
+-------------------------------------------------------------------------------
+
 Creating objects
 ----------------
 
@@ -53,19 +57,23 @@ Creating objects
     >>> band = Band(name="C-Sharps", popularity=100)
     >>> band.save().run_sync()
 
+-------------------------------------------------------------------------------
+
 Updating objects
 ----------------
 
 Objects have a ``save`` method, which is convenient for updating values:
 
 .. code-block:: python
 
-    pythonistas = Band.objects().where(
+    band = Band.objects().where(
         Band.name == 'Pythonistas'
     ).first().run_sync()
 
-    pythonistas.popularity = 100000
-    pythonistas.save().run_sync()
+    band.popularity = 100000
+    band.save().run_sync()
+
+-------------------------------------------------------------------------------
 
 Deleting objects
 ----------------
@@ -74,28 +82,95 @@ Similarly, we can delete objects, using the ``remove`` method.
 
 .. code-block:: python
 
-    pythonistas = Band.objects().where(
+    band = Band.objects().where(
         Band.name == 'Pythonistas'
     ).first().run_sync()
 
-    pythonistas.remove().run_sync()
+    band.remove().run_sync()
+
+-------------------------------------------------------------------------------
+
+Fetching related objects
+------------------------
 
 get_related
------------
+~~~~~~~~~~~
+
+If you have an object from a table with a ``ForeignKey`` column, and you want
+to fetch the related row as an object, you can do so using ``get_related``.
+
+.. code-block:: python
 
-If you have an object with a foreign key, and you want to fetch the related
-object, you can do so using ``get_related``.
+    band = Band.objects().where(
+        Band.name == 'Pythonistas'
+    ).first().run_sync()
+
+    manager = band.get_related(Band.manager).run_sync()
+    >>> manager
+    <Manager: 1>
+    >>> manager.name
+    'Guido'
+
+Prefetching related objects
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can also prefetch the rows from related tables, and store them as child
+objects. To do this, pass ``ForeignKey`` columns into ``objects``, which
+refer to the related rows you want to load.
 
 .. code-block:: python
 
-    pythonistas = Band.objects().where(
+    band = Band.objects(Band.manager).where(
         Band.name == 'Pythonistas'
     ).first().run_sync()
 
-    manager = pythonistas.get_related(Band.manager).run_sync()
-    >>> print(manager.name)
+    >>> band.manager
+    <Manager: 1>
+    >>> band.manager.name
     'Guido'
 
+If you have a table containing lots of ``ForeignKey`` columns, and want to
+prefetch them all you can do so using ``all_related``.
+
+.. code-block:: python
+
+    ticket = Ticket.objects(
+        Ticket.concert.all_related()
+    ).first().run_sync()
+
+    # Any intermediate objects will also be loaded:
+    >>> ticket.concert
+    <Concert: 1>
+
+    >>> ticket.concert.band_1
+    <Band: 1>
+    >>> ticket.concert.band_2
+    <Band: 2>
+
+You can manipulate these nested objects, and save the values back to the
+database, just as you would expect:
+
+.. code-block:: python
+
+    ticket.concert.band_1.name = 'Pythonistas 2'
+    ticket.concert.band_1.save().run_sync()
+
+Instead of passing the ``ForeignKey`` columns into the ``objects`` method, you
+can use the ``prefetch`` clause if you prefer.
+
+.. code-block:: python
+
+    # These are equivalent:
+    ticket = Ticket.objects(
+        Ticket.concert.all_related()
+    ).first().run_sync()
+
+    ticket = Ticket.objects().prefetch(
+        Ticket.concert.all_related()
+    ).run_sync()
+
+-------------------------------------------------------------------------------
+
 get_or_create
 -------------
 
@@ -138,6 +213,8 @@ Complex where clauses are supported, but only within reason. For example:
         defaults={'popularity': 100}
     ).run_sync()
 
+-------------------------------------------------------------------------------
+
 to_dict
 -------
 
@@ -161,6 +238,7 @@ the columns:
     >>> band.to_dict(Band.id, Band.name.as_alias('title'))
     {'id': 1, 'title': 'Pythonistas'}
 
+-------------------------------------------------------------------------------
 
 Query clauses
 -------------

piccolo/columns/column_types.py

@@ -1224,14 +1224,23 @@ def all_columns(
         self, exclude: t.List[t.Union[Column, str]] = []
     ) -> t.List[Column]:
         """
-        Allow a user to access all of the columns on the related table.
+        Allow a user to access all of the columns on the related table. This is
+        intended for use with ``select`` queries, and saves the user from
+        typing out all of the columns by hand.
 
         For example:
 
         .. code-block:: python
 
             Band.select(Band.name, Band.manager.all_columns()).run_sync()
 
+            # Equivalent to:
+            Band.select(
+                Band.name,
+                Band.manager.id,
+                Band.manager.name
+            ).run_sync()
+
         To exclude certain columns:
 
         .. code-block:: python
@@ -1260,6 +1269,60 @@ def all_columns(
             if column._meta.name not in excluded_column_names
         ]
 
+    def all_related(
+        self, exclude: t.List[t.Union[ForeignKey, str]] = []
+    ) -> t.List[ForeignKey]:
+        """
+        Returns each ``ForeignKey`` column on the related table. This is
+        intended for use with ``objects`` queries, where you want to return
+        all of the related tables as nested objects.
+
+        For example:
+
+        .. code-block:: python
+
+            class Band(Table):
+                name = Varchar()
+
+            class Concert(Table):
+                name = Varchar()
+                band_1 = ForeignKey(Band)
+                band_2 = ForeignKey(Band)
+
+            class Tour(Table):
+                name = Varchar()
+                concert = ForeignKey(Concert)
+
+            Tour.objects(Tour.concert, Tour.concert.all_related()).run_sync()
+
+            # Equivalent to
+            Tour.objects(
+                Tour.concert,
+                Tour.concert.band_1,
+                Tour.concert.band_2
+            ).run_sync()
+
+        :param exclude:
+            Columns to exclude - can be the name of a column, or a
+            ``ForeignKey`` instance. For example ``['band_1']`` or
+            ``[Tour.concert.band_1]``.
+
+        """
+        _fk_meta: ForeignKeyMeta = object.__getattribute__(
+            self, "_foreign_key_meta"
+        )
+        related_fk_columns = (
+            _fk_meta.resolved_references._meta.foreign_key_columns
+        )
+        excluded_column_names = [
+            i._meta.name if isinstance(i, ForeignKey) else i for i in exclude
+        ]
+        return [
+            getattr(self, fk_column._meta.name)
+            for fk_column in related_fk_columns
+            if fk_column._meta.name not in excluded_column_names
+        ]
+
     def set_proxy_columns(self):
         """
         In order to allow a fluent interface, where tables can be traversed
@@ -1334,7 +1397,6 @@ def __getattribute__(self, name: str):
                 _column._meta.call_chain = [
                     i for i in new_column._meta.call_chain
                 ]
-                _column._meta.call_chain.append(new_column)
                 setattr(new_column, _column._meta.name, _column)
                 foreign_key_meta.proxy_columns.append(_column)
 

piccolo/query/base.py

@@ -8,6 +8,7 @@
 from piccolo.query.mixins import ColumnsDelegate
 from piccolo.querystring import QueryString
 from piccolo.utils.encoding import dump_json, load_json
+from piccolo.utils.objects import make_nested_object
 from piccolo.utils.sync import run_sync
 
 if t.TYPE_CHECKING:  # pragma: no cover
@@ -109,14 +110,22 @@ async def _process_results(self, results):  # noqa: C901
                 # When using .first() we get a single row, not a list
                 # of rows.
                 if type(raw) is list:
-                    raw = [
-                        self.table(**columns, exists_in_db=True)
-                        for columns in raw
-                    ]
+                    if output._output.nested:
+                        raw = [
+                            make_nested_object(row, self.table) for row in raw
+                        ]
+                    else:
+                        raw = [
+                            self.table(**columns, exists_in_db=True)
+                            for columns in raw
+                        ]
                 elif raw is None:
                     pass
                 else:
-                    raw = self.table(**raw, exists_in_db=True)
+                    if output._output.nested:
+                        raw = make_nested_object(raw, self.table)
+                    else:
+                        raw = self.table(**raw, exists_in_db=True)
             elif type(raw) is list:
                 if output._output.as_list:
                     if len(raw) == 0: