Merge pull request piccolo-orm#122 from piccolo-orm/json_improvements

Json improvements

Author: dantownsend

docs/src/piccolo/query_clauses/index.rst

@@ -14,6 +14,7 @@ by modifying the return values.
     ./limit
     ./offset
     ./order_by
+    ./output
     ./where
     ./batch
     ./freeze

docs/src/piccolo/query_clauses/output.rst

@@ -0,0 +1,60 @@
+.. _output:
+
+output
+======
+
+You can use ``output`` clauses with the following queries:
+
+* :ref:`Select`
+* :ref:`Objects`
+
+-------------------------------------------------------------------------------
+
+Select queries only
+-------------------
+
+as_json
+~~~~~~~
+
+To return the data as a JSON string:
+
+.. code-block:: python
+
+    >>> Band.select().output(as_json=True).run_sync()
+    '[{"name":"Pythonistas","manager":1,"popularity":1000,"id":1},{"name":"Rustaceans","manager":2,"popularity":500,"id":2}]'
+
+Piccolo can use `orjson <https://github.com/ijl/orjson>`_ for JSON serialisation,
+which is blazing fast, and can handle most Python types, including dates,
+datetimes, and UUIDs. To install Piccolo with orjson support use
+``pip install piccolo[orjson]``.
+
+as_list
+~~~~~~~
+
+If you're just querying a single column from a database table, you can use
+``as_list`` to flatten the results into a single list.
+
+.. code-block:: python
+
+    >>> Band.select(Band.id).output(as_list=True).run_sync()
+    [1, 2]
+
+-------------------------------------------------------------------------------
+
+Select and Objects queries
+--------------------------
+
+load_json
+~~~~~~~~~
+
+If querying JSON or JSONB columns, you can tell Piccolo to deserialise the JSON
+values automatically.
+
+.. code-block:: python
+
+    >>> RecordingStudio.select().output(load_json=True).run_sync()
+    [{'id': 1, 'name': 'Abbey Road', 'facilities': {'restaurant': True, 'mixing_desk': True}}]
+
+    >>> studio = RecordingStudio.objects().first().output(load_json=True).run_sync()
+    >>> studio.facilities
+    {'restaurant': True, 'mixing_desk': True}

docs/src/piccolo/query_types/objects.rst

@@ -117,6 +117,11 @@ order_by
 
 See  :ref:`order_by`.
 
+output
+~~~~~~
+
+See  :ref:`output`.
+
 where
 ~~~~~
 

docs/src/piccolo/query_types/select.rst

@@ -147,21 +147,7 @@ See  :ref:`order_by`.
 output
 ~~~~~~
 
-By default, the data is returned as a list of dictionaries (where each
-dictionary represents a row). This can be altered using the ``output`` method.
-
-To return the data as a JSON string:
-
-.. code-block:: python
-
-    >>> b = Band
-    >>> b.select().output(as_json=True).run_sync()
-    '[{"name":"Pythonistas","manager":1,"popularity":1000,"id":1},{"name":"Rustaceans","manager":2,"popularity":500,"id":2}]'
-
-Piccolo can use `orjson <https://github.com/ijl/orjson>`_ for JSON serialisation,
-which is blazing fast, and can handle most Python types, including dates,
-datetimes, and UUIDs. To install Piccolo with orjson support use
-``pip install piccolo[orjson]``.
+See :ref:`output`.
 
 where
 ~~~~~

piccolo/apps/playground/commands/run.py

@@ -14,6 +14,7 @@
     ForeignKey,
     Integer,
     Interval,
+    JSON,
     Numeric,
     Timestamp,
     UUID,
@@ -55,7 +56,12 @@ class DiscountCode(Table):
     active = Boolean(default=True, null=True)
 
 
-TABLES = (Manager, Band, Venue, Concert, Ticket, DiscountCode)
+class RecordingStudio(Table):
+    name = Varchar(length=100)
+    facilities = JSON()
+
+
+TABLES = (Manager, Band, Venue, Concert, Ticket, DiscountCode, RecordingStudio)
 
 
 def populate():
@@ -105,6 +111,11 @@ def populate():
     discount_code = DiscountCode(code=uuid.uuid4())
     discount_code.save().run_sync()
 
+    recording_studio = RecordingStudio(
+        name="Abbey Road", facilities={"restaurant": True, "mixing_desk": True}
+    )
+    recording_studio.save().run_sync()
+
 
 def run(
     engine: str = "sqlite",

piccolo/apps/user/commands/change_permissions.py

@@ -34,7 +34,7 @@ async def change_permissions(
         )
         return
 
-    params: t.Dict[Column, bool] = {}
+    params: t.Dict[t.Union[Column, str], bool] = {}
 
     if admin is not None:
         params[BaseUser.admin] = admin

piccolo/columns/combination.py

@@ -1,10 +1,10 @@
 from __future__ import annotations
-from enum import Enum
 import typing as t
 
 from piccolo.columns.operators.comparison import ComparisonOperator
 from piccolo.custom_types import Combinable, Iterable
 from piccolo.querystring import QueryString
+from piccolo.utils.sql_values import convert_to_sql_value
 
 if t.TYPE_CHECKING:
     from piccolo.columns.base import Column
@@ -121,20 +121,17 @@ def clean_value(self, value: t.Any) -> t.Any:
             # it should still work.
             Band.select().where(Band.manager == guido).run_sync()
 
-        """
-        from piccolo.table import Table
+        Also, convert Enums to their underlying values, and serialise any JSON.
 
-        if isinstance(value, Table):
-            return value.id
-        else:
-            return value
+        """
+        return convert_to_sql_value(value=value, column=self.column)
 
     @property
     def values_querystring(self) -> QueryString:
-        if isinstance(self.values, Undefined):
-            raise ValueError("values is undefined")
+        values = self.values
 
-        values = [i.value if isinstance(i, Enum) else i for i in self.values]
+        if isinstance(values, Undefined):
+            raise ValueError("values is undefined")
 
         template = ", ".join(["{}" for _ in values])
         return QueryString(template, *values)
@@ -143,12 +140,8 @@ def values_querystring(self) -> QueryString:
     def querystring(self) -> QueryString:
         args: t.List[t.Any] = []
         if self.value != UNDEFINED:
-            value = (
-                self.value.value
-                if isinstance(self.value, Enum)
-                else self.value
-            )
-            args.append(value)
+            args.append(self.value)
+
         if self.values != UNDEFINED:
             args.append(self.values_querystring)
 

piccolo/columns/operators/comparison.py

@@ -1,4 +1,4 @@
-from .base import Operator
+from piccolo.columns.operators.base import Operator
 
 
 class ComparisonOperator(Operator):

piccolo/query/base.py

@@ -3,12 +3,15 @@
 from time import time
 import typing as t
 
+from piccolo.columns.column_types import JSON, JSONB
+from piccolo.query.mixins import ColumnsDelegate
 from piccolo.querystring import QueryString
 from piccolo.utils.sync import run_sync
-from piccolo.utils.encoding import dump_json
+from piccolo.utils.encoding import dump_json, load_json
 
 if t.TYPE_CHECKING:  # pragma: no cover
     from piccolo.table import Table  # noqa
+    from piccolo.query.mixins import OutputDelegate
 
 
 class Timer:
@@ -51,9 +54,54 @@ async def _process_results(self, results):
         if hasattr(self, "run_callback"):
             self.run_callback(raw)
 
-        raw = await self.response_handler(raw)
+        output: t.Optional[OutputDelegate] = getattr(
+            self, "output_delegate", None
+        )
+
+        #######################################################################
+
+        if output and output._output.load_json:
+            columns_delegate: t.Optional[ColumnsDelegate] = getattr(
+                self, "columns_delegate", None
+            )
+
+            if columns_delegate is not None:
+                json_columns = [
+                    i
+                    for i in columns_delegate.selected_columns
+                    if isinstance(i, (JSON, JSONB))
+                ]
+            else:
+                json_columns = self.table._meta.json_columns
+
+            json_column_names = []
+            for column in json_columns:
+                if column.alias is not None:
+                    json_column_names.append(column.alias)
+                elif len(column._meta.call_chain) > 0:
+                    json_column_names.append(
+                        column.get_select_string(
+                            engine_type=column._meta.engine_type
+                        )
+                    )
+                else:
+                    json_column_names.append(column._meta.name)
+
+            processed_raw = []
+
+            for row in raw:
+                new_row = {**row}
+                for json_column_name in json_column_names:
+                    value = new_row.get(json_column_name)
+                    if value is not None:
+                        new_row[json_column_name] = load_json(value)
+                processed_raw.append(new_row)
 
-        output = getattr(self, "output_delegate", None)
+            raw = processed_raw
+
+        #######################################################################
+
+        raw = await self.response_handler(raw)
 
         if output:
             if output._output.as_objects:

piccolo/query/methods/__init__.py

@@ -1,4 +1,4 @@
-from .alter import Alter  # noqa: F401: F401
+from .alter import Alter  # noqa: F401
 from .count import Count  # noqa: F401
 from .create import Create  # noqa: F401
 from .create_index import CreateIndex  # noqa: F401