Merge pull request #73 from labthings/cbor

CBOR representations

Author: jtc42

examples/components/pdf_component.py

@@ -21,6 +21,7 @@ def __init__(self):
             "mode": "spectrum",
             "light_on": True,
             "user": {"name": "Squidward", "id": 1},
+            "bytes": b"\x80\x04\x95\x1a\x00\x00\x00\x00\x00\x00\x00\x8c\x08builtins\x94\x8c\x06object\x94\x93\x94)\x81\x94.",
         }
 
     def noisy_pdf(self, x, mu=0.0, sigma=25.0):

labthings/core/utilities.py

@@ -4,6 +4,7 @@
 import sys
 import os
 import copy
+import typing
 from functools import reduce
 
 PY3 = sys.version_info > (3,)
@@ -98,11 +99,11 @@ def rapply(data, func, *args, apply_to_iterables=True, **kwargs):
             )
             for key, val in data.items()
         }
-    # If the object is iterable but NOT a dictionary or a string
+    # If the object is a list, tuple, or range
     elif apply_to_iterables and (
-        isinstance(data, collections.abc.Iterable)
-        and not isinstance(data, collections.abc.Mapping)
-        and not isinstance(data, str)
+        isinstance(data, typing.List)
+        or isinstance(data, typing.Tuple)
+        or isinstance(data, range)
     ):
         return [
             rapply(x, func, *args, apply_to_iterables=apply_to_iterables, **kwargs)

labthings/server/fields.py

@@ -1,4 +1,8 @@
 # Marshmallow fields
+from marshmallow import ValidationError
+
+from base64 import b64decode
+
 from marshmallow.fields import (
     Field,
     Raw,
@@ -33,6 +37,7 @@
 )
 
 __all__ = [
+    "Bytes",
     "Field",
     "Raw",
     "Nested",
@@ -64,3 +69,20 @@
     "Constant",
     "Pluck",
 ]
+
+
+class Bytes(Field):
+    def _validate(self, value):
+        if not isinstance(value, bytes):
+            raise ValidationError("Invalid input type.")
+
+        if value is None or value == b"":
+            raise ValidationError("Invalid value")
+
+    def _deserialize(self, value, attr, data, **kwargs):
+        if isinstance(value, bytes):
+            return value
+        if isinstance(value, str):
+            return b64decode(value)
+        else:
+            raise self.make_error("invalid", input=value)

labthings/server/labthing.py

@@ -1,6 +1,7 @@
 from flask import url_for
 from apispec import APISpec
-from apispec.ext.marshmallow import MarshmallowPlugin
+
+# from apispec.ext.marshmallow import MarshmallowPlugin
 
 from .names import (
     EXTENSION_NAME,
@@ -14,6 +15,7 @@
 from .logging import LabThingLogger
 from .representations import LabThingsJSONEncoder
 from .spec.apispec import rule_to_apispec_path
+from .spec.apispec_plugins import MarshmallowPlugin
 from .spec.utilities import get_spec
 from .spec.td import ThingDescription
 from .decorators import tag
@@ -82,13 +84,15 @@ def __init__(
         self.log_handler = LabThingLogger()
         logging.getLogger().addHandler(self.log_handler)
 
+        # API Spec
         self.spec = APISpec(
             title=self.title,
             version=self.version,
             openapi_version="3.0.2",
             plugins=[MarshmallowPlugin()],
         )
 
+        # Thing description
         self.thing_description = ThingDescription(self.spec)
 
         if app is not None:

labthings/server/representations.py

@@ -2,7 +2,9 @@
 
 # Flask JSON encoder so we get UUID, datetime etc support
 from flask.json import JSONEncoder
-from json import dumps
+from base64 import b64encode
+import json
+import cbor2
 
 
 from ..core.utilities import PY3
@@ -16,17 +18,14 @@ class LabThingsJSONEncoder(JSONEncoder):
     def default(self, o):
         if isinstance(o, set):
             return list(o)
+        if isinstance(o, bytes):
+            return b64encode(o).decode()
         return JSONEncoder.default(self, o)
 
 
 def encode_json(data, encoder=LabThingsJSONEncoder, **settings):
     """Makes JSON encoded data using the LabThings JSON encoder"""
-
-    # always end the json dumps with a new line
-    # see https://github.com/mitsuhiko/flask/pull/1262
-    dumped = dumps(data, cls=encoder, **settings) + "\n"
-
-    return dumped
+    return json.dumps(data, cls=encoder, **settings) + "\n"
 
 
 def output_json(data, code, headers=None):
@@ -35,9 +34,6 @@ def output_json(data, code, headers=None):
     settings = current_app.config.get("LABTHINGS_JSON", {})
     encoder = current_app.json_encoder
 
-    # If we're in debug mode, and the indent is not set, we set it to a
-    # reasonable value here.  Note that this won't override any existing value
-    # that was set.  We also set the "sort_keys" value.
     if current_app.debug:
         settings.setdefault("indent", 4)
         settings.setdefault("sort_keys", not PY3)
@@ -46,7 +42,29 @@ def output_json(data, code, headers=None):
 
     resp = make_response(dumped, code)
     resp.headers.extend(headers or {})
+    resp.mimetype = "application/json"
+    return resp
+
+
+def encode_cbor(data, **settings):
+    """Makes CBOR encoded data using the default CBOR encoder"""
+    return cbor2.dumps(data, **settings)
+
+
+def output_cbor(data, code, headers=None):
+    """Makes a Flask response with a CBOR encoded body, using app CBOR settings"""
+
+    settings = current_app.config.get("LABTHINGS_CBOR", {})
+
+    dumped = encode_cbor(data, **settings)
+
+    resp = make_response(dumped, code)
+    resp.headers.extend(headers or {})
+    resp.mimetype = "application/cbor"
     return resp
 
 
-DEFAULT_REPRESENTATIONS = [("application/json", output_json)]
+DEFAULT_REPRESENTATIONS = [
+    ("application/json", output_json),
+    ("application/cbor", output_cbor),
+]

labthings/server/schema.py

@@ -1,32 +1,13 @@
 # -*- coding: utf-8 -*-
-from flask import jsonify, url_for
+from flask import url_for
 from werkzeug.routing import BuildError
-import marshmallow
+from marshmallow import Schema, pre_dump
 
 from .names import TASK_ENDPOINT, EXTENSION_LIST_ENDPOINT
 from .utilities import view_class_from_endpoint, description_from_view
 from . import fields
 
-sentinel = object()
-
-
-class Schema(marshmallow.Schema):
-    """Base serializer with which to define custom serializers.
-    See `marshmallow.Schema` for more details about the `Schema` API.
-    """
-
-    def jsonify(self, obj, *args, many=sentinel, **kwargs):
-        """Return a JSON response containing the serialized data.
-        :param obj: Object to serialize.
-        :param bool many: Whether `obj` should be serialized as an instance
-            or as a collection. If unset, defaults to the value of the
-            `many` attribute on this Schema.
-        :param kwargs: Additional keyword arguments passed to `flask.jsonify`.
-        """
-        if many is sentinel:
-            many = self.many
-        data = self.dump(obj, many=many)
-        return jsonify(data, *args, **kwargs)
+__all__ = ["Schema", "FieldSchema", "TaskSchema", "ExtensionSchema"]
 
 
 class FieldSchema:
@@ -64,17 +45,6 @@ def serialize(self, value):
     def dump(self, value):
         return self.serialize(value)
 
-    def jsonify(self, value):
-        """Serialize a value to JSON
-
-        Args:
-            value: Data to serialize
-
-        Returns:
-            Serialized JSON data
-        """
-        return jsonify(self.serialize(value))
-
 
 class TaskSchema(Schema):
     _ID = fields.String(data_key="id")
@@ -89,7 +59,7 @@ class TaskSchema(Schema):
 
     links = fields.Dict()
 
-    @marshmallow.pre_dump
+    @pre_dump
     def generate_links(self, data, **kwargs):
         try:
             url = url_for(TASK_ENDPOINT, task_id=data.id, _external=True)
@@ -114,7 +84,7 @@ class ExtensionSchema(Schema):
 
     links = fields.Dict()
 
-    @marshmallow.pre_dump
+    @pre_dump
     def generate_links(self, data, **kwargs):
         d = {}
         for view_id, view_data in data.views.items():

labthings/server/spec/apispec_plugins.py

@@ -0,0 +1,24 @@
+from apispec.ext.marshmallow import (
+    MarshmallowPlugin as _MarshmallowPlugin,
+    OpenAPIConverter,
+)
+from labthings.server.fields import Bytes as BytesField
+
+
+class ExtendedOpenAPIConverter(OpenAPIConverter):
+    field_mapping = OpenAPIConverter.field_mapping
+    field_mapping.update({BytesField: ("string", None)})
+
+    def init_attribute_functions(self, *args, **kwargs):
+        OpenAPIConverter.init_attribute_functions(self, *args, **kwargs)
+        self.attribute_functions.append(self.bytes2json)
+
+    def bytes2json(self, field, **kwargs):
+        ret = {}
+        if isinstance(field, BytesField):
+            ret.update({"contentEncoding": "base64"})
+        return ret
+
+
+class MarshmallowPlugin(_MarshmallowPlugin):
+    Converter = ExtendedOpenAPIConverter

labthings/server/types/registry.py

@@ -23,6 +23,7 @@
     Dict,
     List,
     list,
+    bytes,
 ]
 
 
@@ -59,6 +60,7 @@ def __init__(self) -> None:
                 UUID: fields.UUID,
                 dict: fields.Dict,
                 Dict: fields.Dict,
+                bytes: fields.Bytes,
                 _empty: fields.Field,
             }.items()
         }

poetry.lock

@@ -21,6 +21,7 @@ flask-cors = "^3.0.8"
 gevent = ">=1.4,<21.0"
 gevent-websocket = "^0.10.1"
 zeroconf = ">=0.24.5,<0.27.0"
+cbor2 = "^5.1.0"
 
 [tool.poetry.dev-dependencies]
 pytest = "^5.2"