Datetime serialization (#851)

* Add datetime and uuid serialization for AioHttp

* Remove ujson dependency

* fix merge error

* Retry CI

* remove bad jsonifier import

* remove ujson import

.gitignore

@@ -11,4 +11,4 @@ htmlcov/
 *.swp
 .tox/
 .idea/
-venv/
+venv/

connexion/apis/abstract.py

@@ -7,11 +7,11 @@ import six
 
 from ..exceptions import ResolverError
 from ..http_facts import METHODS
+from ..jsonifier import Jsonifier
 from ..operations import make_operation
 from ..options import ConnexionOptions
 from ..resolver import Resolver
 from ..spec import Specification
-from ..utils import Jsonifier
 
 MODULE_PATH = pathlib.Path(__file__).absolute().parent.parent
 SWAGGER_UI_URL = 'ui'
@@ -260,5 +260,4 @@ class AbstractAPI(object):
 
     @classmethod
     def _set_jsonifier(cls):
-        import json
-        cls.jsonifier = Jsonifier(json)
+        cls.jsonifier = Jsonifier()

connexion/apis/aiohttp_api.py

@@ -14,18 +14,12 @@ from aiohttp.web_middlewares import normalize_path_middleware
 from connexion.apis.abstract import AbstractAPI
 from connexion.exceptions import ProblemException
 from connexion.handlers import AuthErrorHandler
+from connexion.jsonifier import JSONEncoder, Jsonifier
 from connexion.lifecycle import ConnexionRequest, ConnexionResponse
 from connexion.problem import problem
-from connexion.utils import Jsonifier, is_json_mimetype, yamldumper
+from connexion.utils import is_json_mimetype, yamldumper
 from werkzeug.exceptions import HTTPException as werkzeug_HTTPException
 
-try:
-    import ujson as json
-    from functools import partial
-    json.dumps = partial(json.dumps, escape_forward_slashes=True)
-
-except ImportError:  # pragma: no cover
-    import json
 
 logger = logging.getLogger('connexion.apis.aiohttp_api')
 
@@ -367,7 +361,7 @@ class AioHttpApi(AbstractAPI):
     def _cast_body(cls, body, content_type=None):
         if not isinstance(body, bytes):
             if content_type and is_json_mimetype(content_type):
-                return json.dumps(body).encode()
+                return cls.jsonifier.dumps(body).encode()
 
             elif isinstance(body, str):
                 return body.encode()
@@ -379,7 +373,7 @@ class AioHttpApi(AbstractAPI):
 
     @classmethod
     def _set_jsonifier(cls):
-        cls.jsonifier = Jsonifier(json)
+        cls.jsonifier = Jsonifier(cls=JSONEncoder)
 
 
 class _HttpNotFoundError(HTTPNotFound):

connexion/apis/flask_api.py

@@ -7,8 +7,9 @@ from connexion.apis import flask_utils
 from connexion.apis.abstract import AbstractAPI
 from connexion.decorators.produces import NoContent
 from connexion.handlers import AuthErrorHandler
+from connexion.jsonifier import Jsonifier
 from connexion.lifecycle import ConnexionRequest, ConnexionResponse
-from connexion.utils import Jsonifier, is_json_mimetype, yamldumper
+from connexion.utils import is_json_mimetype, yamldumper
 from werkzeug.local import LocalProxy
 
 logger = logging.getLogger('connexion.apis.flask_api')
@@ -283,7 +284,7 @@ class FlaskApi(AbstractAPI):
         """
         Use Flask specific JSON loader
         """
-        cls.jsonifier = Jsonifier(flask.json)
+        cls.jsonifier = Jsonifier(flask.json, indent=2)
 
 
 def _get_context():

connexion/jsonifier.py

@@ -0,0 +1,59 @@
+import datetime
+import json
+import uuid
+
+import six
+
+
+class JSONEncoder(json.JSONEncoder):
+    def default(self, o):
+        if isinstance(o, datetime.datetime):
+            if o.tzinfo:
+                # eg: '2015-09-25T23:14:42.588601+00:00'
+                return o.isoformat('T')
+            else:
+                # No timezone present - assume UTC.
+                # eg: '2015-09-25T23:14:42.588601Z'
+                return o.isoformat('T') + 'Z'
+
+        if isinstance(o, datetime.date):
+            return o.isoformat()
+
+        if isinstance(o, uuid.UUID):
+            return str(o)
+
+        return json.JSONEncoder.default(self, o)
+
+
+class Jsonifier(object):
+    """
+    Used to serialized and deserialize to/from JSon
+    """
+    def __init__(self, json_=json, **kwargs):
+        """
+        :param json_: json library to use. Must have loads() and dumps() method
+        :param kwargs: default arguments to pass to json.dumps()
+        """
+        self.json = json_
+        self.dumps_args = kwargs
+
+    def dumps(self, data, **kwargs):
+        """ Central point where JSON serialization happens inside
+        Connexion.
+        """
+        for k, v in six.iteritems(self.dumps_args):
+            kwargs.setdefault(k, v)
+        return self.json.dumps(data, **kwargs) + '\n'
+
+    def loads(self, data):
+        """ Central point where JSON deserialization happens inside
+        Connexion.
+        """
+        if isinstance(data, six.binary_type):
+            data = data.decode()
+
+        try:
+            return self.json.loads(data)
+        except Exception:
+            if isinstance(data, six.string_types):
+                return data

connexion/utils.py

@@ -165,30 +165,6 @@ def is_null(value):
     return False
 
 
-class Jsonifier(object):
-    def __init__(self, json_):
-        self.json = json_
-
-    def dumps(self, data):
-        """ Central point where JSON serialization happens inside
-        Connexion.
-        """
-        return "{}\n".format(self.json.dumps(data, indent=2))
-
-    def loads(self, data):
-        """ Central point where JSON serialization happens inside
-        Connexion.
-        """
-        if isinstance(data, six.binary_type):
-            data = data.decode()
-
-        try:
-            return self.json.loads(data)
-        except Exception:
-            if isinstance(data, six.string_types):
-                return data
-
-
 def has_coroutine(function, api=None):
     """
     Checks if function is a coroutine.

examples/openapi3/helloworld_aiohttp/README.rst

@@ -0,0 +1,11 @@
+===================
+Hello World Example
+===================
+
+Running:
+
+.. code-block:: bash
+
+    $ ./hello.py
+
+Now open your browser and go to http://localhost:9090/v1.0/ui/ to see the Swagger UI.

examples/openapi3/helloworld_aiohttp/hello.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+import asyncio
+
+import connexion
+from aiohttp import web
+
+
+@asyncio.coroutine
+def post_greeting(name):
+    return web.Response(text='Hello {name}'.format(name=name))
+
+
+if __name__ == '__main__':
+    app = connexion.AioHttpApp(__name__, port=9090, specification_dir='openapi/')
+    app.add_api('helloworld-api.yaml', arguments={'title': 'Hello World Example'})
+    app.run()

examples/openapi3/helloworld_aiohttp/openapi/helloworld-api.yaml

@@ -0,0 +1,30 @@
+openapi: "3.0.0"
+
+info:
+  title: Hello World
+  version: "1.0"
+servers:
+  - url: http://localhost:9090/v1.0
+
+paths:
+  /greeting/{name}:
+    post:
+      summary: Generate greeting
+      description: Generates a greeting message.
+      operationId: hello.post_greeting
+      responses:
+        200:
+          description: greeting response
+          content:
+            text/plain:
+              schema:
+                type: string
+                example: "hello dave!"
+      parameters:
+        - name: name
+          in: path
+          description: Name of the person to greet.
+          required: true
+          schema:
+            type: string
+            example: "dave"

requirements-aiohttp.txt

@@ -1,4 +1,3 @@
 aiohttp>=2.2.5
 aiohttp-swagger>=1.0.5
-ujson>=1.35
 aiohttp_jinja2==0.15.0

setup.py

@@ -36,7 +36,6 @@ aiohttp_require = [
     'aiohttp>=2.3.10',
     'aiohttp-jinja2>=0.14.0'
 ]
-ujson_require = 'ujson>=1.35'
 
 tests_require = [
     'decorator',
@@ -50,7 +49,6 @@ tests_require = [
 
 if sys.version_info[0] >= 3:
     tests_require.extend(aiohttp_require)
-    tests_require.append(ujson_require)
     tests_require.append('pytest-aiohttp')
 
 
@@ -108,8 +106,7 @@ setup(
         'tests': tests_require,
         'flask': flask_require,
         'swagger-ui': swagger_ui_require,
-        'aiohttp': aiohttp_require,
-        'ujson': ujson_require
+        'aiohttp': aiohttp_require
     },
     cmdclass={'test': PyTest},
     test_suite='tests',

tests/aiohttp/test_aiohttp_api_secure.py

@@ -1,9 +1,6 @@
 import asyncio
 import base64
 
-import ujson
-
-from conftest import TEST_FOLDER
 from connexion import AioHttpApp
 
 
@@ -50,7 +47,7 @@ def test_secure_app(oauth_requests, aiohttp_api_spec_dir, aiohttp_client):
     )
 
     assert post_hello.status == 200
-    assert (yield from post_hello.read()) == b'{"greeting":"Hello jsantos"}'
+    assert (yield from post_hello.json()) == {"greeting": "Hello jsantos"}
 
     headers = {'authorization': 'Bearer 100'}
     post_hello = yield from app_client.post(
@@ -59,7 +56,7 @@ def test_secure_app(oauth_requests, aiohttp_api_spec_dir, aiohttp_client):
     )
 
     assert post_hello.status == 200, "Authorization header in lower case should be accepted"
-    assert (yield from post_hello.read()) == b'{"greeting":"Hello jsantos"}'
+    assert (yield from post_hello.json()) == {"greeting": "Hello jsantos"}
 
     headers = {'AUTHORIZATION': 'Bearer 100'}
     post_hello = yield from app_client.post(
@@ -68,7 +65,7 @@ def test_secure_app(oauth_requests, aiohttp_api_spec_dir, aiohttp_client):
     )
 
     assert post_hello.status == 200, "Authorization header in upper case should be accepted"
-    assert (yield from post_hello.read()) == b'{"greeting":"Hello jsantos"}'
+    assert (yield from post_hello.json()) == {"greeting": "Hello jsantos"}
 
     no_authorization = yield from app_client.post(
         '/v1.0/greeting/jsantos',

tests/aiohttp/test_aiohttp_datetime.py

@@ -0,0 +1,49 @@
+import asyncio
+
+from connexion import AioHttpApp
+
+try:
+    import ujson as json
+except ImportError:
+    import json
+
+
+@asyncio.coroutine
+def test_swagger_json(aiohttp_api_spec_dir, aiohttp_client):
+    """ Verify the swagger.json file is returned for default setting passed to app. """
+    app = AioHttpApp(__name__, port=5001,
+                     specification_dir=aiohttp_api_spec_dir,
+                     debug=True)
+    app.add_api('datetime_support.yaml')
+
+    app_client = yield from aiohttp_client(app.app)
+    swagger_json = yield from app_client.get('/v1.0/openapi.json')
+    spec_data = yield from swagger_json.json()
+
+    def get_value(data, path):
+        for part in path.split('.'):
+            data = data.get(part)
+            assert data, "No data in part '{}' of '{}'".format(part, path)
+        return data
+
+    example = get_value(spec_data, 'paths./datetime.get.responses.200.content.application/json.schema.example.value')
+    assert example == '2000-01-23T04:56:07.000008Z'
+    example = get_value(spec_data, 'paths./date.get.responses.200.content.application/json.schema.example.value')
+    assert example == '2000-01-23'
+    example = get_value(spec_data, 'paths./uuid.get.responses.200.content.application/json.schema.example.value')
+    assert example == 'a7b8869c-5f24-4ce0-a5d1-3e44c3663aa9'
+
+    resp = yield from app_client.get('/v1.0/datetime')
+    assert resp.status == 200
+    json_data = yield from resp.json()
+    assert json_data == {'value': '2000-01-02T03:04:05.000006Z'}
+
+    resp = yield from app_client.get('/v1.0/date')
+    assert resp.status == 200
+    json_data = yield from resp.json()
+    assert json_data == {'value': '2000-01-02'}
+
+    resp = yield from app_client.get('/v1.0/uuid')
+    assert resp.status == 200
+    json_data = yield from resp.json()
+    assert json_data == {'value': 'e7ff66d0-3ec2-4c4e-bed0-6e4723c24c51'}

tests/aiohttp/test_aiohttp_simple_api.py

@@ -57,10 +57,10 @@ def test_swagger_json(aiohttp_api_spec_dir, aiohttp_client):
 
     app_client = yield from aiohttp_client(app.app)
     swagger_json = yield from app_client.get('/v1.0/swagger.json')
-    json_ = yield from swagger_json.read()
 
     assert swagger_json.status == 200
-    assert api.specification.raw == json.loads(json_)
+    json_ = yield from swagger_json.json()
+    assert api.specification.raw == json_
 
 
 @asyncio.coroutine

tests/conftest.py

@@ -97,6 +97,11 @@ def json_validation_spec_dir():
     return FIXTURES_FOLDER / 'json_validation'
 
 
+@pytest.fixture(scope='session')
+def json_datetime_dir():
+    return FIXTURES_FOLDER / 'datetime_support'
+
+
 def build_app_from_fixture(api_spec_folder, spec_file='openapi.yaml', **kwargs):
     debug = True
     if 'debug' in kwargs:

tests/fakeapi/aiohttp_handlers.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python3
 import asyncio
+import datetime
+import uuid
 
 import aiohttp
 from aiohttp.web import Request
@@ -80,3 +82,18 @@ def aiohttp_users_post(user):
     user['id'] = len(USERS) + 1
     USERS.append(user)
     return aiohttp.web.json_response(data=USERS[-1], status=201)
+
+
+@asyncio.coroutine
+def get_datetime():
+    return ConnexionResponse(body={'value': datetime.datetime(2000, 1, 2, 3, 4, 5, 6)})
+
+
+@asyncio.coroutine
+def get_date():
+    return ConnexionResponse(body={'value': datetime.date(2000, 1, 2)})
+
+
+@asyncio.coroutine
+def get_uuid():
+    return ConnexionResponse(body={'value': uuid.UUID(hex='e7ff66d0-3ec2-4c4e-bed0-6e4723c24c51')})

tests/fakeapi/hello.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python3
-import flask
+import datetime
+import uuid
+
 from flask import jsonify, redirect
 
 from connexion import NoContent, ProblemException, context
@@ -575,3 +577,15 @@ def patch_add_operation_on_http_methods_only():
 
 def trace_add_operation_on_http_methods_only():
     return ""
+
+
+def get_datetime():
+    return {'value': datetime.datetime(2000, 1, 2, 3, 4, 5, 6)}
+
+
+def get_date():
+    return {'value': datetime.date(2000, 1, 2)}
+
+
+def get_uuid():
+    return {'value': uuid.UUID(hex='e7ff66d0-3ec2-4c4e-bed0-6e4723c24c51')}

tests/fixtures/aiohttp/datetime_support.yaml

@@ -0,0 +1,60 @@
+openapi: "3.0.1"
+
+info:
+  title: "{{title}}"
+  version: "1.0"
+servers:
+  - url: http://localhost:8080/v1.0
+
+paths:
+  /datetime:
+    get:
+      summary: Generate data with date time
+      operationId: fakeapi.aiohttp_handlers.get_datetime
+      responses:
+        200:
+          description: date time example
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  value:
+                    type: string
+                    format: date-time
+                example:
+                  value: 2000-01-23T04:56:07.000008+00:00
+  /date:
+    get:
+      summary: Generate data with date
+      operationId: fakeapi.aiohttp_handlers.get_date
+      responses:
+        200:
+          description: date example
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  value:
+                    type: string
+                    format: date
+                example:
+                  value: 2000-01-23
+  /uuid:
+    get:
+      summary: Generate data with uuid
+      operationId: fakeapi.aiohttp_handlers.get_uuid
+      responses:
+        200:
+          description: uuid handler
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  value:
+                    type: string
+                    format: uuid
+                example:
+                  value: 'a7b8869c-5f24-4ce0-a5d1-3e44c3663aa9'

tests/fixtures/datetime_support/openapi.yaml

@@ -0,0 +1,60 @@
+openapi: "3.0.1"
+
+info:
+  title: "{{title}}"
+  version: "1.0"
+servers:
+  - url: http://localhost:8080/v1.0
+
+paths:
+  /datetime:
+    get:
+      summary: Generate data with date time
+      operationId: fakeapi.hello.get_datetime
+      responses:
+        200:
+          description: date time example
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  value:
+                    type: string
+                    format: date-time
+                example:
+                  value: 2000-01-23T04:56:07.000008+00:00
+  /date:
+    get:
+      summary: Generate data with date
+      operationId: fakeapi.hello.get_date
+      responses:
+        200:
+          description: date example
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  value:
+                    type: string
+                    format: date
+                example:
+                  value: 2000-01-23
+  /uuid:
+    get:
+      summary: Generate data with uuid
+      operationId: fakeapi.hello.get_uuid
+      responses:
+        200:
+          description: uuid example
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  value:
+                    type: string
+                    format: uuid
+                example:
+                  value: 'a7b8869c-5f24-4ce0-a5d1-3e44c3663aa9'

tests/fixtures/datetime_support/swagger.yaml

@@ -0,0 +1,52 @@
+swagger: "2.0"
+
+info:
+  title: "{{title}}"
+  version: "1.0"
+
+basePath: /v1.0
+
+paths:
+  /datetime:
+    get:
+      operationId: fakeapi.hello.get_datetime
+      responses:
+        200:
+          description: date time example
+          schema:
+            type: object
+            properties:
+              value:
+                type: string
+                format: date-time
+            example:
+              value: 2000-01-23T04:56:07.000008+00:00
+  /date:
+    get:
+      operationId: fakeapi.hello.get_date
+      responses:
+        200:
+          description: date example
+          schema:
+            type: object
+            properties:
+              value:
+                type: string
+                format: date
+            example:
+              value: 2000-01-23
+  /uuid:
+    get:
+      summary: Generate data with uuid
+      operationId: fakeapi.hello.get_uuid
+      responses:
+        200:
+          description: uuid example
+          schema:
+            type: object
+            properties:
+              value:
+                type: string
+                format: uuid
+            example:
+              value: 'a7b8869c-5f24-4ce0-a5d1-3e44c3663aa9'

tests/test_flask_encoder.py

@@ -3,8 +3,13 @@ import json
 import math
 from decimal import Decimal
 
+import pytest
+
+from conftest import build_app_from_fixture
 from connexion.apps.flask_app import FlaskJSONEncoder
 
+SPECS = ["swagger.yaml", "openapi.yaml"]
+
 
 def test_json_encoder():
     s = json.dumps({1: 2}, cls=FlaskJSONEncoder)
@@ -35,3 +40,46 @@ def test_json_encoder_datetime_with_timezone():
 
     s = json.dumps(datetime.datetime.now(DummyTimezone()), cls=FlaskJSONEncoder)
     assert s.endswith('+00:00"')
+
+
+@pytest.mark.parametrize("spec", SPECS)
+def test_readonly(json_datetime_dir, spec):
+    app = build_app_from_fixture(json_datetime_dir, spec, validate_responses=True)
+    app_client = app.app.test_client()
+
+    res = app_client.get('/v1.0/' + spec.replace('yaml', 'json'))
+    assert res.status_code == 200, "Error is {}".format(res.data)
+    spec_data = json.loads(res.data.decode())
+
+    if spec == 'openapi.yaml':
+        response_path = 'responses.200.content.application/json.schema'
+    else:
+        response_path = 'responses.200.schema'
+
+    def get_value(data, path):
+        for part in path.split('.'):
+            data = data.get(part)
+            assert data, "No data in part '{}' of '{}'".format(part, path)
+        return data
+
+    example = get_value(spec_data, 'paths./datetime.get.{}.example.value'.format(response_path))
+    assert example == '2000-01-23T04:56:07.000008Z'
+    example = get_value(spec_data, 'paths./date.get.{}.example.value'.format(response_path))
+    assert example == '2000-01-23'
+    example = get_value(spec_data, 'paths./uuid.get.{}.example.value'.format(response_path))
+    assert example == 'a7b8869c-5f24-4ce0-a5d1-3e44c3663aa9'
+
+    res = app_client.get('/v1.0/datetime')
+    assert res.status_code == 200, "Error is {}".format(res.data)
+    data = json.loads(res.data.decode())
+    assert data == {'value': '2000-01-02T03:04:05.000006Z'}
+
+    res = app_client.get('/v1.0/date')
+    assert res.status_code == 200, "Error is {}".format(res.data)
+    data = json.loads(res.data.decode())
+    assert data == {'value': '2000-01-02'}
+
+    res = app_client.get('/v1.0/uuid')
+    assert res.status_code == 200, "Error is {}".format(res.data)
+    data = json.loads(res.data.decode())
+    assert data == {'value': 'e7ff66d0-3ec2-4c4e-bed0-6e4723c24c51'}