From 50062c4d8c4108d17b7f12d9518ce883956d3921 Mon Sep 17 00:00:00 2001

From: David Lord <davidism@gmail.com>

Date: Tue, 10 Apr 2018 09:29:48 -0700

Subject: [PATCH] detect UTF encodings when loading json

(cherry picked from commit 0e1e9a04aaf29ab78f721cfc79ac2a691f6e3929)

---

 flask/json.py         | 49 ++++++++++++++++++++++++++++++++++++++++++-

 flask/wrappers.py     | 13 +++---------

 tests/test_helpers.py | 28 ++++++++++++++-----------

 3 files changed, 67 insertions(+), 23 deletions(-)

diff --git a/flask/json.py b/flask/json.py

index 16e0c29..114873e 100644

--- a/flask/json.py

+++ b/flask/json.py

@@ -8,6 +8,7 @@

     :copyright: (c) 2015 by Armin Ronacher.

     :license: BSD, see LICENSE for more details.

 """

+import codecs

 import io

 import uuid

 from datetime import date

@@ -108,6 +109,49 @@ def _load_arg_defaults(kwargs):

         kwargs.setdefault('cls', JSONDecoder)

+def detect_encoding(data):

+    """Detect which UTF codec was used to encode the given bytes.

+

+    The latest JSON standard (:rfc:`8259`) suggests that only UTF-8 is

+    accepted. Older documents allowed 8, 16, or 32. 16 and 32 can be big

+    or little endian. Some editors or libraries may prepend a BOM.

+

+    :param data: Bytes in unknown UTF encoding.

+    :return: UTF encoding name

+    """

+    head = data[:4]

+

+    if head[:3] == codecs.BOM_UTF8:

+        return 'utf-8-sig'

+

+    if b'\x00' not in head:

+        return 'utf-8'

+

+    if head in (codecs.BOM_UTF32_BE, codecs.BOM_UTF32_LE):

+        return 'utf-32'

+

+    if head[:2] in (codecs.BOM_UTF16_BE, codecs.BOM_UTF16_LE):

+        return 'utf-16'

+

+    if len(head) == 4:

+        if head[:3] == b'\x00\x00\x00':

+            return 'utf-32-be'

+

+        if head[::2] == b'\x00\x00':

+            return 'utf-16-be'

+

+        if head[1:] == b'\x00\x00\x00':

+            return 'utf-32-le'

+

+        if head[1::2] == b'\x00\x00':

+            return 'utf-16-le'

+

+    if len(head) == 2:

+        return 'utf-16-be' if head.startswith(b'\x00') else 'utf-16-le'

+

+    return 'utf-8'

+

+

 def dumps(obj, **kwargs):

     """Serialize ``obj`` to a JSON formatted ``str`` by using the application's

     configured encoder (:attr:`~flask.Flask.json_encoder`) if there is an

@@ -142,7 +186,10 @@ def loads(s, **kwargs):

     """

     _load_arg_defaults(kwargs)

     if isinstance(s, bytes):

-        s = s.decode(kwargs.pop('encoding', None) or 'utf-8')

+        encoding = kwargs.pop('encoding', None)

+        if encoding is None:

+            encoding = detect_encoding(s)

+        s = s.decode(encoding)

     return _json.loads(s, **kwargs)

diff --git a/flask/wrappers.py b/flask/wrappers.py

index 04bdcb5..3e600fc 100644

--- a/flask/wrappers.py

+++ b/flask/wrappers.py

@@ -144,17 +144,10 @@ class Request(RequestBase):

         if not (force or self.is_json):

             return None

-        # We accept a request charset against the specification as

-        # certain clients have been using this in the past.  This

-        # fits our general approach of being nice in what we accept

-        # and strict in what we send out.

-        request_charset = self.mimetype_params.get('charset')

+        data = _get_data(self, cache)

+

         try:

-            data = _get_data(self, cache)

-            if request_charset is not None:

-                rv = json.loads(data, encoding=request_charset)

-            else:

-                rv = json.loads(data)

+            rv = json.loads(data)

         except ValueError as e:

             if silent:

                 rv = None

diff --git a/tests/test_helpers.py b/tests/test_helpers.py

index 9320ef7..9990782 100644

--- a/tests/test_helpers.py

+++ b/tests/test_helpers.py

@@ -21,6 +21,8 @@ from werkzeug.datastructures import Range

 from werkzeug.exceptions import BadRequest, NotFound

 from werkzeug.http import parse_cache_control_header, parse_options_header

 from werkzeug.http import http_date

+

+from flask import json

 from flask._compat import StringIO, text_type

@@ -34,6 +36,20 @@ def has_encoding(name):

 class TestJSON(object):

+    @pytest.mark.parametrize('value', (

+        1, 't', True, False, None,

+        [], [1, 2, 3],

+        {}, {'foo': u'🐍'},

+    ))

+    @pytest.mark.parametrize('encoding', (

+        'utf-8', 'utf-8-sig',

+        'utf-16-le', 'utf-16-be', 'utf-16',

+        'utf-32-le', 'utf-32-be', 'utf-32',

+    ))

+    def test_detect_encoding(self, value, encoding):

+        data = json.dumps(value).encode(encoding)

+        assert json.detect_encoding(data) == encoding

+        assert json.loads(data) == value

     def test_ignore_cached_json(self):

         app = flask.Flask(__name__)

@@ -85,18 +101,6 @@ class TestJSON(object):

         rv = c.post('/json', data='"foo"', content_type='application/x+json')

         assert rv.data == b'foo'

-    def test_json_body_encoding(self):

-        app = flask.Flask(__name__)

-        app.testing = True

-        @app.route('/')

-        def index():

-            return flask.request.get_json()

-

-        c = app.test_client()

-        resp = c.get('/', data=u'"Hällo Wörld"'.encode('iso-8859-15'),

-                     content_type='application/json; charset=iso-8859-15')

-        assert resp.data == u'Hällo Wörld'.encode('utf-8')

-

     def test_json_as_unicode(self):

         app = flask.Flask(__name__)

-- 

2.17.1